From b7c6e6142871d792399e804a482722bb0eadeccf Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sun, 17 Mar 2019 18:41:19 +0100 Subject: [PATCH 01/74] fix: pruning HID devices after 10 minutes of inactivity --- modules/hid/hid_recon.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/modules/hid/hid_recon.go b/modules/hid/hid_recon.go index fee44273..4cf6b650 100644 --- a/modules/hid/hid_recon.go +++ b/modules/hid/hid_recon.go @@ -61,6 +61,25 @@ func (mod *HIDRecon) onDeviceDetected(buf []byte) { } } +var maxDeviceTTL = 10 * time.Minute + +func (mod *HIDRecon) devPruner() { + mod.waitGroup.Add(1) + defer mod.waitGroup.Done() + + mod.Debug("devices pruner started.") + for mod.Running() { + for _, dev := range mod.Session.HID.Devices() { + sinceLastSeen := time.Since(dev.LastSeen) + if sinceLastSeen > maxDeviceTTL { + mod.Debug("device %s not seen in %s, removing.", dev.Address, sinceLastSeen) + mod.Session.HID.Remove(dev.Address) + } + } + time.Sleep(30 * time.Second) + } +} + func (mod *HIDRecon) Start() error { if err := mod.Configure(); err != nil { return err @@ -70,6 +89,8 @@ func (mod *HIDRecon) Start() error { mod.waitGroup.Add(1) defer mod.waitGroup.Done() + go mod.devPruner() + mod.Info("hopping on %d channels every %s", nrf24.TopChannel, mod.hopPeriod) for mod.Running() { if mod.isSniffing() { From 756c04fd95af1ab1a254b1078d47b7a7142b07fc Mon Sep 17 00:00:00 2001 From: evilsocket Date: Mon, 18 Mar 2019 12:07:00 +0100 Subject: [PATCH 02/74] new: module can now export a State map with specific information --- modules/ble/ble_recon.go | 4 ++++ modules/wifi/wifi.go | 23 +++++++++++++++++---- session/module.go | 44 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/modules/ble/ble_recon.go b/modules/ble/ble_recon.go index 1bea0d99..d6a5cddd 100644 --- a/modules/ble/ble_recon.go +++ b/modules/ble/ble_recon.go @@ -42,6 +42,8 @@ func NewBLERecon(s *session.Session) *BLERecon { connected: false, } + mod.InitState("scanning") + mod.selector = utils.ViewSelectorFor(&mod.SessionModule, "ble.show", []string{"rssi", "mac", "seen"}, "rssi asc") @@ -196,6 +198,7 @@ func (mod *BLERecon) Stop() error { mod.Debug("module stopped, cleaning state") mod.gattDevice = nil mod.setCurrentDevice(nil) + mod.ResetState() }) } @@ -216,6 +219,7 @@ func (mod *BLERecon) pruner() { func (mod *BLERecon) setCurrentDevice(dev *network.BLEDevice) { mod.connected = false mod.currDevice = dev + mod.State.Store("scanning", dev) } func (mod *BLERecon) writeBuffer(mac string, uuid gatt.UUID, data []byte) error { diff --git a/modules/wifi/wifi.go b/modules/wifi/wifi.go index bdd513d5..c5d5ab90 100644 --- a/modules/wifi/wifi.go +++ b/modules/wifi/wifi.go @@ -81,6 +81,8 @@ func NewWiFiModule(s *session.Session) *WiFiModule { chanLock: &sync.Mutex{}, } + mod.InitState("channels") + mod.AddParam(session.NewStringParameter("wifi.interface", "", "", @@ -124,7 +126,8 @@ func NewWiFiModule(s *session.Session) *WiFiModule { func(args []string) (err error) { mod.ap = nil mod.stickChan = 0 - mod.frequencies, err = network.GetSupportedFrequencies(mod.iface.Name()) + freqs, err := network.GetSupportedFrequencies(mod.iface.Name()) + mod.setFrequencies(freqs) mod.hopChanges <- true return err })) @@ -285,8 +288,7 @@ func NewWiFiModule(s *session.Session) *WiFiModule { } } - mod.Debug("new frequencies: %v", freqs) - mod.frequencies = freqs + mod.setFrequencies(freqs) // if wifi.recon is not running, this would block forever if mod.Running() { @@ -330,6 +332,17 @@ const ( ErrIfaceNotUp = "Interface Not Up" ) +func (mod *WiFiModule) setFrequencies(freqs []int) { + mod.Debug("new frequencies: %v", freqs) + + mod.frequencies = freqs + channels := []int{} + for _, freq := range freqs { + channels = append(channels, network.Dot11Freq2Chan(freq)) + } + mod.State.Store("channels", channels) +} + func (mod *WiFiModule) Configure() error { var ifName string var hopPeriod int @@ -430,8 +443,10 @@ func (mod *WiFiModule) Configure() error { if mod.source == "" { // No channels setted, retrieve frequencies supported by the card if len(mod.frequencies) == 0 { - if mod.frequencies, err = network.GetSupportedFrequencies(ifName); err != nil { + if freqs, err := network.GetSupportedFrequencies(ifName); err != nil { return fmt.Errorf("error while getting supported frequencies of %s: %s", ifName, err) + } else { + mod.setFrequencies(freqs) } mod.Debug("wifi supported frequencies: %v", mod.frequencies) diff --git a/session/module.go b/session/module.go index 5accba3d..f8433551 100644 --- a/session/module.go +++ b/session/module.go @@ -1,6 +1,7 @@ package session import ( + "encoding/json" "fmt" "net" "strings" @@ -25,16 +26,23 @@ type Module interface { } type SessionModule struct { - Name string `json:"name"` - Session *Session `json:"-"` - Started bool `json:"started"` - StatusLock *sync.RWMutex `json:"-"` + Name string + Session *Session + Started bool + StatusLock *sync.RWMutex + State sync.Map handlers []ModuleHandler params map[string]*ModuleParam tag string } +type sessionModuleJSON struct { + Name string `json:"name"` + Started bool `json:"started"` + State map[string]interface{} `json:"state"` +} + func AsTag(name string) string { return fmt.Sprintf("%s ", tui.Wrap(tui.BACKLIGHTBLUE, tui.Wrap(tui.FOREBLACK, name))) } @@ -54,6 +62,34 @@ func NewSessionModule(name string, s *Session) SessionModule { return m } +func (m *SessionModule) InitState(keys ...string) { + for _, key := range keys { + m.State.Store(key, nil) + } +} + +func (m *SessionModule) ResetState() { + m.State.Range(func(k, v interface{}) bool { + m.State.Store(k, nil) + return true + }) +} + +func (m *SessionModule) MarshalJSON() ([]byte, error) { + doc := sessionModuleJSON{ + Name: m.Name, + Started: m.Started, + State: make(map[string]interface{}), + } + + m.State.Range(func(k, v interface{}) bool { + doc.State[k.(string)] = v + return true + }) + + return json.Marshal(doc) +} + func (m *SessionModule) Debug(format string, args ...interface{}) { m.Session.Events.Log(log.DEBUG, m.tag+format, args...) } From ba4793f980b2238ea11ad1dd7bd9e2a45c9222c9 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Mon, 18 Mar 2019 12:23:55 +0100 Subject: [PATCH 03/74] misc: small fix or general refactoring i did not bother commenting --- session/module.go | 60 ++++++++++++++++++++++++++++++---------------- session/session.go | 27 --------------------- 2 files changed, 39 insertions(+), 48 deletions(-) diff --git a/session/module.go b/session/module.go index f8433551..dbdf6d1b 100644 --- a/session/module.go +++ b/session/module.go @@ -20,11 +20,41 @@ type Module interface { Handlers() []ModuleHandler Parameters() map[string]*ModuleParam + Extra() map[string]interface{} Running() bool Start() error Stop() error } +type ModuleList []Module + +type moduleJSON struct { + Name string `json:"name"` + Description string `json:"description"` + Author string `json:"author"` + Parameters map[string]*ModuleParam `json:"parameters"` + Handlers []ModuleHandler `json:"handlers"` + Running bool `json:"running"` + State map[string]interface{} `json:"state"` +} + +func (mm ModuleList) MarshalJSON() ([]byte, error) { + mods := []moduleJSON{} + for _, m := range mm { + mJSON := moduleJSON{ + Name: m.Name(), + Description: m.Description(), + Author: m.Author(), + Parameters: m.Parameters(), + Handlers: m.Handlers(), + Running: m.Running(), + State: m.Extra(), + } + mods = append(mods, mJSON) + } + return json.Marshal(mods) +} + type SessionModule struct { Name string Session *Session @@ -37,12 +67,6 @@ type SessionModule struct { tag string } -type sessionModuleJSON struct { - Name string `json:"name"` - Started bool `json:"started"` - State map[string]interface{} `json:"state"` -} - func AsTag(name string) string { return fmt.Sprintf("%s ", tui.Wrap(tui.BACKLIGHTBLUE, tui.Wrap(tui.FOREBLACK, name))) } @@ -62,6 +86,15 @@ func NewSessionModule(name string, s *Session) SessionModule { return m } +func (m *SessionModule) Extra() map[string]interface{} { + extra := make(map[string]interface{}) + m.State.Range(func(k, v interface{}) bool { + extra[k.(string)] = v + return true + }) + return extra +} + func (m *SessionModule) InitState(keys ...string) { for _, key := range keys { m.State.Store(key, nil) @@ -75,21 +108,6 @@ func (m *SessionModule) ResetState() { }) } -func (m *SessionModule) MarshalJSON() ([]byte, error) { - doc := sessionModuleJSON{ - Name: m.Name, - Started: m.Started, - State: make(map[string]interface{}), - } - - m.State.Range(func(k, v interface{}) bool { - doc.State[k.(string)] = v - return true - }) - - return json.Marshal(doc) -} - func (m *SessionModule) Debug(format string, args ...interface{}) { m.Session.Events.Log(log.DEBUG, m.tag+format, args...) } diff --git a/session/session.go b/session/session.go index aa5e96c0..e80ee4e8 100644 --- a/session/session.go +++ b/session/session.go @@ -1,7 +1,6 @@ package session import ( - "encoding/json" "errors" "fmt" "net" @@ -45,32 +44,6 @@ var ( type UnknownCommandCallback func(cmd string) bool -type ModuleList []Module - -type JSONModule struct { - Name string `json:"name"` - Description string `json:"description"` - Author string `json:"author"` - Parameters map[string]*ModuleParam `json:"parameters"` - Handlers []ModuleHandler `json:"handlers"` - Running bool `json:"running"` -} - -func (mm ModuleList) MarshalJSON() ([]byte, error) { - mods := []JSONModule{} - for _, m := range mm { - mods = append(mods, JSONModule{ - Name: m.Name(), - Description: m.Description(), - Author: m.Author(), - Parameters: m.Parameters(), - Handlers: m.Handlers(), - Running: m.Running(), - }) - } - return json.Marshal(mods) -} - type GPS struct { Latitude float64 // Latitude. Longitude float64 // Longitude. From 49aeb37b5c279b4abd40e2b8f620e5866e03d7e8 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Mon, 18 Mar 2019 12:59:39 +0100 Subject: [PATCH 04/74] fix: exposing session modules as a map in api.rest for quick lookup --- session/module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/session/module.go b/session/module.go index dbdf6d1b..f7ec2064 100644 --- a/session/module.go +++ b/session/module.go @@ -39,7 +39,7 @@ type moduleJSON struct { } func (mm ModuleList) MarshalJSON() ([]byte, error) { - mods := []moduleJSON{} + mods := make(map[string]moduleJSON) for _, m := range mm { mJSON := moduleJSON{ Name: m.Name(), @@ -50,7 +50,7 @@ func (mm ModuleList) MarshalJSON() ([]byte, error) { Running: m.Running(), State: m.Extra(), } - mods = append(mods, mJSON) + mods[m.Name()] = mJSON } return json.Marshal(mods) } From a5116d353319be0283ca62ae4047a4bd510e4c6f Mon Sep 17 00:00:00 2001 From: evilsocket Date: Mon, 18 Mar 2019 13:12:42 +0100 Subject: [PATCH 05/74] revert --- session/module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/session/module.go b/session/module.go index f7ec2064..dbdf6d1b 100644 --- a/session/module.go +++ b/session/module.go @@ -39,7 +39,7 @@ type moduleJSON struct { } func (mm ModuleList) MarshalJSON() ([]byte, error) { - mods := make(map[string]moduleJSON) + mods := []moduleJSON{} for _, m := range mm { mJSON := moduleJSON{ Name: m.Name(), @@ -50,7 +50,7 @@ func (mm ModuleList) MarshalJSON() ([]byte, error) { Running: m.Running(), State: m.Extra(), } - mods[m.Name()] = mJSON + mods = append(mods, mJSON) } return json.Marshal(mods) } From 09c09e647b2a2221ecb0b909ed25a2c5a8c2938c Mon Sep 17 00:00:00 2001 From: evilsocket Date: Mon, 18 Mar 2019 22:16:32 +0100 Subject: [PATCH 06/74] misc: small fix or general refactoring i did not bother commenting --- modules/net_recon/net_recon.go | 4 ++-- modules/syn_scan/syn_scan.go | 2 +- modules/wifi/wifi.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/net_recon/net_recon.go b/modules/net_recon/net_recon.go index 3ec71168..5208a010 100644 --- a/modules/net_recon/net_recon.go +++ b/modules/net_recon/net_recon.go @@ -48,13 +48,13 @@ func NewDiscovery(s *session.Session) *Discovery { })) mod.AddHandler(session.NewModuleHandler("net.show ADDRESS1, ADDRESS2", `net.show (.+)`, - "Show information about a specific list of addresses (by IP or MAC).", + "Show information about a specific comma separated list of addresses (by IP or MAC).", func(args []string) error { return mod.Show(args[0]) })) mod.AddHandler(session.NewModuleHandler("net.show.meta ADDRESS1, ADDRESS2", `net\.show\.meta (.+)`, - "Show meta information about a specific list of addresses (by IP or MAC).", + "Show meta information about a specific comma separated list of addresses (by IP or MAC).", func(args []string) error { return mod.showMeta(args[0]) })) diff --git a/modules/syn_scan/syn_scan.go b/modules/syn_scan/syn_scan.go index 76d36dda..b113208a 100644 --- a/modules/syn_scan/syn_scan.go +++ b/modules/syn_scan/syn_scan.go @@ -58,7 +58,7 @@ func NewSynScanner(s *session.Session) *SynScanner { return mod.Stop() })) - mod.AddHandler(session.NewModuleHandler("syn.scan IP-RANGE [START-PORT] [END-PORT]", "syn.scan ([^\\s]+) ?(\\d+)?([\\s\\d]*)?", + mod.AddHandler(session.NewModuleHandler("syn.scan IP-RANGE START-PORT END-PORT", "syn.scan ([^\\s]+) ?(\\d+)?([\\s\\d]*)?", "Perform a syn port scanning against an IP address within the provided ports range.", func(args []string) error { period := 0 diff --git a/modules/wifi/wifi.go b/modules/wifi/wifi.go index c5d5ab90..50b17ed9 100644 --- a/modules/wifi/wifi.go +++ b/modules/wifi/wifi.go @@ -261,7 +261,7 @@ func NewWiFiModule(s *session.Session) *WiFiModule { "false", "If true, wifi.show will also show the devices manufacturers.")) - mod.AddHandler(session.NewModuleHandler("wifi.recon.channel", `wifi\.recon\.channel[\s]+([0-9]+(?:[, ]+[0-9]+)*|clear)`, + mod.AddHandler(session.NewModuleHandler("wifi.recon.channel CHANNEL", `wifi\.recon\.channel[\s]+([0-9]+(?:[, ]+[0-9]+)*|clear)`, "WiFi channels (comma separated) or 'clear' for channel hopping.", func(args []string) (err error) { freqs := []int{} From d7b6cdb8a1b29341f6a98ce9440f840203bc6cbd Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 00:40:28 +0100 Subject: [PATCH 07/74] fix: printing path for unauthorized attempts to request api.rest routes --- modules/api_rest/api_rest_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/api_rest/api_rest_controller.go b/modules/api_rest/api_rest_controller.go index e9765f19..2e03bfc8 100644 --- a/modules/api_rest/api_rest_controller.go +++ b/modules/api_rest/api_rest_controller.go @@ -22,7 +22,7 @@ type APIResponse struct { } func (mod *RestAPI) setAuthFailed(w http.ResponseWriter, r *http.Request) { - mod.Warning("Unauthorized authentication attempt from %s", r.RemoteAddr) + mod.Warning("Unauthorized authentication attempt from %s to %s", r.RemoteAddr, r.URL.String()) w.Header().Set("WWW-Authenticate", `Basic realm="auth"`) w.WriteHeader(401) From 51667a039fbd56fd14d451777f787244574294e9 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 01:12:36 +0100 Subject: [PATCH 08/74] fix: gracefully handling http.server starting errors --- modules/http_server/http_server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/http_server/http_server.go b/modules/http_server/http_server.go index f03aff73..a398dfbb 100644 --- a/modules/http_server/http_server.go +++ b/modules/http_server/http_server.go @@ -110,7 +110,8 @@ func (mod *HttpServer) Start() error { var err error mod.Info("starting on http://%s", mod.server.Addr) if err = mod.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - panic(err) + mod.Error("%v", err) + mod.Stop() } }) } From c830e643098c01556d1ad4bb1a6371b2b01bd278 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 01:13:30 +0100 Subject: [PATCH 09/74] fix: gracefully handling https.server starting errors --- modules/https_server/https_server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/https_server/https_server.go b/modules/https_server/https_server.go index 0ed5e96a..ae26c009 100644 --- a/modules/https_server/https_server.go +++ b/modules/https_server/https_server.go @@ -159,7 +159,8 @@ func (mod *HttpsServer) Start() error { return mod.SetRunning(true, func() { mod.Info("starting on https://%s", mod.server.Addr) if err := mod.server.ListenAndServeTLS(mod.certFile, mod.keyFile); err != nil && err != http.ErrServerClosed { - panic(err) + mod.Error("%v", err) + mod.Stop() } }) } From 1cb5e821865ae0bc7e5efa5ee8adf673e1ce686f Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 03:03:03 +0100 Subject: [PATCH 10/74] fix: checking dongle pointer in hid.recon off --- modules/hid/hid.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/hid/hid.go b/modules/hid/hid.go index 74897f95..8ac9c09a 100644 --- a/modules/hid/hid.go +++ b/modules/hid/hid.go @@ -203,7 +203,9 @@ func (mod *HIDRecon) Configure() error { func (mod *HIDRecon) Stop() error { return mod.SetRunning(false, func() { mod.waitGroup.Wait() - mod.dongle.Close() - mod.Debug("device closed") + if mod.dongle != nil { + mod.dongle.Close() + mod.Debug("device closed") + } }) } From e7606589f34f079f76188fcde26ee334fec84d23 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 11:21:45 +0100 Subject: [PATCH 11/74] new: syn.scan now exposes progress via api.rest --- modules/syn_scan/syn_scan.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/syn_scan/syn_scan.go b/modules/syn_scan/syn_scan.go index b113208a..95e683b5 100644 --- a/modules/syn_scan/syn_scan.go +++ b/modules/syn_scan/syn_scan.go @@ -45,6 +45,9 @@ func NewSynScanner(s *session.Session) *SynScanner { progressEvery: time.Duration(1) * time.Second, } + mod.State.Store("scanning", &mod.addresses) + mod.State.Store("progress", 0.0) + mod.AddParam(session.NewIntParameter("syn.scan.show-progress-every", "1", "Period in seconds for the scanning progress reporting.")) @@ -155,6 +158,7 @@ func plural(n uint64) string { func (mod *SynScanner) showProgress() error { progress := 100.0 * (float64(mod.stats.doneProbes) / float64(mod.stats.totProbes)) + mod.State.Store("progress", progress) mod.Info("[%.2f%%] found %d open port%s for %d address%s, sent %d/%d packets in %s", progress, mod.stats.openPorts, @@ -172,12 +176,17 @@ func (mod *SynScanner) Stop() error { return mod.SetRunning(false, func() { mod.waitGroup.Wait() mod.showProgress() + mod.addresses = []net.IP{} + mod.State.Store("progress", 0.0) }) } func (mod *SynScanner) synScan() error { mod.SetRunning(true, func() { - defer mod.SetRunning(false, nil) + defer mod.SetRunning(false, func() { + mod.addresses = []net.IP{} + mod.State.Store("progress", 0.0) + }) mod.waitGroup.Add(1) defer mod.waitGroup.Done() @@ -199,6 +208,8 @@ func (mod *SynScanner) synScan() error { mod.Info("scanning %d address%s on port %d ...", mod.stats.numAddresses, plural, mod.startPort) } + mod.State.Store("progress", 0.0) + // set the collector mod.Session.Queue.OnPacket(mod.onPacket) defer mod.Session.Queue.OnPacket(nil) From e1558413b2cd4c28e51c181c57c9b88198d0d93c Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 11:29:06 +0100 Subject: [PATCH 12/74] new: the hid module now exposes its status via api.rest --- modules/hid/hid.go | 3 +++ modules/hid/hid_sniff.go | 1 + 2 files changed, 4 insertions(+) diff --git a/modules/hid/hid.go b/modules/hid/hid.go index 8ac9c09a..2c1d2069 100644 --- a/modules/hid/hid.go +++ b/modules/hid/hid.go @@ -63,6 +63,9 @@ func NewHIDRecon(s *session.Session) *HIDRecon { scriptPath: "", } + mod.State.Store("sniffing", &mod.sniffAddr) + mod.State.Store("injecting", &mod.inInjectMode) + mod.AddHandler(session.NewModuleHandler("hid.recon on", "", "Start scanning for HID devices on the 2.4Ghz spectrum.", func(args []string) error { diff --git a/modules/hid/hid_sniff.go b/modules/hid/hid_sniff.go index 98198359..a6fd7ec4 100644 --- a/modules/hid/hid_sniff.go +++ b/modules/hid/hid_sniff.go @@ -41,6 +41,7 @@ func (mod *HIDRecon) setSniffMode(mode string, silent bool) error { mod.sniffAddrRaw = raw } } + return nil } From 3b4432f072b28b4ec0310a7a9aee207fb90fc553 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 13:29:23 +0100 Subject: [PATCH 13/74] new: new /api/file route for api.rest to read and write files --- modules/api_rest/api_rest.go | 1 + modules/api_rest/api_rest_controller.go | 61 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 11da44aa..444e422d 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -191,6 +191,7 @@ func (mod *RestAPI) Configure() error { router.HandleFunc("/api/session/started-at", mod.sessionRoute) router.HandleFunc("/api/session/wifi", mod.sessionRoute) router.HandleFunc("/api/session/wifi/{mac}", mod.sessionRoute) + router.HandleFunc("/api/file", mod.fileRoute) mod.server.Handler = router diff --git a/modules/api_rest/api_rest_controller.go b/modules/api_rest/api_rest_controller.go index 2e03bfc8..0d7f61f0 100644 --- a/modules/api_rest/api_rest_controller.go +++ b/modules/api_rest/api_rest_controller.go @@ -3,7 +3,11 @@ package api_rest import ( "crypto/subtle" "encoding/json" + "fmt" + "io" + "io/ioutil" "net/http" + "os" "strconv" "strings" @@ -258,6 +262,44 @@ func (mod *RestAPI) sessionRoute(w http.ResponseWriter, r *http.Request) { } } +func (mod *RestAPI) readFile(fileName string, w http.ResponseWriter, r *http.Request) { + fp, err := os.Open(fileName) + if err != nil { + msg := fmt.Sprintf("could not open %s for reading: %s", fileName, err) + mod.Debug(msg) + http.Error(w, msg, 404) + return + } + defer fp.Close() + + w.Header().Set("Content-type", "application/octet-stream") + + io.Copy(w, fp) +} + +func (mod *RestAPI) writeFile(fileName string, w http.ResponseWriter, r *http.Request) { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + msg := fmt.Sprintf("invalid file upload: %s", err) + mod.Warning(msg) + http.Error(w, msg, 404) + return + } + + err = ioutil.WriteFile(fileName, data, 0666) + if err != nil { + msg := fmt.Sprintf("can't write to %s: %s", fileName, err) + mod.Warning(msg) + http.Error(w, msg, 404) + return + } + + mod.toJSON(w, APIResponse{ + Success: true, + Message: fmt.Sprintf("%s created", fileName), + }) +} + func (mod *RestAPI) eventsRoute(w http.ResponseWriter, r *http.Request) { mod.setSecurityHeaders(w) @@ -274,3 +316,22 @@ func (mod *RestAPI) eventsRoute(w http.ResponseWriter, r *http.Request) { http.Error(w, "Bad Request", 400) } } + +func (mod *RestAPI) fileRoute(w http.ResponseWriter, r *http.Request) { + mod.setSecurityHeaders(w) + + if !mod.checkAuth(r) { + mod.setAuthFailed(w, r) + return + } + + fileName := r.URL.Query().Get("name") + + if fileName != "" && r.Method == "GET" { + mod.readFile(fileName, w, r) + } else if fileName != "" && r.Method == "POST" { + mod.writeFile(fileName, w, r) + } else { + http.Error(w, "Bad Request", 400) + } +} From af1bfb3894a0d0cbbb2bd8ab99d57d44167ca70d Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 13:41:06 +0100 Subject: [PATCH 14/74] new: hid module now exposes the available layouts via api.rest --- modules/hid/hid.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/hid/hid.go b/modules/hid/hid.go index 2c1d2069..134f8cb2 100644 --- a/modules/hid/hid.go +++ b/modules/hid/hid.go @@ -65,6 +65,7 @@ func NewHIDRecon(s *session.Session) *HIDRecon { mod.State.Store("sniffing", &mod.sniffAddr) mod.State.Store("injecting", &mod.inInjectMode) + mod.State.Store("layouts", SupportedLayouts()) mod.AddHandler(session.NewModuleHandler("hid.recon on", "", "Start scanning for HID devices on the 2.4Ghz spectrum.", From 3e8d6512e1eaa08a1331c12d2ba3d80a0ad4b87c Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 13:55:23 +0100 Subject: [PATCH 15/74] fix: using fallback hid.device.type if the device was detected but not its type --- modules/hid/hid_inject.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/hid/hid_inject.go b/modules/hid/hid_inject.go index f5e10c3a..fbf90ddc 100644 --- a/modules/hid/hid_inject.go +++ b/modules/hid/hid_inject.go @@ -55,13 +55,10 @@ func (mod *HIDRecon) prepInjection() (error, *network.HIDDevice, []*Command) { } var builder FrameBuilder - if found { + if found && dev.Type != network.HIDTypeUnknown { // get the device specific protocol handler builder, found = FrameBuilders[dev.Type] if found == false { - if dev.Type == network.HIDTypeUnknown { - return errNoType(mod.sniffAddr), nil, nil - } return errNotSupported(dev), nil, nil } } else { From 39f9a02c98a4118120f753d475ae65fc9ddd06e6 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 13:57:34 +0100 Subject: [PATCH 16/74] fix: increased hid pruning time to 20 minutes --- modules/hid/hid_inject.go | 2 ++ modules/hid/hid_recon.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/hid/hid_inject.go b/modules/hid/hid_inject.go index fbf90ddc..bcba94c5 100644 --- a/modules/hid/hid_inject.go +++ b/modules/hid/hid_inject.go @@ -52,6 +52,8 @@ func (mod *HIDRecon) prepInjection() (error, *network.HIDDevice, []*Command) { dev, found := mod.Session.HID.Get(mod.sniffAddr) if found == false { mod.Warning("device %s is not visible, will use HID type %s", mod.sniffAddr, tui.Yellow(mod.sniffType)) + } else if dev.Type == network.HIDTypeUnknown { + mod.Warning("device %s type has not been detected yet, falling back to '%s'", mod.sniffAddr, tui.Yellow(mod.sniffType)) } var builder FrameBuilder diff --git a/modules/hid/hid_recon.go b/modules/hid/hid_recon.go index 4cf6b650..b4771a03 100644 --- a/modules/hid/hid_recon.go +++ b/modules/hid/hid_recon.go @@ -61,7 +61,7 @@ func (mod *HIDRecon) onDeviceDetected(buf []byte) { } } -var maxDeviceTTL = 10 * time.Minute +var maxDeviceTTL = 20 * time.Minute func (mod *HIDRecon) devPruner() { mod.waitGroup.Add(1) From 6b9b27302e7c7ed88c89101c620a23e91260860a Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 14:59:08 +0100 Subject: [PATCH 17/74] new: hid devices now exports the last 50 sniffed payloads via api.rest --- network/hid_device.go | 46 ++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/network/hid_device.go b/network/hid_device.go index 1562f9e1..8104bd6b 100644 --- a/network/hid_device.go +++ b/network/hid_device.go @@ -1,12 +1,15 @@ package network import ( + "encoding/hex" "encoding/json" "fmt" "sort" "strings" "sync" "time" + + "github.com/evilsocket/islazy/str" ) type HIDType int @@ -47,10 +50,12 @@ type HIDDevice struct { } type hidDeviceJSON struct { - LastSeen time.Time `json:"last_seen"` - Type string `json:"type"` - Address string `json:"address"` - Channels []string `json:"channels"` + LastSeen time.Time `json:"last_seen"` + Type string `json:"type"` + Address string `json:"address"` + Channels []string `json:"channels"` + Payloads []string `json:"payloads"` + PayloadsSize uint64 `json:"payloads_size"` } func NormalizeHIDAddress(address string) string { @@ -90,12 +95,27 @@ func NewHIDDevice(address []byte, channel int, payload []byte) *HIDDevice { } func (dev *HIDDevice) MarshalJSON() ([]byte, error) { + dev.Lock() + defer dev.Unlock() + doc := hidDeviceJSON{ - LastSeen: dev.LastSeen, - Type: dev.Type.String(), - Address: dev.Address, - Channels: dev.ChannelsList(), + LastSeen: dev.LastSeen, + Type: dev.Type.String(), + Address: dev.Address, + Channels: dev.channelsListUnlocked(), + Payloads: make([]string, 0), + PayloadsSize: dev.payloadsSz, } + + // get the latest 50 payloads + for i := len(dev.payloads) - 1; i >= 0; i-- { + data := str.Trim(hex.Dump(dev.payloads[i])) + doc.Payloads = append([]string{data}, doc.Payloads...) + if len(doc.Payloads) == 50 { + break + } + } + return json.Marshal(doc) } @@ -106,10 +126,7 @@ func (dev *HIDDevice) AddChannel(ch int) { dev.channels[ch] = true } -func (dev *HIDDevice) ChannelsList() []string { - dev.Lock() - defer dev.Unlock() - +func (dev *HIDDevice) channelsListUnlocked() []string { chans := []string{} for ch := range dev.channels { chans = append(chans, fmt.Sprintf("%d", ch)) @@ -119,6 +136,11 @@ func (dev *HIDDevice) ChannelsList() []string { return chans } +func (dev *HIDDevice) ChannelsList() []string { + dev.Lock() + defer dev.Unlock() + return dev.channelsListUnlocked() +} func (dev *HIDDevice) Channels() string { return strings.Join(dev.ChannelsList(), ",") From 3edf80fc991e0ead704870a563490b6416cc8bab Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 19 Mar 2019 19:10:40 +0100 Subject: [PATCH 18/74] new: preloading and exporting caplets from api.rest --- caplets/caplet.go | 17 +++++++++++++---- caplets/manager.go | 29 ++++++++++++++++------------- session/session.go | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/caplets/caplet.go b/caplets/caplet.go index b36c1a8a..8a876a4d 100644 --- a/caplets/caplet.go +++ b/caplets/caplet.go @@ -9,10 +9,19 @@ import ( ) type Caplet struct { - Name string - Path string - Size int64 - Code []string + Name string `json:"name"` + Path string `json:"path"` + Size int64 `json:"size"` + Code []string `json:"code"` +} + +func NewCaplet(name string, path string, size int64) Caplet { + return Caplet{ + Name: name, + Path: path, + Size: size, + Code: make([]string, 0), + } } func (cap *Caplet) Eval(argv []string, lineCb func(line string) error) error { diff --git a/caplets/manager.go b/caplets/manager.go index 88620c61..a3f3ebe7 100644 --- a/caplets/manager.go +++ b/caplets/manager.go @@ -16,22 +16,22 @@ var ( cacheLock = sync.Mutex{} ) -func List() []Caplet { - caplets := make([]Caplet, 0) +func List() []*Caplet { + caplets := make([]*Caplet, 0) for _, searchPath := range LoadPaths { files, _ := filepath.Glob(searchPath + "/*" + Suffix) files2, _ := filepath.Glob(searchPath + "/*/*" + Suffix) for _, fileName := range append(files, files2...) { - if stats, err := os.Stat(fileName); err == nil { + if _, err := os.Stat(fileName); err == nil { base := strings.Replace(fileName, searchPath+"/", "", -1) base = strings.Replace(base, Suffix, "", -1) - caplets = append(caplets, Caplet{ - Name: base, - Path: fileName, - Size: stats.Size(), - }) + if err, caplet := Load(base); err != nil { + fmt.Fprintf(os.Stderr, "wtf: %v\n", err) + } else { + caplets = append(caplets, caplet) + } } } } @@ -51,6 +51,7 @@ func Load(name string) (error, *Caplet) { return nil, caplet } + baseName := name names := []string{} if !strings.HasSuffix(name, Suffix) { name += Suffix @@ -64,16 +65,18 @@ func Load(name string) (error, *Caplet) { names = append(names, name) } - for _, filename := range names { - if fs.Exists(filename) { + for _, fileName := range names { + if stats, err := os.Stat(fileName); err == nil { cap := &Caplet{ - Path: filename, + Name: baseName, + Path: fileName, Code: make([]string, 0), + Size: stats.Size(), } cache[name] = cap - if reader, err := fs.LineReader(filename); err != nil { - return fmt.Errorf("error reading caplet %s: %v", filename, err), nil + if reader, err := fs.LineReader(fileName); err != nil { + return fmt.Errorf("error reading caplet %s: %v", fileName, err), nil } else { for line := range reader { if line == "" || line[0] == '#' { diff --git a/session/session.go b/session/session.go index e80ee4e8..efdfd579 100644 --- a/session/session.go +++ b/session/session.go @@ -1,6 +1,7 @@ package session import ( + "encoding/json" "errors" "fmt" "net" @@ -77,6 +78,23 @@ type Session struct { Firewall firewall.FirewallManager `json:"-"` } +type sessionJSON struct { + Options core.Options `json:"options"` + Interface *network.Endpoint `json:"interface"` + Gateway *network.Endpoint `json:"gateway"` + Env *Environment `json:"env"` + Lan *network.LAN `json:"lan"` + WiFi *network.WiFi `json:"wifi"` + BLE *network.BLE `json:"ble"` + HID *network.HID `json:"hid"` + Queue *packets.Queue `json:"packets"` + StartedAt time.Time `json:"started_at"` + Active bool `json:"active"` + GPS GPS `json:"gps"` + Modules ModuleList `json:"modules"` + Caplets []*caplets.Caplet `json:"caplets"` +} + func New() (*Session, error) { opts, err := core.ParseOptions() if err != nil { @@ -124,6 +142,26 @@ func New() (*Session, error) { return s, nil } +func (s *Session) MarshalJSON() ([]byte, error) { + doc := sessionJSON{ + Options: s.Options, + Interface: s.Interface, + Gateway: s.Gateway, + Env: s.Env, + Lan: s.Lan, + WiFi: s.WiFi, + BLE: s.BLE, + HID: s.HID, + Queue: s.Queue, + StartedAt: s.StartedAt, + Active: s.Active, + GPS: s.GPS, + Modules: s.Modules, + Caplets: caplets.List(), + } + return json.Marshal(doc) +} + func (s *Session) Lock() { s.Env.Lock() s.Lan.Lock() From 5aa57f5cd7f4d8ad65ee74832aeee095eabd645f Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 20 Mar 2019 01:45:10 +0100 Subject: [PATCH 19/74] new: exporting version, os, arch, and goversion from api.rest --- session/session.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/session/session.go b/session/session.go index efdfd579..fffff4d0 100644 --- a/session/session.go +++ b/session/session.go @@ -79,6 +79,10 @@ type Session struct { } type sessionJSON struct { + Version string `json:"version"` + OS string `json:"os"` + Arch string `json:"arch"` + GoVersion string `json:"goversion"` Options core.Options `json:"options"` Interface *network.Endpoint `json:"interface"` Gateway *network.Endpoint `json:"gateway"` @@ -144,6 +148,10 @@ func New() (*Session, error) { func (s *Session) MarshalJSON() ([]byte, error) { doc := sessionJSON{ + Version: core.Version, + OS: runtime.GOOS, + Arch: runtime.GOARCH, + GoVersion: runtime.Version(), Options: s.Options, Interface: s.Interface, Gateway: s.Gateway, From 4f54b12ee1d5d8d82838653d272aea4cfb171442 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 20 Mar 2019 13:36:44 +0100 Subject: [PATCH 20/74] fix: fixed a bug which prevented 'set alias MAC ""' to clear an alias --- session/session_core_handlers.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/session/session_core_handlers.go b/session/session_core_handlers.go index 1a8fa83e..13f48419 100644 --- a/session/session_core_handlers.go +++ b/session/session_core_handlers.go @@ -274,6 +274,9 @@ func (s *Session) shHandler(args []string, sess *Session) error { func (s *Session) aliasHandler(args []string, sess *Session) error { mac := args[0] alias := str.Trim(args[1]) + if alias == "\"\"" || alias == "''" { + alias = "" + } s.Lan.SetAliasFor(mac, alias) return nil } From 72370dec580a3174ccc00c30da82e48158241e09 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 20 Mar 2019 13:48:34 +0100 Subject: [PATCH 21/74] new: exporting available network interfaces in /api/session for api.rest --- session/session.go | 84 ++++++++++------------------------------- session/session_json.go | 84 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 65 deletions(-) create mode 100644 session/session_json.go diff --git a/session/session.go b/session/session.go index fffff4d0..647eb684 100644 --- a/session/session.go +++ b/session/session.go @@ -1,7 +1,6 @@ package session import ( - "encoding/json" "errors" "fmt" "net" @@ -56,47 +55,26 @@ type GPS struct { } type Session struct { - Options core.Options `json:"options"` - Interface *network.Endpoint `json:"interface"` - Gateway *network.Endpoint `json:"gateway"` - Env *Environment `json:"env"` - Lan *network.LAN `json:"lan"` - WiFi *network.WiFi `json:"wifi"` - BLE *network.BLE `json:"ble"` - HID *network.HID `json:"hid"` - Queue *packets.Queue `json:"packets"` - StartedAt time.Time `json:"started_at"` - Active bool `json:"active"` - GPS GPS `json:"gps"` - Modules ModuleList `json:"modules"` + Options core.Options + Interface *network.Endpoint + Gateway *network.Endpoint + Env *Environment + Lan *network.LAN + WiFi *network.WiFi + BLE *network.BLE + HID *network.HID + Queue *packets.Queue + StartedAt time.Time + Active bool + GPS GPS + Modules ModuleList - Input *readline.Instance `json:"-"` - Prompt Prompt `json:"-"` - CoreHandlers []CommandHandler `json:"-"` - Events *EventPool `json:"-"` - UnkCmdCallback UnknownCommandCallback `json:"-"` - Firewall firewall.FirewallManager `json:"-"` -} - -type sessionJSON struct { - Version string `json:"version"` - OS string `json:"os"` - Arch string `json:"arch"` - GoVersion string `json:"goversion"` - Options core.Options `json:"options"` - Interface *network.Endpoint `json:"interface"` - Gateway *network.Endpoint `json:"gateway"` - Env *Environment `json:"env"` - Lan *network.LAN `json:"lan"` - WiFi *network.WiFi `json:"wifi"` - BLE *network.BLE `json:"ble"` - HID *network.HID `json:"hid"` - Queue *packets.Queue `json:"packets"` - StartedAt time.Time `json:"started_at"` - Active bool `json:"active"` - GPS GPS `json:"gps"` - Modules ModuleList `json:"modules"` - Caplets []*caplets.Caplet `json:"caplets"` + Input *readline.Instance + Prompt Prompt + CoreHandlers []CommandHandler + Events *EventPool + UnkCmdCallback UnknownCommandCallback + Firewall firewall.FirewallManager } func New() (*Session, error) { @@ -146,30 +124,6 @@ func New() (*Session, error) { return s, nil } -func (s *Session) MarshalJSON() ([]byte, error) { - doc := sessionJSON{ - Version: core.Version, - OS: runtime.GOOS, - Arch: runtime.GOARCH, - GoVersion: runtime.Version(), - Options: s.Options, - Interface: s.Interface, - Gateway: s.Gateway, - Env: s.Env, - Lan: s.Lan, - WiFi: s.WiFi, - BLE: s.BLE, - HID: s.HID, - Queue: s.Queue, - StartedAt: s.StartedAt, - Active: s.Active, - GPS: s.GPS, - Modules: s.Modules, - Caplets: caplets.List(), - } - return json.Marshal(doc) -} - func (s *Session) Lock() { s.Env.Lock() s.Lan.Lock() diff --git a/session/session_json.go b/session/session_json.go new file mode 100644 index 00000000..8a024771 --- /dev/null +++ b/session/session_json.go @@ -0,0 +1,84 @@ +package session + +import ( + "encoding/json" + "net" + "runtime" + "time" + + "github.com/bettercap/bettercap/caplets" + "github.com/bettercap/bettercap/core" + "github.com/bettercap/bettercap/network" + "github.com/bettercap/bettercap/packets" +) + +type ifaceJSON struct { + Index int `json:"index"` + MTU int `json:"mtu"` + Name string `json:"name"` + MAC string `json:"mac"` + Flags net.Flags `json:"flags"` +} + +type sessionJSON struct { + Version string `json:"version"` + OS string `json:"os"` + Arch string `json:"arch"` + GoVersion string `json:"goversion"` + Interfaces []ifaceJSON `json:"interfaces"` + Options core.Options `json:"options"` + Interface *network.Endpoint `json:"interface"` + Gateway *network.Endpoint `json:"gateway"` + Env *Environment `json:"env"` + Lan *network.LAN `json:"lan"` + WiFi *network.WiFi `json:"wifi"` + BLE *network.BLE `json:"ble"` + HID *network.HID `json:"hid"` + Queue *packets.Queue `json:"packets"` + StartedAt time.Time `json:"started_at"` + Active bool `json:"active"` + GPS GPS `json:"gps"` + Modules ModuleList `json:"modules"` + Caplets []*caplets.Caplet `json:"caplets"` +} + +func (s *Session) MarshalJSON() ([]byte, error) { + doc := sessionJSON{ + Version: core.Version, + OS: runtime.GOOS, + Arch: runtime.GOARCH, + GoVersion: runtime.Version(), + Interfaces: make([]ifaceJSON, 0), + Options: s.Options, + Interface: s.Interface, + Gateway: s.Gateway, + Env: s.Env, + Lan: s.Lan, + WiFi: s.WiFi, + BLE: s.BLE, + HID: s.HID, + Queue: s.Queue, + StartedAt: s.StartedAt, + Active: s.Active, + GPS: s.GPS, + Modules: s.Modules, + Caplets: caplets.List(), + } + + ifaces, err := net.Interfaces() + if err != nil { + return nil, err + } + + for _, iface := range ifaces { + doc.Interfaces = append(doc.Interfaces, ifaceJSON{ + Index: iface.Index, + MTU: iface.MTU, + Name: iface.Name, + MAC: iface.HardwareAddr.String(), + Flags: iface.Flags, + }) + } + + return json.Marshal(doc) +} From f8dda1e8e9f86f02f86fe3e9bac9941998e01114 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 20 Mar 2019 14:49:12 +0100 Subject: [PATCH 22/74] adding networking interfaces addresses for api.rest --- session/session_json.go | 60 ++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/session/session_json.go b/session/session_json.go index 8a024771..84bfe3db 100644 --- a/session/session_json.go +++ b/session/session_json.go @@ -12,12 +12,27 @@ import ( "github.com/bettercap/bettercap/packets" ) +var flagNames = []string{ + "UP", + "BROADCAST", + "LOOPBACK", + "POINT2POINT", + "MULTICAST", +} + +type addrJSON struct { + Address string `json:"address"` + Type string `json:"type"` +} + type ifaceJSON struct { - Index int `json:"index"` - MTU int `json:"mtu"` - Name string `json:"name"` - MAC string `json:"mac"` - Flags net.Flags `json:"flags"` + Index int `json:"index"` + MTU int `json:"mtu"` + Name string `json:"name"` + MAC string `json:"mac"` + Vendor string `json:"vendor"` + Flags []string `json:"flags"` + Addresses []addrJSON `json:"addresses"` } type sessionJSON struct { @@ -71,13 +86,34 @@ func (s *Session) MarshalJSON() ([]byte, error) { } for _, iface := range ifaces { - doc.Interfaces = append(doc.Interfaces, ifaceJSON{ - Index: iface.Index, - MTU: iface.MTU, - Name: iface.Name, - MAC: iface.HardwareAddr.String(), - Flags: iface.Flags, - }) + mac := network.NormalizeMac(iface.HardwareAddr.String()) + + ij := ifaceJSON{ + Index: iface.Index, + MTU: iface.MTU, + Name: iface.Name, + MAC: mac, + Vendor: network.ManufLookup(mac), + Flags: make([]string, 0), + Addresses: make([]addrJSON, 0), + } + + if addrs, err := iface.Addrs(); err == nil { + for _, addr := range addrs { + ij.Addresses = append(ij.Addresses, addrJSON{ + Address: addr.String(), + Type: addr.Network(), + }) + } + } + + for bit, name := range flagNames { + if iface.Flags&(1< Date: Wed, 20 Mar 2019 19:28:04 +0100 Subject: [PATCH 23/74] fix: now api.rest can execute multiple commands divided by ; --- modules/api_rest/api_rest_controller.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/api_rest/api_rest_controller.go b/modules/api_rest/api_rest_controller.go index 0d7f61f0..af087179 100644 --- a/modules/api_rest/api_rest_controller.go +++ b/modules/api_rest/api_rest_controller.go @@ -158,11 +158,16 @@ func (mod *RestAPI) runSessionCommand(w http.ResponseWriter, r *http.Request) { http.Error(w, "Bad Request", 400) } else if err = json.NewDecoder(r.Body).Decode(&cmd); err != nil { http.Error(w, "Bad Request", 400) - } else if err = session.I.Run(cmd.Command); err != nil { - http.Error(w, err.Error(), 400) - } else { - mod.toJSON(w, APIResponse{Success: true}) } + + for _, aCommand := range session.ParseCommands(cmd.Command) { + if err = mod.Session.Run(aCommand); err != nil { + http.Error(w, err.Error(), 400) + return + } + } + + mod.toJSON(w, APIResponse{Success: true}) } func (mod *RestAPI) showEvents(w http.ResponseWriter, r *http.Request) { From 0cfe6eebcc31b417aeadd376ff78d726e3a2c8ff Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 20 Mar 2019 21:16:34 +0100 Subject: [PATCH 24/74] fix: do not start arp.spoof if the list of targets is empty --- modules/arp_spoof/arp_spoof.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/arp_spoof/arp_spoof.go b/modules/arp_spoof/arp_spoof.go index 6dc5a573..507d38af 100644 --- a/modules/arp_spoof/arp_spoof.go +++ b/modules/arp_spoof/arp_spoof.go @@ -127,9 +127,14 @@ func (mod *ArpSpoofer) Start() error { return err } + nTargets := len(mod.addresses) + len(mod.macs) + if nTargets == 0 { + mod.Warning("list of targets is empty, module not starting.") + return nil + } + return mod.SetRunning(true, func() { neighbours := []net.IP{} - nTargets := len(mod.addresses) + len(mod.macs) if mod.internal { list, _ := iprange.ParseList(mod.Session.Interface.CIDR()) From f23c780eee9c085a5675075075ba6c2a55ff6ffb Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 10:57:16 +0100 Subject: [PATCH 25/74] fix: fixed a bug in ble.recon which sometimes caused a crash if ble.recon off was called but no device was present --- modules/ble/ble_recon.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/modules/ble/ble_recon.go b/modules/ble/ble_recon.go index d6a5cddd..b44c7746 100644 --- a/modules/ble/ble_recon.go +++ b/modules/ble/ble_recon.go @@ -173,18 +173,20 @@ func (mod *BLERecon) Start() error { <-mod.quit - mod.Info("stopping scan ...") + if mod.gattDevice != nil { + mod.Info("stopping scan ...") - if mod.currDevice != nil && mod.currDevice.Device != nil && mod.gattDevice != nil { - mod.Debug("resetting connection with %v", mod.currDevice.Device) - mod.gattDevice.CancelConnection(mod.currDevice.Device) - } + if mod.currDevice != nil && mod.currDevice.Device != nil { + mod.Debug("resetting connection with %v", mod.currDevice.Device) + mod.gattDevice.CancelConnection(mod.currDevice.Device) + } - mod.Debug("stopping device") - if err := mod.gattDevice.Stop(); err != nil { - mod.Warning("error while stopping device: %v", err) - } else { - mod.Debug("gatt device closed") + mod.Debug("stopping device") + if err := mod.gattDevice.Stop(); err != nil { + mod.Warning("error while stopping device: %v", err) + } else { + mod.Debug("gatt device closed") + } } mod.done <- true From 2e3e4f453b438272a972987df5f101592bcfccf6 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 11:20:48 +0100 Subject: [PATCH 26/74] fix: events.include and events.ignore now filter for both events.stream and api.rest --- modules/api_rest/api_rest.go | 4 +- modules/api_rest/api_rest_controller.go | 8 +++- modules/events_stream/events_stream.go | 20 ++++----- .../events_ignore_list.go | 42 ++++++++++--------- session/session.go | 22 +++++----- 5 files changed, 54 insertions(+), 42 deletions(-) rename modules/events_stream/ignore_list.go => session/events_ignore_list.go (63%) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 444e422d..04cb455d 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -174,7 +174,10 @@ func (mod *RestAPI) Configure() error { router.Methods("OPTIONS").HandlerFunc(mod.corsRoute) + router.HandleFunc("/api/file", mod.fileRoute) + router.HandleFunc("/api/events", mod.eventsRoute) + router.HandleFunc("/api/session", mod.sessionRoute) router.HandleFunc("/api/session/ble", mod.sessionRoute) router.HandleFunc("/api/session/ble/{mac}", mod.sessionRoute) @@ -191,7 +194,6 @@ func (mod *RestAPI) Configure() error { router.HandleFunc("/api/session/started-at", mod.sessionRoute) router.HandleFunc("/api/session/wifi", mod.sessionRoute) router.HandleFunc("/api/session/wifi/{mac}", mod.sessionRoute) - router.HandleFunc("/api/file", mod.fileRoute) mod.server.Handler = router diff --git a/modules/api_rest/api_rest_controller.go b/modules/api_rest/api_rest_controller.go index af087179..16738d89 100644 --- a/modules/api_rest/api_rest_controller.go +++ b/modules/api_rest/api_rest_controller.go @@ -176,7 +176,13 @@ func (mod *RestAPI) showEvents(w http.ResponseWriter, r *http.Request) { if mod.useWebsocket { mod.startStreamingEvents(w, r) } else { - events := session.I.Events.Sorted() + events := make([]session.Event, 0) + for _, e := range session.I.Events.Sorted() { + if mod.Session.EventsIgnoreList.Ignored(e) == false { + events = append(events, e) + } + } + nevents := len(events) nmax := nevents n := nmax diff --git a/modules/events_stream/events_stream.go b/modules/events_stream/events_stream.go index 517e4db1..8fc56466 100644 --- a/modules/events_stream/events_stream.go +++ b/modules/events_stream/events_stream.go @@ -29,7 +29,6 @@ type EventsStream struct { outputName string output *os.File rotation rotation - ignoreList *IgnoreList triggerList *TriggerList waitFor string waitChan chan *session.Event @@ -47,7 +46,6 @@ func NewEventsStream(s *session.Session) *EventsStream { quit: make(chan bool), waitChan: make(chan *session.Event), waitFor: "", - ignoreList: NewIgnoreList(), triggerList: NewTriggerList(), } @@ -127,7 +125,7 @@ func NewEventsStream(s *session.Session) *EventsStream { ignore := session.NewModuleHandler("events.ignore FILTER", "events.ignore ([^\\s]+)", "Events with an identifier matching this filter will not be shown (use multiple times to add more filters).", func(args []string) error { - return mod.ignoreList.Add(args[0]) + return mod.Session.EventsIgnoreList.Add(args[0]) }) ignore.Complete("events.ignore", s.EventsCompleter) @@ -137,7 +135,7 @@ func NewEventsStream(s *session.Session) *EventsStream { include := session.NewModuleHandler("events.include FILTER", "events.include ([^\\s]+)", "Used to remove filters passed with the events.ignore command.", func(args []string) error { - return mod.ignoreList.Remove(args[0]) + return mod.Session.EventsIgnoreList.Remove(args[0]) }) include.Complete("events.include", s.EventsCompleter) @@ -147,13 +145,13 @@ func NewEventsStream(s *session.Session) *EventsStream { mod.AddHandler(session.NewModuleHandler("events.filters", "", "Print the list of filters used to ignore events.", func(args []string) error { - if mod.ignoreList.Empty() { + if mod.Session.EventsIgnoreList.Empty() { fmt.Printf("Ignore filters list is empty.\n") } else { - mod.ignoreList.RLock() - defer mod.ignoreList.RUnlock() + mod.Session.EventsIgnoreList.RLock() + defer mod.Session.EventsIgnoreList.RUnlock() - for _, filter := range mod.ignoreList.Filters() { + for _, filter := range mod.Session.EventsIgnoreList.Filters() { fmt.Printf(" '%s'\n", string(filter)) } } @@ -163,7 +161,7 @@ func NewEventsStream(s *session.Session) *EventsStream { mod.AddHandler(session.NewModuleHandler("events.filters.clear", "", "Clear the list of filters passed with the events.ignore command.", func(args []string) error { - mod.ignoreList = NewIgnoreList() + mod.Session.EventsIgnoreList.Clear() return nil })) @@ -281,7 +279,7 @@ func (mod *EventsStream) Start() error { mod.waitChan <- &e } - if !mod.ignoreList.Ignored(e) { + if !mod.Session.EventsIgnoreList.Ignored(e) { mod.View(e, true) } @@ -303,7 +301,7 @@ func (mod *EventsStream) Show(limit int) error { selected := []session.Event{} for i := range events { e := events[num-1-i] - if !mod.ignoreList.Ignored(e) { + if !mod.Session.EventsIgnoreList.Ignored(e) { selected = append(selected, e) if len(selected) == limit { break diff --git a/modules/events_stream/ignore_list.go b/session/events_ignore_list.go similarity index 63% rename from modules/events_stream/ignore_list.go rename to session/events_ignore_list.go index 036142e2..05f23212 100644 --- a/modules/events_stream/ignore_list.go +++ b/session/events_ignore_list.go @@ -1,4 +1,4 @@ -package events_stream +package session import ( "errors" @@ -6,8 +6,6 @@ import ( "strings" "sync" - "github.com/bettercap/bettercap/session" - "github.com/evilsocket/islazy/str" ) @@ -15,24 +13,24 @@ var ( ErrEmptyExpression = errors.New("expression can not be empty") ) -type IgnoreFilter string +type filter string -func (f IgnoreFilter) Matches(s string) bool { +func (f filter) Matches(s string) bool { return string(f) == s || strings.HasPrefix(s, string(f)) } -type IgnoreList struct { +type EventsIgnoreList struct { sync.RWMutex - filters []IgnoreFilter + filters []filter } -func NewIgnoreList() *IgnoreList { - return &IgnoreList{ - filters: make([]IgnoreFilter, 0), +func NewEventsIgnoreList() *EventsIgnoreList { + return &EventsIgnoreList{ + filters: make([]filter, 0), } } -func (l *IgnoreList) checkExpression(expr string) (string, error) { +func (l *EventsIgnoreList) checkExpression(expr string) (string, error) { expr = str.Trim(expr) if expr == "" { return "", ErrEmptyExpression @@ -41,7 +39,7 @@ func (l *IgnoreList) checkExpression(expr string) (string, error) { return expr, nil } -func (l *IgnoreList) Add(expr string) (err error) { +func (l *EventsIgnoreList) Add(expr string) (err error) { if expr, err = l.checkExpression(expr); err != nil { return err } @@ -57,12 +55,12 @@ func (l *IgnoreList) Add(expr string) (err error) { } // all good - l.filters = append(l.filters, IgnoreFilter(expr)) + l.filters = append(l.filters, filter(expr)) return nil } -func (l *IgnoreList) Remove(expr string) (err error) { +func (l *EventsIgnoreList) Remove(expr string) (err error) { if expr, err = l.checkExpression(expr); err != nil { return err } @@ -71,8 +69,8 @@ func (l *IgnoreList) Remove(expr string) (err error) { defer l.Unlock() // build a new list with everything that does not match - toRemove := IgnoreFilter(expr) - newList := make([]IgnoreFilter, 0) + toRemove := filter(expr) + newList := make([]filter, 0) for _, filter := range l.filters { if !toRemove.Matches(string(filter)) { newList = append(newList, filter) @@ -89,7 +87,13 @@ func (l *IgnoreList) Remove(expr string) (err error) { return nil } -func (l *IgnoreList) Ignored(e session.Event) bool { +func (l *EventsIgnoreList) Clear() { + l.RLock() + defer l.RUnlock() + l.filters = make([]filter, 0) +} + +func (l *EventsIgnoreList) Ignored(e Event) bool { l.RLock() defer l.RUnlock() @@ -102,12 +106,12 @@ func (l *IgnoreList) Ignored(e session.Event) bool { return false } -func (l *IgnoreList) Empty() bool { +func (l *EventsIgnoreList) Empty() bool { l.RLock() defer l.RUnlock() return len(l.filters) == 0 } -func (l *IgnoreList) Filters() []IgnoreFilter { +func (l *EventsIgnoreList) Filters() []filter { return l.filters } diff --git a/session/session.go b/session/session.go index 647eb684..2dd8a04f 100644 --- a/session/session.go +++ b/session/session.go @@ -69,12 +69,13 @@ type Session struct { GPS GPS Modules ModuleList - Input *readline.Instance - Prompt Prompt - CoreHandlers []CommandHandler - Events *EventPool - UnkCmdCallback UnknownCommandCallback - Firewall firewall.FirewallManager + Input *readline.Instance + Prompt Prompt + CoreHandlers []CommandHandler + Events *EventPool + EventsIgnoreList *EventsIgnoreList + UnkCmdCallback UnknownCommandCallback + Firewall firewall.FirewallManager } func New() (*Session, error) { @@ -95,10 +96,11 @@ func New() (*Session, error) { Active: false, Queue: nil, - CoreHandlers: make([]CommandHandler, 0), - Modules: make([]Module, 0), - Events: nil, - UnkCmdCallback: nil, + CoreHandlers: make([]CommandHandler, 0), + Modules: make([]Module, 0), + Events: nil, + EventsIgnoreList: NewEventsIgnoreList(), + UnkCmdCallback: nil, } if *s.Options.CpuProfile != "" { From 80000ed73799aec98aa180d3b27a4e3ac4cb7704 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 11:59:41 +0100 Subject: [PATCH 27/74] new: the events ignore list is now exported as the events.stream state object via api.rest --- modules/events_stream/events_stream.go | 2 ++ session/events_ignore_list.go | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/events_stream/events_stream.go b/modules/events_stream/events_stream.go index 8fc56466..58b0e00b 100644 --- a/modules/events_stream/events_stream.go +++ b/modules/events_stream/events_stream.go @@ -49,6 +49,8 @@ func NewEventsStream(s *session.Session) *EventsStream { triggerList: NewTriggerList(), } + mod.State.Store("ignoring", &mod.Session.EventsIgnoreList) + mod.AddHandler(session.NewModuleHandler("events.stream on", "", "Start events stream.", func(args []string) error { diff --git a/session/events_ignore_list.go b/session/events_ignore_list.go index 05f23212..711a57cb 100644 --- a/session/events_ignore_list.go +++ b/session/events_ignore_list.go @@ -1,6 +1,7 @@ package session import ( + "encoding/json" "errors" "fmt" "strings" @@ -30,6 +31,12 @@ func NewEventsIgnoreList() *EventsIgnoreList { } } +func (l *EventsIgnoreList) MarshalJSON() ([]byte, error) { + l.RLock() + defer l.RUnlock() + return json.Marshal(l.filters) +} + func (l *EventsIgnoreList) checkExpression(expr string) (string, error) { expr = str.Trim(expr) if expr == "" { @@ -88,8 +95,8 @@ func (l *EventsIgnoreList) Remove(expr string) (err error) { } func (l *EventsIgnoreList) Clear() { - l.RLock() - defer l.RUnlock() + l.Lock() + defer l.Unlock() l.filters = make([]filter, 0) } From 821f885332ba29f68aec7218727e83b5280630ee Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 12:55:46 +0100 Subject: [PATCH 28/74] misc: added debug panic for #500 --- packets/queue.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packets/queue.go b/packets/queue.go index 9546f738..7007979b 100644 --- a/packets/queue.go +++ b/packets/queue.go @@ -62,6 +62,7 @@ func NewQueue(iface *network.Endpoint) (q *Queue, err error) { q = &Queue{ Protos: sync.Map{}, Traffic: sync.Map{}, + Stats: Stats{}, Activities: make(chan Activity), writes: &sync.WaitGroup{}, @@ -165,6 +166,10 @@ func (q *Queue) trackActivity(eth *layers.Ethernet, ip4 *layers.IPv4, address ne } func (q *Queue) TrackPacket(size uint64) { + // https://github.com/bettercap/bettercap/issues/500 + if q == nil { + panic("track packet on nil queue!") + } atomic.AddUint64(&q.Stats.PktReceived, 1) atomic.AddUint64(&q.Stats.Received, size) } From 9693c06cef4329c6ae413d2c2eb7c91d01398693 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 13:34:09 +0100 Subject: [PATCH 29/74] fix: attempt to fix #500 by structure realigning --- packets/queue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packets/queue.go b/packets/queue.go index 7007979b..0ecbfcd9 100644 --- a/packets/queue.go +++ b/packets/queue.go @@ -38,10 +38,10 @@ type PacketCallback func(pkt gopacket.Packet) type Queue struct { sync.RWMutex - Activities chan Activity Stats Stats Protos sync.Map Traffic sync.Map + Activities chan Activity iface *network.Endpoint handle *pcap.Handle From 3bbfdbad2944ddaf0b44c2771533c0d542cb4a6b Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 14:08:16 +0100 Subject: [PATCH 30/74] misc: small fix or general refactoring i did not bother commenting --- packets/queue.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packets/queue.go b/packets/queue.go index 0ecbfcd9..1a7eeaf0 100644 --- a/packets/queue.go +++ b/packets/queue.go @@ -38,6 +38,7 @@ type PacketCallback func(pkt gopacket.Packet) type Queue struct { sync.RWMutex + // keep on top because of https://github.com/bettercap/bettercap/issues/500 Stats Stats Protos sync.Map Traffic sync.Map From 96853e663b5d7a660680f1b7a4559a45a65d1fe8 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 18:26:55 +0100 Subject: [PATCH 31/74] fix: api.rest.address now defaults to 127.0.0.1 --- modules/api_rest/api_rest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 04cb455d..406cb7fa 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -42,7 +42,7 @@ func NewRestAPI(s *session.Session) *RestAPI { } mod.AddParam(session.NewStringParameter("api.rest.address", - session.ParamIfaceAddress, + "127.0.0.1", session.IPv4Validator, "Address to bind the API REST server to.")) From 7e7f2ef6453d9c13058f22e8a58a22145233b48b Mon Sep 17 00:00:00 2001 From: evilsocket Date: Thu, 21 Mar 2019 19:39:08 +0100 Subject: [PATCH 32/74] new: aliases are now centralized and can be used for any type of device (closes #504) --- modules/wifi/wifi_recon.go | 2 +- modules/wifi/wifi_recon_handshakes.go | 2 +- network/ble.go | 11 ++++++- network/ble_device.go | 3 ++ network/hid.go | 12 ++++++- network/hid_device.go | 3 ++ network/lan.go | 25 +-------------- network/wifi.go | 29 +++++++++++------ network/wifi_ap.go | 15 +++++---- session/session.go | 18 ++++++++--- session/session_core_handlers.go | 46 +++++++++++++++++++++++++-- 11 files changed, 115 insertions(+), 51 deletions(-) diff --git a/modules/wifi/wifi_recon.go b/modules/wifi/wifi_recon.go index dab0360b..6e62d0d9 100644 --- a/modules/wifi/wifi_recon.go +++ b/modules/wifi/wifi_recon.go @@ -124,7 +124,7 @@ func (mod *WiFiModule) discoverClients(radiotap *layers.RadioTap, dot11 *layers. freq := int(radiotap.ChannelFrequency) rssi := radiotap.DBMAntennaSignal - if station, isNew := ap.AddClientIfNew(bssid, freq, rssi, mod.Session.Lan.Aliases()); isNew { + if station, isNew := ap.AddClientIfNew(bssid, freq, rssi); isNew { mod.Session.Events.Add("wifi.client.new", ClientEvent{ AP: ap, Client: station, diff --git a/modules/wifi/wifi_recon_handshakes.go b/modules/wifi/wifi_recon_handshakes.go index 7c5045a5..a8eb676e 100644 --- a/modules/wifi/wifi_recon_handshakes.go +++ b/modules/wifi/wifi_recon_handshakes.go @@ -36,7 +36,7 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye staIsUs := bytes.Equal(staMac, mod.iface.HW) station, found := ap.Get(staMac.String()) if !found { - station, _ = ap.AddClientIfNew(staMac.String(), ap.Frequency, ap.RSSI, mod.Session.Lan.Aliases()) + station, _ = ap.AddClientIfNew(staMac.String(), ap.Frequency, ap.RSSI) } rawPMKID := []byte(nil) diff --git a/network/ble.go b/network/ble.go index c167ae17..0b492194 100644 --- a/network/ble.go +++ b/network/ble.go @@ -9,6 +9,8 @@ import ( "time" "github.com/bettercap/gatt" + + "github.com/evilsocket/islazy/data" ) const BLEMacValidator = "([a-fA-F0-9]{1,2}:[a-fA-F0-9]{1,2}:[a-fA-F0-9]{1,2}:[a-fA-F0-9]{1,2}:[a-fA-F0-9]{1,2}:[a-fA-F0-9]{1,2})" @@ -18,6 +20,7 @@ type BLEDevLostCallback func(dev *BLEDevice) type BLE struct { sync.RWMutex + aliases *data.UnsortedKV devices map[string]*BLEDevice newCb BLEDevNewCallback lostCb BLEDevLostCallback @@ -27,9 +30,10 @@ type bleJSON struct { Devices []*BLEDevice `json:"devices"` } -func NewBLE(newcb BLEDevNewCallback, lostcb BLEDevLostCallback) *BLE { +func NewBLE(aliases *data.UnsortedKV, newcb BLEDevNewCallback, lostcb BLEDevLostCallback) *BLE { return &BLE{ devices: make(map[string]*BLEDevice), + aliases: aliases, newCb: newcb, lostCb: lostcb, } @@ -55,14 +59,19 @@ func (b *BLE) AddIfNew(id string, p gatt.Peripheral, a *gatt.Advertisement, rssi defer b.Unlock() id = NormalizeMac(id) + alias := b.aliases.GetOr(id, "") if dev, found := b.devices[id]; found { dev.LastSeen = time.Now() dev.RSSI = rssi dev.Advertisement = a + if alias != "" { + dev.Alias = alias + } return dev } newDev := NewBLEDevice(p, a, rssi) + newDev.Alias = alias b.devices[id] = newDev if b.newCb != nil { diff --git a/network/ble_device.go b/network/ble_device.go index 3cb115aa..0b1ef7bb 100644 --- a/network/ble_device.go +++ b/network/ble_device.go @@ -27,6 +27,7 @@ type BLEService struct { } type BLEDevice struct { + Alias string LastSeen time.Time DeviceName string Vendor string @@ -40,6 +41,7 @@ type bleDeviceJSON struct { LastSeen time.Time `json:"last_seen"` Name string `json:"name"` MAC string `json:"mac"` + Alias string `json:"alias"` Vendor string `json:"vendor"` RSSI int `json:"rssi"` Connectable bool `json:"connectable"` @@ -81,6 +83,7 @@ func (d *BLEDevice) MarshalJSON() ([]byte, error) { LastSeen: d.LastSeen, Name: d.Name(), MAC: d.Device.ID(), + Alias: d.Alias, Vendor: d.Vendor, RSSI: d.RSSI, Connectable: d.Advertisement.Connectable, diff --git a/network/hid.go b/network/hid.go index bb8e3605..ed332381 100644 --- a/network/hid.go +++ b/network/hid.go @@ -4,6 +4,8 @@ import ( "encoding/json" "sync" "time" + + "github.com/evilsocket/islazy/data" ) type HIDDevNewCallback func(dev *HIDDevice) @@ -11,6 +13,7 @@ type HIDDevLostCallback func(dev *HIDDevice) type HID struct { sync.RWMutex + aliases *data.UnsortedKV devices map[string]*HIDDevice newCb HIDDevNewCallback lostCb HIDDevLostCallback @@ -20,9 +23,10 @@ type hidJSON struct { Devices []*HIDDevice `json:"devices"` } -func NewHID(newcb HIDDevNewCallback, lostcb HIDDevLostCallback) *HID { +func NewHID(aliases *data.UnsortedKV, newcb HIDDevNewCallback, lostcb HIDDevLostCallback) *HID { return &HID{ devices: make(map[string]*HIDDevice), + aliases: aliases, newCb: newcb, lostCb: lostcb, } @@ -52,14 +56,20 @@ func (b *HID) AddIfNew(address []byte, channel int, payload []byte) (bool, *HIDD defer b.Unlock() id := HIDAddress(address) + alias := b.aliases.GetOr(id, "") + if dev, found := b.devices[id]; found { dev.LastSeen = time.Now() dev.AddChannel(channel) dev.AddPayload(payload) + if alias != "" { + dev.Alias = alias + } return false, dev } newDev := NewHIDDevice(address, channel, payload) + newDev.Alias = alias b.devices[id] = newDev if b.newCb != nil { diff --git a/network/hid_device.go b/network/hid_device.go index 8104bd6b..16dac6f1 100644 --- a/network/hid_device.go +++ b/network/hid_device.go @@ -42,6 +42,7 @@ type HIDDevice struct { sync.Mutex LastSeen time.Time Type HIDType + Alias string Address string RawAddress []byte channels map[int]bool @@ -53,6 +54,7 @@ type hidDeviceJSON struct { LastSeen time.Time `json:"last_seen"` Type string `json:"type"` Address string `json:"address"` + Alias string `json:"alias"` Channels []string `json:"channels"` Payloads []string `json:"payloads"` PayloadsSize uint64 `json:"payloads_size"` @@ -102,6 +104,7 @@ func (dev *HIDDevice) MarshalJSON() ([]byte, error) { LastSeen: dev.LastSeen, Type: dev.Type.String(), Address: dev.Address, + Alias: dev.Alias, Channels: dev.channelsListUnlocked(), Payloads: make([]string, 0), PayloadsSize: dev.payloadsSz, diff --git a/network/lan.go b/network/lan.go index 9363c0ba..0c2bef8f 100644 --- a/network/lan.go +++ b/network/lan.go @@ -2,23 +2,18 @@ package network import ( "encoding/json" - "fmt" "net" "strings" "sync" "github.com/evilsocket/islazy/data" - "github.com/evilsocket/islazy/fs" ) const LANDefaultttl = 10 -const LANAliasesFile = "~/bettercap.aliases" type EndpointNewCallback func(e *Endpoint) type EndpointLostCallback func(e *Endpoint) -var aliasesFileName, _ = fs.Expand(LANAliasesFile) - type LAN struct { sync.Mutex hosts map[string]*Endpoint @@ -34,12 +29,7 @@ type lanJSON struct { Hosts []*Endpoint `json:"hosts"` } -func NewLAN(iface, gateway *Endpoint, newcb EndpointNewCallback, lostcb EndpointLostCallback) *LAN { - aliases, err := data.NewUnsortedKV(aliasesFileName, data.FlushOnEdit) - if err != nil { - fmt.Printf("error loading %s: %s", aliasesFileName, err) - } - +func NewLAN(iface, gateway *Endpoint, aliases *data.UnsortedKV, newcb EndpointNewCallback, lostcb EndpointLostCallback) *LAN { return &LAN{ iface: iface, gateway: gateway, @@ -63,19 +53,6 @@ func (l *LAN) MarshalJSON() ([]byte, error) { return json.Marshal(doc) } -func (lan *LAN) SetAliasFor(mac, alias string) bool { - lan.Lock() - defer lan.Unlock() - - mac = NormalizeMac(mac) - lan.aliases.Set(mac, alias) - if e, found := lan.hosts[mac]; found { - e.Alias = alias - return true - } - return false -} - func (lan *LAN) Get(mac string) (*Endpoint, bool) { lan.Lock() defer lan.Unlock() diff --git a/network/wifi.go b/network/wifi.go index 9d991652..db416c32 100644 --- a/network/wifi.go +++ b/network/wifi.go @@ -11,6 +11,7 @@ import ( "github.com/google/gopacket/layers" "github.com/google/gopacket/pcapgo" + "github.com/evilsocket/islazy/data" "github.com/evilsocket/islazy/fs" ) @@ -43,22 +44,24 @@ type APLostCallback func(ap *AccessPoint) type WiFi struct { sync.Mutex - aps map[string]*AccessPoint - iface *Endpoint - newCb APNewCallback - lostCb APLostCallback + aliases *data.UnsortedKV + aps map[string]*AccessPoint + iface *Endpoint + newCb APNewCallback + lostCb APLostCallback } type wifiJSON struct { AccessPoints []*AccessPoint `json:"aps"` } -func NewWiFi(iface *Endpoint, newcb APNewCallback, lostcb APLostCallback) *WiFi { +func NewWiFi(iface *Endpoint, aliases *data.UnsortedKV, newcb APNewCallback, lostcb APLostCallback) *WiFi { return &WiFi{ - aps: make(map[string]*AccessPoint), - iface: iface, - newCb: newcb, - lostCb: lostcb, + aps: make(map[string]*AccessPoint), + aliases: aliases, + iface: iface, + newCb: newcb, + lostCb: lostcb, } } @@ -134,6 +137,7 @@ func (w *WiFi) AddIfNew(ssid, mac string, frequency int, rssi int8) (*AccessPoin defer w.Unlock() mac = NormalizeMac(mac) + alias := w.aliases.GetOr(mac, "") if ap, found := w.aps[mac]; found { ap.LastSeen = time.Now() if rssi != 0 { @@ -143,10 +147,15 @@ func (w *WiFi) AddIfNew(ssid, mac string, frequency int, rssi int8) (*AccessPoin if !isBogusMacESSID(ssid) { ap.Hostname = ssid } + + if alias != "" { + ap.Alias = alias + } return ap, false } - newAp := NewAccessPoint(ssid, mac, frequency, rssi) + newAp := NewAccessPoint(ssid, mac, frequency, rssi, w.aliases) + newAp.Alias = alias w.aps[mac] = newAp if w.newCb != nil { diff --git a/network/wifi_ap.go b/network/wifi_ap.go index ec864916..58143cc0 100644 --- a/network/wifi_ap.go +++ b/network/wifi_ap.go @@ -12,6 +12,7 @@ type AccessPoint struct { *Station sync.Mutex + aliases *data.UnsortedKV clients map[string]*Station withKeyMaterial bool } @@ -22,9 +23,10 @@ type apJSON struct { Handshake bool `json:"handshake"` } -func NewAccessPoint(essid, bssid string, frequency int, rssi int8) *AccessPoint { +func NewAccessPoint(essid, bssid string, frequency int, rssi int8, aliases *data.UnsortedKV) *AccessPoint { return &AccessPoint{ Station: NewStation(essid, bssid, frequency, rssi), + aliases: aliases, clients: make(map[string]*Station), } } @@ -67,11 +69,12 @@ func (ap *AccessPoint) RemoveClient(mac string) { } } -func (ap *AccessPoint) AddClientIfNew(bssid string, frequency int, rssi int8, aliases *data.UnsortedKV) (*Station, bool) { +func (ap *AccessPoint) AddClientIfNew(bssid string, frequency int, rssi int8) (*Station, bool) { ap.Lock() defer ap.Unlock() bssid = NormalizeMac(bssid) + alias := ap.aliases.GetOr(bssid, "") if s, found := ap.clients[bssid]; found { // update @@ -79,17 +82,15 @@ func (ap *AccessPoint) AddClientIfNew(bssid string, frequency int, rssi int8, al s.RSSI = rssi s.LastSeen = time.Now() - if aliases != nil { - s.Alias = aliases.GetOr(bssid, "") + if alias != "" { + s.Alias = alias } return s, false } s := NewStation("", bssid, frequency, rssi) - if aliases != nil { - s.Alias = aliases.GetOr(bssid, "") - } + s.Alias = alias ap.clients[bssid] = s return s, true diff --git a/session/session.go b/session/session.go index 2dd8a04f..ba49a03d 100644 --- a/session/session.go +++ b/session/session.go @@ -20,6 +20,7 @@ import ( "github.com/bettercap/bettercap/network" "github.com/bettercap/bettercap/packets" + "github.com/evilsocket/islazy/data" "github.com/evilsocket/islazy/fs" "github.com/evilsocket/islazy/log" "github.com/evilsocket/islazy/ops" @@ -54,6 +55,10 @@ type GPS struct { Separation float64 // Geoidal separation } +const AliasesFile = "~/bettercap.aliases" + +var aliasesFileName, _ = fs.Expand(AliasesFile) + type Session struct { Options core.Options Interface *network.Endpoint @@ -68,6 +73,7 @@ type Session struct { Active bool GPS GPS Modules ModuleList + Aliases *data.UnsortedKV Input *readline.Instance Prompt Prompt @@ -115,6 +121,10 @@ func New() (*Session, error) { return nil, err } + if s.Aliases, err = data.NewUnsortedKV(aliasesFileName, data.FlushOnEdit); err != nil { + return nil, err + } + s.Events = NewEventPool(*s.Options.Debug, *s.Options.Silent) s.registerCoreHandlers() @@ -235,25 +245,25 @@ func (s *Session) Start() error { s.Firewall = firewall.Make(s.Interface) - s.HID = network.NewHID(func(dev *network.HIDDevice) { + s.HID = network.NewHID(s.Aliases, func(dev *network.HIDDevice) { s.Events.Add("hid.device.new", dev) }, func(dev *network.HIDDevice) { s.Events.Add("hid.device.lost", dev) }) - s.BLE = network.NewBLE(func(dev *network.BLEDevice) { + s.BLE = network.NewBLE(s.Aliases, func(dev *network.BLEDevice) { s.Events.Add("ble.device.new", dev) }, func(dev *network.BLEDevice) { s.Events.Add("ble.device.lost", dev) }) - s.WiFi = network.NewWiFi(s.Interface, func(ap *network.AccessPoint) { + s.WiFi = network.NewWiFi(s.Interface, s.Aliases, func(ap *network.AccessPoint) { s.Events.Add("wifi.ap.new", ap) }, func(ap *network.AccessPoint) { s.Events.Add("wifi.ap.lost", ap) }) - s.Lan = network.NewLAN(s.Interface, s.Gateway, func(e *network.Endpoint) { + s.Lan = network.NewLAN(s.Interface, s.Gateway, s.Aliases, func(e *network.Endpoint) { s.Events.Add("endpoint.new", e) }, func(e *network.Endpoint) { s.Events.Add("endpoint.lost", e) diff --git a/session/session_core_handlers.go b/session/session_core_handlers.go index 13f48419..09da1776 100644 --- a/session/session_core_handlers.go +++ b/session/session_core_handlers.go @@ -271,13 +271,55 @@ func (s *Session) shHandler(args []string, sess *Session) error { return err } +func normalizeMac(mac string) string { + var parts []string + if strings.ContainsRune(mac, '-') { + parts = strings.Split(mac, "-") + } else { + parts = strings.Split(mac, ":") + } + + for i, p := range parts { + if len(p) < 2 { + parts[i] = "0" + p + } + } + return strings.ToLower(strings.Join(parts, ":")) +} + +func (s *Session) propagateAlias(mac, alias string) { + mac = normalizeMac(mac) + + s.Aliases.Set(mac, alias) + + if dev, found := s.BLE.Get(mac); found { + dev.Alias = alias + } + + if dev, found := s.HID.Get(mac); found { + dev.Alias = alias + } + + if ap, found := s.WiFi.Get(mac); found { + ap.Alias = alias + } + + if sta, found := s.WiFi.GetClient(mac); found { + sta.Alias = alias + } + + if host, found := s.Lan.Get(mac); found { + host.Alias = alias + } +} + func (s *Session) aliasHandler(args []string, sess *Session) error { mac := args[0] alias := str.Trim(args[1]) if alias == "\"\"" || alias == "''" { alias = "" } - s.Lan.SetAliasFor(mac, alias) + s.propagateAlias(mac, alias) return nil } @@ -383,7 +425,7 @@ func (s *Session) registerCoreHandlers() { readline.PcItem("!")) s.addHandler(NewCommandHandler("alias MAC NAME", - "^alias\\s+([a-fA-F0-9:]{17})\\s*(.*)", + "^alias\\s+([a-fA-F0-9:]{14,17})\\s*(.*)", "Assign an alias to a given endpoint given its MAC address.", s.aliasHandler), readline.PcItem("alias", readline.PcItemDynamic(func(prefix string) []string { From 9f24800e6d092e8f670464e571af45ac05c762f5 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 22 Mar 2019 15:11:07 +0100 Subject: [PATCH 33/74] fix: wifi clients sent and received data frame counters are now updated correctly --- modules/wifi/wifi.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/wifi/wifi.go b/modules/wifi/wifi.go index 50b17ed9..2007aa62 100644 --- a/modules/wifi/wifi.go +++ b/modules/wifi/wifi.go @@ -495,13 +495,17 @@ func (mod *WiFiModule) updateStats(dot11 *layers.Dot11, packet gopacket.Packet) bytes := uint64(len(packet.Data())) dst := dot11.Address1.String() - if station, found := mod.Session.WiFi.Get(dst); found { - station.Received += bytes + if ap, found := mod.Session.WiFi.Get(dst); found { + ap.Received += bytes + } else if sta, found := mod.Session.WiFi.GetClient(dst); found { + sta.Received += bytes } src := dot11.Address2.String() - if station, found := mod.Session.WiFi.Get(src); found { - station.Sent += bytes + if ap, found := mod.Session.WiFi.Get(src); found { + ap.Sent += bytes + } else if sta, found := mod.Session.WiFi.GetClient(src); found { + sta.Sent += bytes } } } From c7f28855ca4f02acc3827b61aab67222d313b723 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 22 Mar 2019 21:28:57 +0100 Subject: [PATCH 34/74] fix: clear handler caused issues on some terminals, fixed by using os specific clear command --- session/session_core_handlers.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/session/session_core_handlers.go b/session/session_core_handlers.go index 09da1776..51f40be0 100644 --- a/session/session_core_handlers.go +++ b/session/session_core_handlers.go @@ -4,7 +4,9 @@ import ( "bufio" "fmt" "os" + "os/exec" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -249,13 +251,14 @@ func (s *Session) readHandler(args []string, sess *Session) error { } func (s *Session) clsHandler(args []string, sess *Session) error { - // fixes a weird bug which causes the screen not to be fully - // cleared if a "clear; net.show" commands chain is executed - // in the interactive session. - for i := 0; i < 180; i++ { - fmt.Println() + cmd := "clear" + if runtime.GOOS == "windows" { + cmd = "cls" } - readline.ClearScreen(s.Input.Stdout()) + + c := exec.Command(cmd) + c.Stdout = os.Stdout + c.Run() return nil } From 9a3c252c4f95c9147fcdd5ca739439b75d15281f Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 22 Mar 2019 23:31:11 +0100 Subject: [PATCH 35/74] new: handshakes history is now loaded from the configured pcap --- modules/wifi/wifi.go | 68 +++++++++++++++++++++------ modules/wifi/wifi_recon.go | 6 +++ modules/wifi/wifi_recon_handshakes.go | 24 ++++++---- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/modules/wifi/wifi.go b/modules/wifi/wifi.go index 2007aa62..a61af5e3 100644 --- a/modules/wifi/wifi.go +++ b/modules/wifi/wifi.go @@ -23,21 +23,31 @@ import ( "github.com/evilsocket/islazy/tui" ) +type parsedShake struct { + Radiotap *layers.RadioTap + Dot11 *layers.Dot11 + Packet gopacket.Packet +} + type WiFiModule struct { session.SessionModule - iface *network.Endpoint - handle *pcap.Handle - source string - region string - txPower int - minRSSI int - channel int - hopPeriod time.Duration - hopChanges chan bool - frequencies []int - ap *network.AccessPoint - stickChan int + iface *network.Endpoint + handle *pcap.Handle + source string + region string + txPower int + minRSSI int + channel int + hopPeriod time.Duration + hopChanges chan bool + frequencies []int + ap *network.AccessPoint + stickChan int + + shakesFile string + shakesHistory []parsedShake + skipBroken bool pktSourceChan chan gopacket.Packet pktSourceChanClosed bool @@ -47,7 +57,6 @@ type WiFiModule struct { assocSkip []net.HardwareAddr assocSilent bool assocOpen bool - shakesFile string apRunning bool showManuf bool apConfig packets.Dot11ApConfig @@ -66,6 +75,7 @@ func NewWiFiModule(s *session.Session) *WiFiModule { stickChan: 0, hopPeriod: 250 * time.Millisecond, hopChanges: make(chan bool), + shakesHistory: make([]parsedShake, 0), ap: nil, skipBroken: true, apRunning: false, @@ -510,11 +520,41 @@ func (mod *WiFiModule) updateStats(dot11 *layers.Dot11, packet gopacket.Packet) } } +func (mod *WiFiModule) loadHandshakes() { + mod.shakesHistory = make([]parsedShake, 0) + + if !fs.Exists(mod.shakesFile) { + return + } + + handle, err := pcap.OpenOffline(mod.shakesFile) + if err != nil { + mod.Debug("can't open handshakes file: %v", mod.shakesFile) + return + } + defer handle.Close() + + mod.Info("loading handshakes from %s", mod.shakesFile) + + src := gopacket.NewPacketSource(handle, handle.LinkType()) + for packet := range src.Packets() { + if ok, radiotap, dot11 := packets.Dot11Parse(packet); ok { + mod.shakesHistory = append(mod.shakesHistory, parsedShake{ + Radiotap: radiotap, + Dot11: dot11, + Packet: packet, + }) + } + } +} + func (mod *WiFiModule) Start() error { if err := mod.Configure(); err != nil { return err } + mod.loadHandshakes() + mod.SetRunning(true, func() { // start channel hopper if needed if mod.channel == 0 && mod.source == "" { @@ -551,7 +591,7 @@ func (mod *WiFiModule) Start() error { mod.discoverProbes(radiotap, dot11, packet) mod.discoverAccessPoints(radiotap, dot11, packet) mod.discoverClients(radiotap, dot11, packet) - mod.discoverHandshakes(radiotap, dot11, packet) + mod.discoverHandshakes(radiotap, dot11, packet, false) mod.updateInfo(dot11, packet) mod.updateStats(dot11, packet) } diff --git a/modules/wifi/wifi_recon.go b/modules/wifi/wifi_recon.go index 6e62d0d9..5ec36e0a 100644 --- a/modules/wifi/wifi_recon.go +++ b/modules/wifi/wifi_recon.go @@ -70,6 +70,12 @@ func (mod *WiFiModule) discoverAccessPoints(radiotap *layers.RadioTap, dot11 *la ap.EachClient(func(mac string, station *network.Station) { station.Handshake.SetBeacon(packet) }) + } else { + // every time we detect a new ap, see if we have + // its handshakes in our pcap already + for _, h := range mod.shakesHistory { + mod.discoverHandshakes(h.Radiotap, h.Dot11, h.Packet, true) + } } } else { mod.Debug("skipping %s with %d dBm", from.String(), radiotap.DBMAntennaSignal) diff --git a/modules/wifi/wifi_recon_handshakes.go b/modules/wifi/wifi_recon_handshakes.go index a8eb676e..2896debb 100644 --- a/modules/wifi/wifi_recon_handshakes.go +++ b/modules/wifi/wifi_recon_handshakes.go @@ -18,12 +18,14 @@ func allZeros(s []byte) bool { return true } -func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *layers.Dot11, packet gopacket.Packet) { +func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *layers.Dot11, packet gopacket.Packet, readOnly bool) { if ok, key, apMac, staMac := packets.Dot11ParseEAPOL(packet, dot11); ok { // first, locate the AP in our list by its BSSID ap, found := mod.Session.WiFi.Get(apMac.String()) if !found { - mod.Warning("could not find AP with BSSID %s", apMac.String()) + if !readOnly { + mod.Warning("could not find AP with BSSID %s", apMac.String()) + } return } @@ -75,7 +77,7 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye // if we have unsaved packets as part of the handshake, save them. numUnsaved := station.Handshake.NumUnsaved() doSave := numUnsaved > 0 - if doSave && mod.shakesFile != "" { + if !readOnly && doSave && mod.shakesFile != "" { mod.Debug("saving handshake frames to %s", mod.shakesFile) if err := mod.Session.WiFi.SaveHandshakesTo(mod.shakesFile, mod.handle.LinkType()); err != nil { mod.Error("error while saving handshake frames to %s: %s", mod.shakesFile, err) @@ -85,13 +87,15 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye // if we had unsaved packets and either the handshake is complete // or it contains the PMKID, generate a new event. if doSave && (rawPMKID != nil || station.Handshake.Complete()) { - mod.Session.Events.Add("wifi.client.handshake", HandshakeEvent{ - File: mod.shakesFile, - NewPackets: numUnsaved, - AP: apMac.String(), - Station: staMac.String(), - PMKID: rawPMKID, - }) + if !readOnly { + mod.Session.Events.Add("wifi.client.handshake", HandshakeEvent{ + File: mod.shakesFile, + NewPackets: numUnsaved, + AP: apMac.String(), + Station: staMac.String(), + PMKID: rawPMKID, + }) + } // make sure the info that we have key material for this AP // is persisted even after stations are pruned due to inactivity ap.WithKeyMaterial(true) From b7f4a3024b7de51a3c39d9aa71ee9afeeed59e3c Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 22 Mar 2019 23:49:21 +0100 Subject: [PATCH 36/74] misc: small fix or general refactoring i did not bother commenting --- modules/wifi/wifi_recon_handshakes.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/wifi/wifi_recon_handshakes.go b/modules/wifi/wifi_recon_handshakes.go index 2896debb..a9c4a15d 100644 --- a/modules/wifi/wifi_recon_handshakes.go +++ b/modules/wifi/wifi_recon_handshakes.go @@ -37,8 +37,9 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye // in order to have a consistent association of AP, client and handshakes. staIsUs := bytes.Equal(staMac, mod.iface.HW) station, found := ap.Get(staMac.String()) + staAdded := false if !found { - station, _ = ap.AddClientIfNew(staMac.String(), ap.Frequency, ap.RSSI) + station, staAdded = ap.AddClientIfNew(staMac.String(), ap.Frequency, ap.RSSI) } rawPMKID := []byte(nil) @@ -100,10 +101,10 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye // is persisted even after stations are pruned due to inactivity ap.WithKeyMaterial(true) } - - // if we added ourselves as a client station but we didn't get any + // if we're only collecting handshakes from history or if we + // added ourselves as a client station but we didn't get any // PMKID, just remove it from the list of clients of this AP. - if staIsUs && rawPMKID == nil { + if (readOnly && staAdded) || (staIsUs && rawPMKID == nil) { ap.RemoveClient(staMac.String()) } } From 5b02f14cab67a47743df3fa261fc2a67c85b2210 Mon Sep 17 00:00:00 2001 From: Zane Date: Fri, 22 Mar 2019 16:15:41 -0700 Subject: [PATCH 37/74] Fixed certificate issues --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5ffdaac7..a1696565 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ RUN git clone https://github.com/bettercap/caplets /usr/local/share/bettercap/ca # final stage FROM alpine +RUN apk add --update ca-certificates RUN apk add --no-cache --update bash iproute2 libpcap libusb-dev libnetfilter_queue wireless-tools COPY --from=build-env /go/src/github.com/bettercap/bettercap/bettercap /app/ COPY --from=build-env /go/src/github.com/bettercap/bettercap/caplets /app/ From b8056e2026c1d6f71133ecfefa3aea51cb97f5af Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 23 Mar 2019 14:11:28 +0100 Subject: [PATCH 38/74] misc: small fix or general refactoring i did not bother commenting --- modules/wifi/wifi.go | 68 ++++++--------------------- modules/wifi/wifi_recon.go | 6 --- modules/wifi/wifi_recon_handshakes.go | 31 +++++------- 3 files changed, 27 insertions(+), 78 deletions(-) diff --git a/modules/wifi/wifi.go b/modules/wifi/wifi.go index a61af5e3..998c2121 100644 --- a/modules/wifi/wifi.go +++ b/modules/wifi/wifi.go @@ -23,31 +23,22 @@ import ( "github.com/evilsocket/islazy/tui" ) -type parsedShake struct { - Radiotap *layers.RadioTap - Dot11 *layers.Dot11 - Packet gopacket.Packet -} - type WiFiModule struct { session.SessionModule - iface *network.Endpoint - handle *pcap.Handle - source string - region string - txPower int - minRSSI int - channel int - hopPeriod time.Duration - hopChanges chan bool - frequencies []int - ap *network.AccessPoint - stickChan int - - shakesFile string - shakesHistory []parsedShake - + iface *network.Endpoint + handle *pcap.Handle + source string + region string + txPower int + minRSSI int + channel int + hopPeriod time.Duration + hopChanges chan bool + frequencies []int + ap *network.AccessPoint + stickChan int + shakesFile string skipBroken bool pktSourceChan chan gopacket.Packet pktSourceChanClosed bool @@ -75,7 +66,6 @@ func NewWiFiModule(s *session.Session) *WiFiModule { stickChan: 0, hopPeriod: 250 * time.Millisecond, hopChanges: make(chan bool), - shakesHistory: make([]parsedShake, 0), ap: nil, skipBroken: true, apRunning: false, @@ -520,41 +510,11 @@ func (mod *WiFiModule) updateStats(dot11 *layers.Dot11, packet gopacket.Packet) } } -func (mod *WiFiModule) loadHandshakes() { - mod.shakesHistory = make([]parsedShake, 0) - - if !fs.Exists(mod.shakesFile) { - return - } - - handle, err := pcap.OpenOffline(mod.shakesFile) - if err != nil { - mod.Debug("can't open handshakes file: %v", mod.shakesFile) - return - } - defer handle.Close() - - mod.Info("loading handshakes from %s", mod.shakesFile) - - src := gopacket.NewPacketSource(handle, handle.LinkType()) - for packet := range src.Packets() { - if ok, radiotap, dot11 := packets.Dot11Parse(packet); ok { - mod.shakesHistory = append(mod.shakesHistory, parsedShake{ - Radiotap: radiotap, - Dot11: dot11, - Packet: packet, - }) - } - } -} - func (mod *WiFiModule) Start() error { if err := mod.Configure(); err != nil { return err } - mod.loadHandshakes() - mod.SetRunning(true, func() { // start channel hopper if needed if mod.channel == 0 && mod.source == "" { @@ -591,7 +551,7 @@ func (mod *WiFiModule) Start() error { mod.discoverProbes(radiotap, dot11, packet) mod.discoverAccessPoints(radiotap, dot11, packet) mod.discoverClients(radiotap, dot11, packet) - mod.discoverHandshakes(radiotap, dot11, packet, false) + mod.discoverHandshakes(radiotap, dot11, packet) mod.updateInfo(dot11, packet) mod.updateStats(dot11, packet) } diff --git a/modules/wifi/wifi_recon.go b/modules/wifi/wifi_recon.go index 5ec36e0a..6e62d0d9 100644 --- a/modules/wifi/wifi_recon.go +++ b/modules/wifi/wifi_recon.go @@ -70,12 +70,6 @@ func (mod *WiFiModule) discoverAccessPoints(radiotap *layers.RadioTap, dot11 *la ap.EachClient(func(mac string, station *network.Station) { station.Handshake.SetBeacon(packet) }) - } else { - // every time we detect a new ap, see if we have - // its handshakes in our pcap already - for _, h := range mod.shakesHistory { - mod.discoverHandshakes(h.Radiotap, h.Dot11, h.Packet, true) - } } } else { mod.Debug("skipping %s with %d dBm", from.String(), radiotap.DBMAntennaSignal) diff --git a/modules/wifi/wifi_recon_handshakes.go b/modules/wifi/wifi_recon_handshakes.go index a9c4a15d..c8977b01 100644 --- a/modules/wifi/wifi_recon_handshakes.go +++ b/modules/wifi/wifi_recon_handshakes.go @@ -18,14 +18,12 @@ func allZeros(s []byte) bool { return true } -func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *layers.Dot11, packet gopacket.Packet, readOnly bool) { +func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *layers.Dot11, packet gopacket.Packet) { if ok, key, apMac, staMac := packets.Dot11ParseEAPOL(packet, dot11); ok { // first, locate the AP in our list by its BSSID ap, found := mod.Session.WiFi.Get(apMac.String()) if !found { - if !readOnly { - mod.Warning("could not find AP with BSSID %s", apMac.String()) - } + mod.Warning("could not find AP with BSSID %s", apMac.String()) return } @@ -78,8 +76,8 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye // if we have unsaved packets as part of the handshake, save them. numUnsaved := station.Handshake.NumUnsaved() doSave := numUnsaved > 0 - if !readOnly && doSave && mod.shakesFile != "" { - mod.Debug("saving handshake frames to %s", mod.shakesFile) + if doSave && mod.shakesFile != "" { + mod.Info("saving handshake frames to %s", mod.shakesFile) if err := mod.Session.WiFi.SaveHandshakesTo(mod.shakesFile, mod.handle.LinkType()); err != nil { mod.Error("error while saving handshake frames to %s: %s", mod.shakesFile, err) } @@ -88,23 +86,20 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye // if we had unsaved packets and either the handshake is complete // or it contains the PMKID, generate a new event. if doSave && (rawPMKID != nil || station.Handshake.Complete()) { - if !readOnly { - mod.Session.Events.Add("wifi.client.handshake", HandshakeEvent{ - File: mod.shakesFile, - NewPackets: numUnsaved, - AP: apMac.String(), - Station: staMac.String(), - PMKID: rawPMKID, - }) - } + mod.Session.Events.Add("wifi.client.handshake", HandshakeEvent{ + File: mod.shakesFile, + NewPackets: numUnsaved, + AP: apMac.String(), + Station: staMac.String(), + PMKID: rawPMKID, + }) // make sure the info that we have key material for this AP // is persisted even after stations are pruned due to inactivity ap.WithKeyMaterial(true) } - // if we're only collecting handshakes from history or if we - // added ourselves as a client station but we didn't get any + // if we added ourselves as a client station but we didn't get any // PMKID, just remove it from the list of clients of this AP. - if (readOnly && staAdded) || (staIsUs && rawPMKID == nil) { + if staAdded || (staIsUs && rawPMKID == nil) { ap.RemoveClient(staMac.String()) } } From afdb11514bf15732af306471511691dd4718d36c Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 23 Mar 2019 15:18:52 +0100 Subject: [PATCH 39/74] fix: fixed macOS compilation --- network/ble_unsupported.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/ble_unsupported.go b/network/ble_unsupported.go index 9765fe9d..2f352775 100644 --- a/network/ble_unsupported.go +++ b/network/ble_unsupported.go @@ -9,6 +9,7 @@ import ( type BLEDevice struct { LastSeen time.Time + Alias string } func NewBLEDevice() *BLEDevice { From 38c94cb10522aaa9206a2ba55a5fa205260eabaf Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 23 Mar 2019 15:21:19 +0100 Subject: [PATCH 40/74] misc: small fix or general refactoring i did not bother commenting --- network/ble_unsupported.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/network/ble_unsupported.go b/network/ble_unsupported.go index 2f352775..b2a523f6 100644 --- a/network/ble_unsupported.go +++ b/network/ble_unsupported.go @@ -5,6 +5,8 @@ package network import ( "encoding/json" "time" + + "github.com/evilsocket/islazy/data" ) type BLEDevice struct { @@ -22,6 +24,7 @@ type BLEDevNewCallback func(dev *BLEDevice) type BLEDevLostCallback func(dev *BLEDevice) type BLE struct { + aliases *data.UnsortedKV devices map[string]*BLEDevice newCb BLEDevNewCallback lostCb BLEDevLostCallback @@ -31,8 +34,9 @@ type bleJSON struct { Devices []*BLEDevice `json:"devices"` } -func NewBLE(newcb BLEDevNewCallback, lostcb BLEDevLostCallback) *BLE { +func NewBLE(aliases *data.UnsortedKV, newcb BLEDevNewCallback, lostcb BLEDevLostCallback) *BLE { return &BLE{ + aliases: aliases, devices: make(map[string]*BLEDevice), newCb: newcb, lostCb: lostcb, From fe568c818880948c1aab1ece6c349b150e1867a6 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 23 Mar 2019 15:59:33 +0100 Subject: [PATCH 41/74] misc: small fix or general refactoring i did not bother commenting --- modules/wifi/wifi_recon_handshakes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/wifi/wifi_recon_handshakes.go b/modules/wifi/wifi_recon_handshakes.go index c8977b01..599eec3e 100644 --- a/modules/wifi/wifi_recon_handshakes.go +++ b/modules/wifi/wifi_recon_handshakes.go @@ -77,7 +77,7 @@ func (mod *WiFiModule) discoverHandshakes(radiotap *layers.RadioTap, dot11 *laye numUnsaved := station.Handshake.NumUnsaved() doSave := numUnsaved > 0 if doSave && mod.shakesFile != "" { - mod.Info("saving handshake frames to %s", mod.shakesFile) + mod.Debug("saving handshake frames to %s", mod.shakesFile) if err := mod.Session.WiFi.SaveHandshakesTo(mod.shakesFile, mod.handle.LinkType()); err != nil { mod.Error("error while saving handshake frames to %s: %s", mod.shakesFile, err) } From bf8a7659b5446ad68e9e9e2fa371efddc8d17906 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 23 Mar 2019 20:52:49 +0100 Subject: [PATCH 42/74] new: caplets will now include the list of sub scripts and files in api.rest --- caplets/caplet.go | 22 +++++++++++++++++----- caplets/manager.go | 28 ++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/caplets/caplet.go b/caplets/caplet.go index 8a876a4d..e838a370 100644 --- a/caplets/caplet.go +++ b/caplets/caplet.go @@ -8,22 +8,34 @@ import ( "github.com/evilsocket/islazy/fs" ) -type Caplet struct { - Name string `json:"name"` +type Script struct { Path string `json:"path"` Size int64 `json:"size"` Code []string `json:"code"` } -func NewCaplet(name string, path string, size int64) Caplet { - return Caplet{ - Name: name, +func newScript(path string, size int64) Script { + return Script{ Path: path, Size: size, Code: make([]string, 0), } } +type Caplet struct { + Script + Name string `json:"name"` + Scripts []Script `json:"scripts"` +} + +func NewCaplet(name string, path string, size int64) Caplet { + return Caplet{ + Script: newScript(path, size), + Name: name, + Scripts: make([]Script, 0), + } +} + func (cap *Caplet) Eval(argv []string, lineCb func(line string) error) error { if argv == nil { argv = []string{} diff --git a/caplets/manager.go b/caplets/manager.go index a3f3ebe7..acc84950 100644 --- a/caplets/manager.go +++ b/caplets/manager.go @@ -2,6 +2,7 @@ package caplets import ( "fmt" + "io/ioutil" "os" "path/filepath" "sort" @@ -68,10 +69,9 @@ func Load(name string) (error, *Caplet) { for _, fileName := range names { if stats, err := os.Stat(fileName); err == nil { cap := &Caplet{ - Name: baseName, - Path: fileName, - Code: make([]string, 0), - Size: stats.Size(), + Script: newScript(fileName, stats.Size()), + Name: baseName, + Scripts: make([]Script, 0), } cache[name] = cap @@ -84,6 +84,26 @@ func Load(name string) (error, *Caplet) { } cap.Code = append(cap.Code, line) } + + // the caplet has a dedicated folder + if strings.Contains(baseName, "/") || strings.Contains(baseName, "\\") { + dir := filepath.Dir(fileName) + // get all secondary .cap and .js files + if files, err := ioutil.ReadDir(dir); err == nil && len(files) > 0 { + for _, f := range files { + subFileName := filepath.Join(dir, f.Name()) + if subFileName != fileName && (strings.HasSuffix(subFileName, ".cap") || strings.HasSuffix(subFileName, ".js")) { + if reader, err := fs.LineReader(subFileName); err == nil { + script := newScript(subFileName, f.Size()) + for line := range reader { + script.Code = append(script.Code, line) + } + cap.Scripts = append(cap.Scripts, script) + } + } + } + } + } } return nil, cap From 08d85251dd911d6293af627d6fca505d8bc370f2 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sun, 24 Mar 2019 01:21:27 +0100 Subject: [PATCH 43/74] fix: caplet code is loaded entirely and comments are only skipped while evaluating it --- caplets/caplet.go | 4 ++++ caplets/manager.go | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/caplets/caplet.go b/caplets/caplet.go index e838a370..f4a6613a 100644 --- a/caplets/caplet.go +++ b/caplets/caplet.go @@ -44,6 +44,10 @@ func (cap *Caplet) Eval(argv []string, lineCb func(line string) error) error { // temporarily change the working directory return fs.Chdir(filepath.Dir(cap.Path), func() error { for _, line := range cap.Code { + // skip empty lines and comments + if line == "" || line[0] == '#' { + continue + } // replace $0 with argv[0], $1 with argv[1] and so on for i, arg := range argv { what := fmt.Sprintf("$%d", i) diff --git a/caplets/manager.go b/caplets/manager.go index acc84950..cb004d0c 100644 --- a/caplets/manager.go +++ b/caplets/manager.go @@ -79,9 +79,6 @@ func Load(name string) (error, *Caplet) { return fmt.Errorf("error reading caplet %s: %v", fileName, err), nil } else { for line := range reader { - if line == "" || line[0] == '#' { - continue - } cap.Code = append(cap.Code, line) } From 6785650887902ae6dca8dea8d764b0505ec36834 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sun, 24 Mar 2019 02:37:23 +0100 Subject: [PATCH 44/74] misc: v2.20 is near --- core/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/banner.go b/core/banner.go index 825af8e2..656eca51 100644 --- a/core/banner.go +++ b/core/banner.go @@ -2,7 +2,7 @@ package core const ( Name = "bettercap" - Version = "2.19" + Version = "2.20" Author = "Simone 'evilsocket' Margaritelli" Website = "https://bettercap.org/" ) From cfd93c555a1ecf6d560ebd7dfa461cbcdc738c82 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 26 Mar 2019 14:20:34 +0100 Subject: [PATCH 45/74] new: implemented (http|https).proxy.whitelist and (http|https).proxy.blacklist parameters (closes #508) --- modules/http_proxy/http_proxy.go | 17 ++++ modules/http_proxy/http_proxy_base.go | 58 +++++++++---- modules/http_proxy/http_proxy_base_filters.go | 82 ++++++++++--------- modules/https_proxy/https_proxy.go | 16 ++++ 4 files changed, 118 insertions(+), 55 deletions(-) diff --git a/modules/http_proxy/http_proxy.go b/modules/http_proxy/http_proxy.go index 90007b9f..7519a411 100644 --- a/modules/http_proxy/http_proxy.go +++ b/modules/http_proxy/http_proxy.go @@ -2,6 +2,8 @@ package http_proxy import ( "github.com/bettercap/bettercap/session" + + "github.com/evilsocket/islazy/str" ) type HttpProxy struct { @@ -38,6 +40,12 @@ func NewHttpProxy(s *session.Session) *HttpProxy { "", "URL, path or javascript code to inject into every HTML page.")) + mod.AddParam(session.NewStringParameter("http.proxy.blacklist", "", "", + "Comma separated list of hostnames to skip while proxying (wildcard expressions can be used).")) + + mod.AddParam(session.NewStringParameter("http.proxy.whitelist", "", "", + "Comma separated list of hostnames to proxy if the blacklist is used (wildcard expressions can be used).")) + mod.AddParam(session.NewBoolParameter("http.proxy.sslstrip", "false", "Enable or disable SSL stripping.")) @@ -77,6 +85,8 @@ func (mod *HttpProxy) Configure() error { var scriptPath string var stripSSL bool var jsToInject string + var blacklist string + var whitelist string if mod.Running() { return session.ErrAlreadyStarted @@ -92,8 +102,15 @@ func (mod *HttpProxy) Configure() error { return err } else if err, jsToInject = mod.StringParam("http.proxy.injectjs"); err != nil { return err + } else if err, blacklist = mod.StringParam("http.proxy.blacklist"); err != nil { + return err + } else if err, whitelist = mod.StringParam("http.proxy.whitelist"); err != nil { + return err } + mod.proxy.Blacklist = str.Comma(blacklist) + mod.proxy.Whitelist = str.Comma(whitelist) + return mod.proxy.Configure(address, proxyPort, httpPort, scriptPath, jsToInject, stripSSL) } diff --git a/modules/http_proxy/http_proxy_base.go b/modules/http_proxy/http_proxy_base.go index 8acaf564..7d77b7ba 100644 --- a/modules/http_proxy/http_proxy_base.go +++ b/modules/http_proxy/http_proxy_base.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "net/url" + "path/filepath" "strconv" "strings" "time" @@ -41,6 +42,8 @@ type HTTPProxy struct { Script *HttpProxyScript CertFile string KeyFile string + Blacklist []string + Whitelist []string jsHook string isTLS bool @@ -61,13 +64,15 @@ func stripPort(s string) string { func NewHTTPProxy(s *session.Session) *HTTPProxy { p := &HTTPProxy{ - Name: "http.proxy", - Proxy: goproxy.NewProxyHttpServer(), - sess: s, - stripper: NewSSLStripper(s, false), - isTLS: false, - Server: nil, - tag: session.AsTag("http.proxy"), + Name: "http.proxy", + Proxy: goproxy.NewProxyHttpServer(), + sess: s, + stripper: NewSSLStripper(s, false), + isTLS: false, + Server: nil, + Blacklist: make([]string, 0), + Whitelist: make([]string, 0), + tag: session.AsTag("http.proxy"), } p.Proxy.Verbose = false @@ -111,20 +116,41 @@ func (p *HTTPProxy) Fatal(format string, args ...interface{}) { } func (p *HTTPProxy) doProxy(req *http.Request) bool { - blacklist := []string{ - "localhost", - "127.0.0.1", - } - if req.Host == "" { p.Error("got request with empty host: %v", req) return false } - host := strings.Split(req.Host, ":")[0] - for _, blacklisted := range blacklist { - if host == blacklisted { - p.Error("got request with blacklisted host: %s", req.Host) + hostname := strings.Split(req.Host, ":")[0] + for _, local := range []string{"localhost", "127.0.0.1"} { + if hostname == local { + p.Error("got request with localed host: %s", req.Host) + return false + } + } + + return true +} + +func (p *HTTPProxy) shouldProxy(req *http.Request) bool { + hostname := strings.Split(req.Host, ":")[0] + + // check for the whitelist + for _, expr := range p.Whitelist { + if matched, err := filepath.Match(expr, hostname); err != nil { + p.Error("error while using proxy whitelist expression '%s': %v", expr, err) + } else if matched { + p.Debug("hostname '%s' matched whitelisted element '%s'", hostname, expr) + return true + } + } + + // then the blacklist + for _, expr := range p.Blacklist { + if matched, err := filepath.Match(expr, hostname); err != nil { + p.Error("error while using proxy blacklist expression '%s': %v", expr, err) + } else if matched { + p.Debug("hostname '%s' matched blacklisted element '%s'", hostname, expr) return false } } diff --git a/modules/http_proxy/http_proxy_base_filters.go b/modules/http_proxy/http_proxy_base_filters.go index d8ad23ce..52bd6e69 100644 --- a/modules/http_proxy/http_proxy_base_filters.go +++ b/modules/http_proxy/http_proxy_base_filters.go @@ -19,32 +19,34 @@ func (p *HTTPProxy) fixRequestHeaders(req *http.Request) { } func (p *HTTPProxy) onRequestFilter(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { - p.Debug("< %s %s %s%s", req.RemoteAddr, req.Method, req.Host, req.URL.Path) + if p.shouldProxy(req) { + p.Debug("< %s %s %s%s", req.RemoteAddr, req.Method, req.Host, req.URL.Path) - p.fixRequestHeaders(req) + p.fixRequestHeaders(req) - redir := p.stripper.Preprocess(req, ctx) - if redir != nil { - // we need to redirect the user in order to make - // some session cookie expire - return req, redir - } + redir := p.stripper.Preprocess(req, ctx) + if redir != nil { + // we need to redirect the user in order to make + // some session cookie expire + return req, redir + } - // do we have a proxy script? - if p.Script == nil { - return req, nil - } + // do we have a proxy script? + if p.Script == nil { + return req, nil + } - // run the module OnRequest callback if defined - jsreq, jsres := p.Script.OnRequest(req) - if jsreq != nil { - // the request has been changed by the script - p.logRequestAction(req, jsreq) - return jsreq.ToRequest(), nil - } else if jsres != nil { - // a fake response has been returned by the script - p.logResponseAction(req, jsres) - return req, jsres.ToResponse(req) + // run the module OnRequest callback if defined + jsreq, jsres := p.Script.OnRequest(req) + if jsreq != nil { + // the request has been changed by the script + p.logRequestAction(req, jsreq) + return jsreq.ToRequest(), nil + } else if jsres != nil { + // a fake response has been returned by the script + p.logResponseAction(req, jsres) + return req, jsres.ToResponse(req) + } } return req, nil @@ -123,28 +125,30 @@ func (p *HTTPProxy) onResponseFilter(res *http.Response, ctx *goproxy.ProxyCtx) return nil } - p.Debug("> %s %s %s%s", res.Request.RemoteAddr, res.Request.Method, res.Request.Host, res.Request.URL.Path) + if p.shouldProxy(res.Request) { + p.Debug("> %s %s %s%s", res.Request.RemoteAddr, res.Request.Method, res.Request.Host, res.Request.URL.Path) - p.fixResponseHeaders(res) + p.fixResponseHeaders(res) - p.stripper.Process(res, ctx) + p.stripper.Process(res, ctx) - // do we have a proxy script? - if p.Script != nil { - _, jsres := p.Script.OnResponse(res) - if jsres != nil { - // the response has been changed by the script - p.logResponseAction(res.Request, jsres) - return jsres.ToResponse(res.Request) + // do we have a proxy script? + if p.Script != nil { + _, jsres := p.Script.OnResponse(res) + if jsres != nil { + // the response has been changed by the script + p.logResponseAction(res.Request, jsres) + return jsres.ToResponse(res.Request) + } } - } - // inject javascript code if specified and needed - if doInject, cType := p.isScriptInjectable(res); doInject { - if err, injectedResponse := p.doScriptInjection(res, cType); err != nil { - p.Error("error while injecting javascript: %s", err) - } else if injectedResponse != nil { - return injectedResponse + // inject javascript code if specified and needed + if doInject, cType := p.isScriptInjectable(res); doInject { + if err, injectedResponse := p.doScriptInjection(res, cType); err != nil { + p.Error("error while injecting javascript: %s", err) + } else if injectedResponse != nil { + return injectedResponse + } } } diff --git a/modules/https_proxy/https_proxy.go b/modules/https_proxy/https_proxy.go index 25a74d08..27a26cf1 100644 --- a/modules/https_proxy/https_proxy.go +++ b/modules/https_proxy/https_proxy.go @@ -6,6 +6,7 @@ import ( "github.com/bettercap/bettercap/tls" "github.com/evilsocket/islazy/fs" + "github.com/evilsocket/islazy/str" ) type HttpsProxy struct { @@ -58,6 +59,12 @@ func NewHttpsProxy(s *session.Session) *HttpsProxy { "", "Path of a proxy JS script.")) + mod.AddParam(session.NewStringParameter("https.proxy.blacklist", "", "", + "Comma separated list of hostnames to skip while proxying (wildcard expressions can be used).")) + + mod.AddParam(session.NewStringParameter("https.proxy.whitelist", "", "", + "Comma separated list of hostnames to proxy if the blacklist is used (wildcard expressions can be used).")) + mod.AddHandler(session.NewModuleHandler("https.proxy on", "", "Start HTTPS proxy.", func(args []string) error { @@ -95,6 +102,8 @@ func (mod *HttpsProxy) Configure() error { var keyFile string var stripSSL bool var jsToInject string + var whitelist string + var blacklist string if mod.Running() { return session.ErrAlreadyStarted @@ -118,8 +127,15 @@ func (mod *HttpsProxy) Configure() error { return err } else if err, jsToInject = mod.StringParam("https.proxy.injectjs"); err != nil { return err + } else if err, blacklist = mod.StringParam("https.proxy.blacklist"); err != nil { + return err + } else if err, whitelist = mod.StringParam("https.proxy.whitelist"); err != nil { + return err } + mod.proxy.Blacklist = str.Comma(blacklist) + mod.proxy.Whitelist = str.Comma(whitelist) + if !fs.Exists(certFile) || !fs.Exists(keyFile) { err, cfg := tls.CertConfigFromModule("https.proxy", mod.SessionModule) if err != nil { From 7d46e7aa7adfd9dd4c43f79f4063615305f9a962 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 26 Mar 2019 14:33:14 +0100 Subject: [PATCH 46/74] fix: fixed a bug when AttEcodeSuccess was returned as an error by the gatt lib (fixes #498) --- modules/ble/ble_recon_events.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ble/ble_recon_events.go b/modules/ble/ble_recon_events.go index a19dc310..fed2b0a8 100644 --- a/modules/ble/ble_recon_events.go +++ b/modules/ble/ble_recon_events.go @@ -67,7 +67,8 @@ func (mod *BLERecon) onPeriphConnected(p gatt.Peripheral, err error) { mod.Info("connected, enumerating all the things for %s!", p.ID()) services, err := p.DiscoverServices(nil) - if err != nil { + // https://github.com/bettercap/bettercap/issues/498 + if err != nil && err.Error() != "success" { mod.Error("error discovering services: %s", err) return } From 529a1b48d9609f842f9d95c2c52095204c220137 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 26 Mar 2019 17:04:59 +0100 Subject: [PATCH 47/74] new: api.rest now exposes polled_at time field for the object session --- session/session_json.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/session/session_json.go b/session/session_json.go index 84bfe3db..fa3537e7 100644 --- a/session/session_json.go +++ b/session/session_json.go @@ -51,6 +51,7 @@ type sessionJSON struct { HID *network.HID `json:"hid"` Queue *packets.Queue `json:"packets"` StartedAt time.Time `json:"started_at"` + PolledAt time.Time `json:"polled_at"` Active bool `json:"active"` GPS GPS `json:"gps"` Modules ModuleList `json:"modules"` @@ -74,6 +75,7 @@ func (s *Session) MarshalJSON() ([]byte, error) { HID: s.HID, Queue: s.Queue, StartedAt: s.StartedAt, + PolledAt: time.Now(), Active: s.Active, GPS: s.GPS, Modules: s.Modules, From a3b730ce69c45f1e1cf5b1585757d9f165b0dcda Mon Sep 17 00:00:00 2001 From: evilsocket Date: Tue, 26 Mar 2019 20:30:33 +0100 Subject: [PATCH 48/74] fixed a bug caused by multiple wifi devices attached with different supported frequencies --- modules/wifi/wifi.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/modules/wifi/wifi.go b/modules/wifi/wifi.go index 998c2121..4ac37e34 100644 --- a/modules/wifi/wifi.go +++ b/modules/wifi/wifi.go @@ -441,23 +441,21 @@ func (mod *WiFiModule) Configure() error { mod.hopPeriod = time.Duration(hopPeriod) * time.Millisecond if mod.source == "" { - // No channels setted, retrieve frequencies supported by the card - if len(mod.frequencies) == 0 { - if freqs, err := network.GetSupportedFrequencies(ifName); err != nil { - return fmt.Errorf("error while getting supported frequencies of %s: %s", ifName, err) - } else { - mod.setFrequencies(freqs) - } - - mod.Debug("wifi supported frequencies: %v", mod.frequencies) - - // we need to start somewhere, this is just to check if - // this OS supports switching channel programmatically. - if err = network.SetInterfaceChannel(ifName, 1); err != nil { - return fmt.Errorf("error while initializing %s to channel 1: %s", ifName, err) - } - mod.Info("started (min rssi: %d dBm)", mod.minRSSI) + if freqs, err := network.GetSupportedFrequencies(ifName); err != nil { + return fmt.Errorf("error while getting supported frequencies of %s: %s", ifName, err) + } else { + mod.setFrequencies(freqs) } + + mod.Debug("wifi supported frequencies: %v", mod.frequencies) + + // we need to start somewhere, this is just to check if + // this OS supports switching channel programmatically. + if err = network.SetInterfaceChannel(ifName, 1); err != nil { + return fmt.Errorf("error while initializing %s to channel 1: %s", ifName, err) + } + + mod.Info("started (min rssi: %d dBm)", mod.minRSSI) } return nil From da2681375f0d1d241b2c89ba1fbdd88a86d5212a Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 13:48:21 +0100 Subject: [PATCH 49/74] fix: updated islazy/zip version to fiz a zip.Unzip related bug --- Gopkg.lock | 6 +-- .../github.com/evilsocket/islazy/zip/unzip.go | 42 +++++++++++++++---- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index bd28eebe..86f30cfe 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -83,7 +83,7 @@ revision = "2ce16c963a8ac5bd6af851d4877e38701346983f" [[projects]] - digest = "1:da1be9af4c3f262bd385cc722b08d98d4a47ddea57731e98b85c7ba21b35bc31" + digest = "1:5247f5757ba31623c464db149dc272a37604516d8fbae1561b36e0d7cee070a5" name = "github.com/evilsocket/islazy" packages = [ "data", @@ -96,8 +96,8 @@ "zip", ] pruneopts = "UT" - revision = "6ef79e84ded205e48f296d21e3bc65d1cf4f5c78" - version = "v1.10.3" + revision = "c5c7a41bb1c20e6df409825ed24af8de5fb7fb70" + version = "v1.10.4" [[projects]] branch = "master" diff --git a/vendor/github.com/evilsocket/islazy/zip/unzip.go b/vendor/github.com/evilsocket/islazy/zip/unzip.go index 79ec84b3..dc6edffb 100644 --- a/vendor/github.com/evilsocket/islazy/zip/unzip.go +++ b/vendor/github.com/evilsocket/islazy/zip/unzip.go @@ -13,6 +13,8 @@ import ( // within the zip file (parameter 1) to an output directory (parameter 2). // Credits to https://golangcode.com/unzip-files-in-go/ func Unzip(src string, dest string) ([]string, error) { + var outFile *os.File + var zipFile io.ReadCloser var filenames []string r, err := zip.OpenReader(src) @@ -21,33 +23,55 @@ func Unzip(src string, dest string) ([]string, error) { } defer r.Close() + clean := func() { + if outFile != nil { + outFile.Close() + outFile = nil + } + + if zipFile != nil { + zipFile.Close() + zipFile = nil + } + } + for _, f := range r.File { - rc, err := f.Open() + zipFile, err = f.Open() if err != nil { return filenames, err } - defer rc.Close() // Store filename/path for returning and using later on fpath := filepath.Join(dest, f.Name) // Check for ZipSlip. More Info: https://snyk.io/research/zip-slip-vulnerability#go if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { + clean() return filenames, fmt.Errorf("%s: illegal file path", fpath) } filenames = append(filenames, fpath) if f.FileInfo().IsDir() { os.MkdirAll(fpath, os.ModePerm) - } else if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { + clean() + continue + } + + if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { + clean() return filenames, err - } else if outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()); err != nil { + } + + outFile, err = os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + clean() + return filenames, err + } + + _, err = io.Copy(outFile, zipFile) + clean() + if err != nil { return filenames, err - } else { - defer outFile.Close() - if _, err = io.Copy(outFile, rc); err != nil { - return filenames, err - } } } From c99a5121fea7302943abb8397c2829aa897e69f6 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 13:48:52 +0100 Subject: [PATCH 50/74] new: [IMPORTANT] net.recon is not in the autostarted list of modules anymore --- core/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/options.go b/core/options.go index e1a478e2..62d4e33d 100644 --- a/core/options.go +++ b/core/options.go @@ -22,7 +22,7 @@ func ParseOptions() (Options, error) { o := Options{ InterfaceName: flag.String("iface", "", "Network interface to bind to, if empty the default interface will be auto selected."), Gateway: flag.String("gateway-override", "", "Use the provided IP address instead of the default gateway. If not specified or invalid, the default gateway will be used."), - AutoStart: flag.String("autostart", "events.stream, net.recon", "Comma separated list of modules to auto start."), + AutoStart: flag.String("autostart", "events.stream", "Comma separated list of modules to auto start."), Caplet: flag.String("caplet", "", "Read commands from this file and execute them in the interactive session."), Debug: flag.Bool("debug", false, "Print debug messages."), PrintVersion: flag.Bool("version", false, "Print the version and exit."), From 92758d7f7ecd68cdfa9e6332cebfa08ec490b1eb Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 13:49:34 +0100 Subject: [PATCH 51/74] fix: http and https servers file access logs are now debug logs --- modules/http_server/http_server.go | 2 +- modules/https_server/https_server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/http_server/http_server.go b/modules/http_server/http_server.go index a398dfbb..4b434429 100644 --- a/modules/http_server/http_server.go +++ b/modules/http_server/http_server.go @@ -82,7 +82,7 @@ func (mod *HttpServer) Configure() error { fileServer := http.FileServer(http.Dir(path)) router.HandleFunc("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - mod.Info("%s %s %s%s", tui.Bold(strings.Split(r.RemoteAddr, ":")[0]), r.Method, r.Host, r.URL.Path) + mod.Debug("%s %s %s%s", tui.Bold(strings.Split(r.RemoteAddr, ":")[0]), r.Method, r.Host, r.URL.Path) fileServer.ServeHTTP(w, r) })) diff --git a/modules/https_server/https_server.go b/modules/https_server/https_server.go index ae26c009..60f998f1 100644 --- a/modules/https_server/https_server.go +++ b/modules/https_server/https_server.go @@ -100,7 +100,7 @@ func (mod *HttpsServer) Configure() error { fileServer := http.FileServer(http.Dir(path)) router.HandleFunc("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - mod.Info("%s %s %s%s", tui.Bold(strings.Split(r.RemoteAddr, ":")[0]), r.Method, r.Host, r.URL.Path) + mod.Debug("%s %s %s%s", tui.Bold(strings.Split(r.RemoteAddr, ":")[0]), r.Method, r.Host, r.URL.Path) fileServer.ServeHTTP(w, r) })) From fbcaf2989f8b82d6dbff081b00d067837ad5419e Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 13:50:00 +0100 Subject: [PATCH 52/74] new: ui module to install/update the UI from github releases --- modules/modules.go | 7 +- modules/ui/ui.go | 190 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 modules/ui/ui.go diff --git a/modules/modules.go b/modules/modules.go index 15e15c4b..fed48f25 100644 --- a/modules/modules.go +++ b/modules/modules.go @@ -24,6 +24,7 @@ import ( "github.com/bettercap/bettercap/modules/syn_scan" "github.com/bettercap/bettercap/modules/tcp_proxy" "github.com/bettercap/bettercap/modules/ticker" + "github.com/bettercap/bettercap/modules/ui" "github.com/bettercap/bettercap/modules/update" "github.com/bettercap/bettercap/modules/wifi" "github.com/bettercap/bettercap/modules/wol" @@ -36,7 +37,6 @@ func LoadModules(sess *session.Session) { sess.Register(arp_spoof.NewArpSpoofer(sess)) sess.Register(api_rest.NewRestAPI(sess)) sess.Register(ble.NewBLERecon(sess)) - sess.Register(caplets.NewCapletsModule(sess)) sess.Register(dhcp6_spoof.NewDHCP6Spoofer(sess)) sess.Register(net_recon.NewDiscovery(sess)) sess.Register(dns_spoof.NewDNSSpoofer(sess)) @@ -54,8 +54,11 @@ func LoadModules(sess *session.Session) { sess.Register(syn_scan.NewSynScanner(sess)) sess.Register(tcp_proxy.NewTcpProxy(sess)) sess.Register(ticker.NewTicker(sess)) - sess.Register(update.NewUpdateModule(sess)) sess.Register(wifi.NewWiFiModule(sess)) sess.Register(wol.NewWOL(sess)) sess.Register(hid.NewHIDRecon(sess)) + + sess.Register(caplets.NewCapletsModule(sess)) + sess.Register(update.NewUpdateModule(sess)) + sess.Register(ui.NewUIModule(sess)) } diff --git a/modules/ui/ui.go b/modules/ui/ui.go new file mode 100644 index 00000000..f5d6950e --- /dev/null +++ b/modules/ui/ui.go @@ -0,0 +1,190 @@ +package ui + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "regexp" + + "github.com/bettercap/bettercap/session" + + "github.com/google/go-github/github" + + "github.com/evilsocket/islazy/fs" + "github.com/evilsocket/islazy/tui" + "github.com/evilsocket/islazy/zip" +) + +var versionParser = regexp.MustCompile(`name:"ui",version:"([^"]+)"`) + +type UIModule struct { + session.SessionModule + client *github.Client + tmpFile string + basePath string + uiPath string +} + +func NewUIModule(s *session.Session) *UIModule { + mod := &UIModule{ + SessionModule: session.NewSessionModule("ui", s), + client: github.NewClient(nil), + } + + mod.AddParam(session.NewStringParameter("ui.basepath", + "/usr/local/share/bettercap/", + "", + "UI base installation path.")) + + mod.AddParam(session.NewStringParameter("ui.tmpfile", + filepath.Join(os.TempDir(), "ui.zip"), + "", + "Temporary file to use while downloading UI updates.")) + + mod.AddHandler(session.NewModuleHandler("ui.version", "", + "Print the currently installed UI version.", + func(args []string) error { + return mod.showVersion() + })) + + mod.AddHandler(session.NewModuleHandler("ui.update", "", + "Download the latest available version of the UI and install it.", + func(args []string) error { + return mod.Start() + })) + + return mod +} + +func (mod *UIModule) Name() string { + return "ui" +} + +func (mod *UIModule) Description() string { + return "A module to manage bettercap's UI updates and installed version." +} + +func (mod *UIModule) Author() string { + return "Simone Margaritelli " +} + +func (mod *UIModule) Configure() (err error) { + if err, mod.basePath = mod.StringParam("ui.basepath"); err != nil { + return err + } else { + mod.uiPath = filepath.Join(mod.basePath, "ui") + } + + if err, mod.tmpFile = mod.StringParam("ui.tmpfile"); err != nil { + return err + } + + return nil +} + +func (mod *UIModule) Stop() error { + return nil +} + +func (mod *UIModule) download(version, url string) error { + if !fs.Exists(mod.basePath) { + mod.Warning("creating ui install path %s ...", mod.basePath) + if err := os.MkdirAll(mod.basePath, os.ModePerm); err != nil { + return err + } + } + + out, err := os.Create(mod.tmpFile) + if err != nil { + return err + } + defer out.Close() + defer os.Remove(mod.tmpFile) + + mod.Info("downloading ui %s from %s ...", tui.Bold(version), url) + + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if _, err := io.Copy(out, resp.Body); err != nil { + return err + } + + if fs.Exists(mod.uiPath) { + mod.Warning("removing previously installed UI from %s ...", mod.uiPath) + if err := os.RemoveAll(mod.uiPath); err != nil { + return err + } + } + + mod.Info("installing to %s ...", mod.uiPath) + + if _, err = zip.Unzip(mod.tmpFile, mod.basePath); err != nil { + return err + } + + mod.Info("installation complete, you can now run the %s (or https-ui) caplet to start the UI.", tui.Bold("http-ui")) + + return nil +} + +func (mod *UIModule) showVersion() error { + if err := mod.Configure(); err != nil { + return err + } + + if !fs.Exists(mod.uiPath) { + return fmt.Errorf("path %s does not exist, ui not installed", mod.uiPath) + } + + search := filepath.Join(mod.uiPath, "/main.*.js") + matches, err := filepath.Glob(search) + if err != nil { + return err + } else if len(matches) == 0 { + return fmt.Errorf("can't find any main.*.js files in %s", mod.uiPath) + } + + for _, filename := range matches { + if raw, err := ioutil.ReadFile(filename); err != nil { + return err + } else if m := versionParser.FindStringSubmatch(string(raw)); m != nil { + version := m[1] + mod.Info("v%s", version) + return nil + } + } + + return fmt.Errorf("can't parse version from %s", search) +} + +func (mod *UIModule) Start() error { + if err := mod.Configure(); err != nil { + return err + } else if err := mod.SetRunning(true, nil); err != nil { + return err + } + defer mod.SetRunning(false, nil) + + mod.Info("checking latest stable release ...") + + if releases, _, err := mod.client.Repositories.ListReleases(context.Background(), "bettercap", "ui", nil); err == nil { + latest := releases[0] + for _, a := range latest.Assets { + if *a.Name == "ui.zip" { + return mod.download(*latest.TagName, *a.BrowserDownloadURL) + } + } + } else { + mod.Error("error while fetching latest release info from GitHub: %s", err) + } + + return nil +} From 40ec724ca6281e183b1e1f33ff87a129117a9346 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 13:59:22 +0100 Subject: [PATCH 53/74] fix: reporting module name when it's already running or already stopped --- modules/any_proxy/any_proxy.go | 2 +- modules/api_rest/api_rest.go | 2 +- modules/ble/ble_recon.go | 4 ++-- modules/dhcp6_spoof/dhcp6_spoof.go | 2 +- modules/dns_spoof/dns_spoof.go | 2 +- modules/gps/gps.go | 2 +- modules/hid/hid.go | 2 +- modules/http_proxy/http_proxy.go | 2 +- modules/http_server/http_server.go | 2 +- modules/https_proxy/https_proxy.go | 2 +- modules/https_server/https_server.go | 2 +- modules/mac_changer/mac_changer.go | 2 +- modules/mysql_server/mysql_server.go | 2 +- modules/net_sniff/net_sniff.go | 2 +- modules/packet_proxy/packet_proxy_linux_amd64.go | 2 +- modules/tcp_proxy/tcp_proxy.go | 2 +- modules/ticker/ticker.go | 2 +- modules/wifi/wifi_ap.go | 2 +- session/module.go | 4 ++-- session/session.go | 12 +++++++++--- 20 files changed, 30 insertions(+), 24 deletions(-) diff --git a/modules/any_proxy/any_proxy.go b/modules/any_proxy/any_proxy.go index 427c0499..a8977d27 100644 --- a/modules/any_proxy/any_proxy.go +++ b/modules/any_proxy/any_proxy.go @@ -80,7 +80,7 @@ func (mod *AnyProxy) Configure() error { var dstAddress string if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, iface = mod.StringParam("any.proxy.iface"); err != nil { return err } else if err, protocol = mod.StringParam("any.proxy.protocol"); err != nil { diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 406cb7fa..6f7fc11d 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -126,7 +126,7 @@ func (mod *RestAPI) Configure() error { var port int if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, ip = mod.StringParam("api.rest.address"); err != nil { return err } else if err, port = mod.IntParam("api.rest.port"); err != nil { diff --git a/modules/ble/ble_recon.go b/modules/ble/ble_recon.go index b44c7746..ef54a5be 100644 --- a/modules/ble/ble_recon.go +++ b/modules/ble/ble_recon.go @@ -140,7 +140,7 @@ func (w dummyWriter) Write(p []byte) (n int, err error) { func (mod *BLERecon) Configure() (err error) { if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if mod.gattDevice == nil { mod.Debug("initializing device ...") @@ -239,7 +239,7 @@ func (mod *BLERecon) enumAllTheThings(mac string) error { } mod.setCurrentDevice(dev) - if err := mod.Configure(); err != nil && err != session.ErrAlreadyStarted { + if err := mod.Configure(); err != nil && err != session.ErrAlreadyStarted(mod.Name()) { return err } diff --git a/modules/dhcp6_spoof/dhcp6_spoof.go b/modules/dhcp6_spoof/dhcp6_spoof.go index f7a7a325..2b0f729b 100644 --- a/modules/dhcp6_spoof/dhcp6_spoof.go +++ b/modules/dhcp6_spoof/dhcp6_spoof.go @@ -78,7 +78,7 @@ func (mod *DHCP6Spoofer) Configure() error { var err error if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } if mod.Handle, err = pcap.OpenLive(mod.Session.Interface.Name(), 65536, true, pcap.BlockForever); err != nil { diff --git a/modules/dns_spoof/dns_spoof.go b/modules/dns_spoof/dns_spoof.go index 6905ff75..2f6bf5f4 100644 --- a/modules/dns_spoof/dns_spoof.go +++ b/modules/dns_spoof/dns_spoof.go @@ -87,7 +87,7 @@ func (mod *DNSSpoofer) Configure() error { var address net.IP if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if mod.Handle, err = pcap.OpenLive(mod.Session.Interface.Name(), 65536, true, pcap.BlockForever); err != nil { return err } else if err = mod.Handle.SetBPFFilter("udp"); err != nil { diff --git a/modules/gps/gps.go b/modules/gps/gps.go index 65c92596..8e46e331 100644 --- a/modules/gps/gps.go +++ b/modules/gps/gps.go @@ -72,7 +72,7 @@ func (mod *GPS) Author() string { func (mod *GPS) Configure() (err error) { if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, mod.serialPort = mod.StringParam("gps.device"); err != nil { return err } else if err, mod.baudRate = mod.IntParam("gps.baudrate"); err != nil { diff --git a/modules/hid/hid.go b/modules/hid/hid.go index 134f8cb2..424267f2 100644 --- a/modules/hid/hid.go +++ b/modules/hid/hid.go @@ -163,7 +163,7 @@ func (mod *HIDRecon) Configure() error { var n int if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } if err, mod.useLNA = mod.BoolParam("hid.lna"); err != nil { diff --git a/modules/http_proxy/http_proxy.go b/modules/http_proxy/http_proxy.go index 7519a411..9cec358d 100644 --- a/modules/http_proxy/http_proxy.go +++ b/modules/http_proxy/http_proxy.go @@ -89,7 +89,7 @@ func (mod *HttpProxy) Configure() error { var whitelist string if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, address = mod.StringParam("http.proxy.address"); err != nil { return err } else if err, proxyPort = mod.IntParam("http.proxy.port"); err != nil { diff --git a/modules/http_server/http_server.go b/modules/http_server/http_server.go index 4b434429..d5ad1845 100644 --- a/modules/http_server/http_server.go +++ b/modules/http_server/http_server.go @@ -71,7 +71,7 @@ func (mod *HttpServer) Configure() error { var port int if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } if err, path = mod.StringParam("http.server.path"); err != nil { diff --git a/modules/https_proxy/https_proxy.go b/modules/https_proxy/https_proxy.go index 27a26cf1..10bfaca1 100644 --- a/modules/https_proxy/https_proxy.go +++ b/modules/https_proxy/https_proxy.go @@ -106,7 +106,7 @@ func (mod *HttpsProxy) Configure() error { var blacklist string if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, address = mod.StringParam("https.proxy.address"); err != nil { return err } else if err, proxyPort = mod.IntParam("https.proxy.port"); err != nil { diff --git a/modules/https_server/https_server.go b/modules/https_server/https_server.go index 60f998f1..6d4a4813 100644 --- a/modules/https_server/https_server.go +++ b/modules/https_server/https_server.go @@ -89,7 +89,7 @@ func (mod *HttpsServer) Configure() error { var keyFile string if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } if err, path = mod.StringParam("https.server.path"); err != nil { diff --git a/modules/mac_changer/mac_changer.go b/modules/mac_changer/mac_changer.go index b0a80acf..a991cfb1 100644 --- a/modules/mac_changer/mac_changer.go +++ b/modules/mac_changer/mac_changer.go @@ -103,7 +103,7 @@ func (mod *MacChanger) setMac(mac net.HardwareAddr) error { func (mod *MacChanger) Start() error { if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err := mod.Configure(); err != nil { return err } else if err := mod.setMac(mod.fakeMac); err != nil { diff --git a/modules/mysql_server/mysql_server.go b/modules/mysql_server/mysql_server.go index eb145c9b..3ddb1d97 100644 --- a/modules/mysql_server/mysql_server.go +++ b/modules/mysql_server/mysql_server.go @@ -79,7 +79,7 @@ func (mod *MySQLServer) Configure() error { var port int if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, mod.infile = mod.StringParam("mysql.server.infile"); err != nil { return err } else if err, mod.outfile = mod.StringParam("mysql.server.outfile"); err != nil { diff --git a/modules/net_sniff/net_sniff.go b/modules/net_sniff/net_sniff.go index 97dc6082..d97780c2 100644 --- a/modules/net_sniff/net_sniff.go +++ b/modules/net_sniff/net_sniff.go @@ -146,7 +146,7 @@ func (mod *Sniffer) Configure() error { var err error if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, mod.Ctx = mod.GetContext(); err != nil { if mod.Ctx != nil { mod.Ctx.Close() diff --git a/modules/packet_proxy/packet_proxy_linux_amd64.go b/modules/packet_proxy/packet_proxy_linux_amd64.go index f934c13b..0f4f7243 100644 --- a/modules/packet_proxy/packet_proxy_linux_amd64.go +++ b/modules/packet_proxy/packet_proxy_linux_amd64.go @@ -190,7 +190,7 @@ func dummyCallback(payload *nfqueue.Payload) int { func (mod *PacketProxy) Start() error { if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err := mod.Configure(); err != nil { return err } diff --git a/modules/tcp_proxy/tcp_proxy.go b/modules/tcp_proxy/tcp_proxy.go index 1a13e3bb..23ac6041 100644 --- a/modules/tcp_proxy/tcp_proxy.go +++ b/modules/tcp_proxy/tcp_proxy.go @@ -95,7 +95,7 @@ func (mod *TcpProxy) Configure() error { var tunnelPort int if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, address = mod.StringParam("tcp.address"); err != nil { return err } else if err, proxyAddress = mod.StringParam("tcp.proxy.address"); err != nil { diff --git a/modules/ticker/ticker.go b/modules/ticker/ticker.go index 5e62139c..3edaafc3 100644 --- a/modules/ticker/ticker.go +++ b/modules/ticker/ticker.go @@ -59,7 +59,7 @@ func (mod *Ticker) Configure() error { var period int if mod.Running() { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } else if err, commands = mod.StringParam("ticker.commands"); err != nil { return err } else if err, period = mod.IntParam("ticker.period"); err != nil { diff --git a/modules/wifi/wifi_ap.go b/modules/wifi/wifi_ap.go index fcc4c29a..7a5ed0b7 100644 --- a/modules/wifi/wifi_ap.go +++ b/modules/wifi/wifi_ap.go @@ -35,7 +35,7 @@ func (mod *WiFiModule) startAp() error { if !mod.Running() { return errNoRecon } else if mod.apRunning { - return session.ErrAlreadyStarted + return session.ErrAlreadyStarted(mod.Name()) } go func() { diff --git a/session/module.go b/session/module.go index dbdf6d1b..149552d1 100644 --- a/session/module.go +++ b/session/module.go @@ -230,9 +230,9 @@ func (m *SessionModule) Running() bool { func (m *SessionModule) SetRunning(running bool, cb func()) error { if running == m.Running() { if m.Started { - return ErrAlreadyStarted + return ErrAlreadyStarted(m.Name) } else { - return ErrAlreadyStopped + return ErrAlreadyStopped(m.Name) } } diff --git a/session/session.go b/session/session.go index ba49a03d..e4661086 100644 --- a/session/session.go +++ b/session/session.go @@ -35,14 +35,20 @@ const ( var ( I = (*Session)(nil) - ErrAlreadyStarted = errors.New("module is already running") - ErrAlreadyStopped = errors.New("module is not running") - ErrNotSupported = errors.New("this component is not supported on this OS") + ErrNotSupported = errors.New("this component is not supported on this OS") reCmdSpaceCleaner = regexp.MustCompile(`^([^\s]+)\s+(.+)$`) reEnvVarCapture = regexp.MustCompile(`{env\.([^}]+)}`) ) +func ErrAlreadyStarted(name string) error { + return fmt.Errorf("module %s is already running", name) +} + +func ErrAlreadyStopped(name string) error { + return fmt.Errorf("module %s is not running", name) +} + type UnknownCommandCallback func(cmd string) bool type GPS struct { From d9a79aace793bd03637d5c3d5824ea5839a7cfe3 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 15:08:41 +0100 Subject: [PATCH 54/74] Releasing v2.21 --- core/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/banner.go b/core/banner.go index 656eca51..075ae2e4 100644 --- a/core/banner.go +++ b/core/banner.go @@ -2,7 +2,7 @@ package core const ( Name = "bettercap" - Version = "2.20" + Version = "2.21" Author = "Simone 'evilsocket' Margaritelli" Website = "https://bettercap.org/" ) From a59c51b825741848ec0f20d65976ae982800c88b Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 16:13:41 +0100 Subject: [PATCH 55/74] fix: fixed a bug which broke ble.enum --- modules/ble/ble_recon.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ble/ble_recon.go b/modules/ble/ble_recon.go index ef54a5be..afb2e216 100644 --- a/modules/ble/ble_recon.go +++ b/modules/ble/ble_recon.go @@ -239,7 +239,7 @@ func (mod *BLERecon) enumAllTheThings(mac string) error { } mod.setCurrentDevice(dev) - if err := mod.Configure(); err != nil && err != session.ErrAlreadyStarted(mod.Name()) { + if err := mod.Configure(); err != nil && err.Error() != session.ErrAlreadyStarted("ble.recon").Error() { return err } From 8085e84394d436d03225be06d9be67e9aaee9169 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 16:14:14 +0100 Subject: [PATCH 56/74] Releasing v2.21.1 --- core/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/banner.go b/core/banner.go index 075ae2e4..3374ee60 100644 --- a/core/banner.go +++ b/core/banner.go @@ -2,7 +2,7 @@ package core const ( Name = "bettercap" - Version = "2.21" + Version = "2.21.1" Author = "Simone 'evilsocket' Margaritelli" Website = "https://bettercap.org/" ) From 4713d25ea7a8137f50cf32eab945869c0ea741ca Mon Sep 17 00:00:00 2001 From: evilsocket Date: Wed, 27 Mar 2019 16:47:28 +0100 Subject: [PATCH 57/74] misc: small fix or general refactoring i did not bother commenting --- modules/dns_spoof/dns_spoof_hosts.go | 2 +- modules/utils/script_builtins.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/dns_spoof/dns_spoof_hosts.go b/modules/dns_spoof/dns_spoof_hosts.go index adb6c3c3..efc0f12d 100644 --- a/modules/dns_spoof/dns_spoof_hosts.go +++ b/modules/dns_spoof/dns_spoof_hosts.go @@ -46,7 +46,7 @@ func NewHostEntry(host string, address net.IP) HostEntry { return entry } -func HostsFromFile(filename string,defaultAddress net.IP) (err error, entries []HostEntry) { +func HostsFromFile(filename string, defaultAddress net.IP) (err error, entries []HostEntry) { input, err := os.Open(filename) if err != nil { return diff --git a/modules/utils/script_builtins.go b/modules/utils/script_builtins.go index 3e4e7426..cc49ec30 100644 --- a/modules/utils/script_builtins.go +++ b/modules/utils/script_builtins.go @@ -38,7 +38,7 @@ func init() { entry_list := []string{} for _, file := range dir { - entry_list = append( entry_list, file.Name() ) + entry_list = append(entry_list, file.Name()) } v, err := otto.Otto.ToValue(*call.Otto, entry_list) @@ -167,21 +167,21 @@ func init() { return errOtto("gzipCompress: expected 1 argument, %d given instead.", argc) } - uncompressed_bytes := []byte( argv[0].String() ) + uncompressed_bytes := []byte(argv[0].String()) var writer_buffer bytes.Buffer gzip_writer := gzip.NewWriter(&writer_buffer) _, err := gzip_writer.Write(uncompressed_bytes) if err != nil { - return errOtto( "gzipCompress: could not compress data: %s", err.Error() ) + return errOtto("gzipCompress: could not compress data: %s", err.Error()) } gzip_writer.Close() compressed_bytes := writer_buffer.Bytes() - v, err := otto.ToValue( string(compressed_bytes) ) + v, err := otto.ToValue(string(compressed_bytes)) if err != nil { - return errOtto( "Could not convert to string: %s", err.Error() ) + return errOtto("Could not convert to string: %s", err.Error()) } return v @@ -195,24 +195,24 @@ func init() { return errOtto("gzipDecompress: expected 1 argument, %d given instead.", argc) } - compressed_bytes := []byte( argv[0].String() ) + compressed_bytes := []byte(argv[0].String()) reader_buffer := bytes.NewBuffer(compressed_bytes) gzip_reader, err := gzip.NewReader(reader_buffer) if err != nil { - return errOtto( "gzipDecompress: could not create gzip reader: %s", err.Error() ) + return errOtto("gzipDecompress: could not create gzip reader: %s", err.Error()) } var decompressed_buffer bytes.Buffer _, err = decompressed_buffer.ReadFrom(gzip_reader) if err != nil { - return errOtto( "gzipDecompress: could not decompress data: %s", err.Error() ) + return errOtto("gzipDecompress: could not decompress data: %s", err.Error()) } decompressed_bytes := decompressed_buffer.Bytes() - v, err := otto.ToValue( string(decompressed_bytes) ) + v, err := otto.ToValue(string(decompressed_bytes)) if err != nil { - return errOtto( "Could not convert to string: %s", err.Error() ) + return errOtto("Could not convert to string: %s", err.Error()) } return v From 0a31ac81677cc7d4dded72ef762351ac7b4403e2 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 16:20:31 +0100 Subject: [PATCH 58/74] new: implemented api.rest.record and api.rest.replay --- Gopkg.lock | 39 + Gopkg.toml | 8 + core/banner.go | 2 +- modules/api_rest/api_rest.go | 55 +- modules/api_rest/api_rest_controller.go | 176 +++-- modules/api_rest/api_rest_record.go | 107 +++ modules/api_rest/api_rest_replay.go | 63 ++ modules/api_rest/record.go | 233 ++++++ session.record | Bin 0 -> 65748 bytes session/events.go | 11 +- session/session_json.go | 4 +- vendor/github.com/dsnet/compress/.travis.yml | 36 + vendor/github.com/dsnet/compress/LICENSE.md | 24 + vendor/github.com/dsnet/compress/README.md | 75 ++ vendor/github.com/dsnet/compress/api.go | 74 ++ vendor/github.com/dsnet/compress/bzip2/bwt.go | 110 +++ .../github.com/dsnet/compress/bzip2/common.go | 110 +++ .../dsnet/compress/bzip2/fuzz_off.go | 13 + .../dsnet/compress/bzip2/fuzz_on.go | 77 ++ .../compress/bzip2/internal/sais/common.go | 28 + .../compress/bzip2/internal/sais/sais_byte.go | 661 ++++++++++++++++ .../compress/bzip2/internal/sais/sais_gen.go | 703 ++++++++++++++++++ .../compress/bzip2/internal/sais/sais_int.go | 661 ++++++++++++++++ .../dsnet/compress/bzip2/mtf_rle2.go | 131 ++++ .../github.com/dsnet/compress/bzip2/prefix.go | 374 ++++++++++ .../github.com/dsnet/compress/bzip2/reader.go | 274 +++++++ .../github.com/dsnet/compress/bzip2/rle1.go | 101 +++ .../github.com/dsnet/compress/bzip2/writer.go | 307 ++++++++ vendor/github.com/dsnet/compress/go.mod | 10 + vendor/github.com/dsnet/compress/go.sum | 8 + .../dsnet/compress/internal/common.go | 107 +++ .../dsnet/compress/internal/debug.go | 12 + .../dsnet/compress/internal/errors/errors.go | 120 +++ .../dsnet/compress/internal/gofuzz.go | 12 + .../dsnet/compress/internal/prefix/debug.go | 159 ++++ .../dsnet/compress/internal/prefix/decoder.go | 136 ++++ .../dsnet/compress/internal/prefix/encoder.go | 66 ++ .../dsnet/compress/internal/prefix/prefix.go | 400 ++++++++++ .../dsnet/compress/internal/prefix/range.go | 93 +++ .../dsnet/compress/internal/prefix/reader.go | 335 +++++++++ .../dsnet/compress/internal/prefix/wrap.go | 146 ++++ .../dsnet/compress/internal/prefix/writer.go | 166 +++++ .../dsnet/compress/internal/release.go | 21 + vendor/github.com/dsnet/compress/zbench.sh | 12 + vendor/github.com/dsnet/compress/zfuzz.sh | 10 + vendor/github.com/dsnet/compress/zprof.sh | 54 ++ vendor/github.com/dsnet/compress/ztest.sh | 54 ++ .../github.com/icedream/go-bsdiff/.gitignore | 2 + .../github.com/icedream/go-bsdiff/README.md | 89 +++ .../icedream/go-bsdiff/bsdiff/LICENSE | 24 + .../icedream/go-bsdiff/diff/diff.go | 34 + vendor/github.com/icedream/go-bsdiff/go.mod | 11 + vendor/github.com/icedream/go-bsdiff/go.sum | 14 + .../icedream/go-bsdiff/internal/magic.go | 39 + .../icedream/go-bsdiff/internal/native/cgo.c | 56 ++ .../icedream/go-bsdiff/internal/native/cgo.h | 15 + .../go-bsdiff/internal/native/cgo_read.go | 43 ++ .../go-bsdiff/internal/native/cgo_write.go | 18 + .../go-bsdiff/internal/native/diff.go | 29 + .../go-bsdiff/internal/native/ext_bsdiff.c | 2 + .../go-bsdiff/internal/native/native.go | 31 + .../go-bsdiff/internal/native/patch.go | 30 + .../go-bsdiff/internal/native/table_reader.go | 44 ++ .../go-bsdiff/internal/native/table_writer.go | 44 ++ vendor/github.com/icedream/go-bsdiff/main.go | 16 + .../icedream/go-bsdiff/patch/patch.go | 31 + vendor/github.com/kr/binarydist/.gitignore | 1 + vendor/github.com/kr/binarydist/License | 22 + vendor/github.com/kr/binarydist/Readme.md | 7 + vendor/github.com/kr/binarydist/bzip2.go | 40 + vendor/github.com/kr/binarydist/diff.go | 408 ++++++++++ vendor/github.com/kr/binarydist/doc.go | 24 + vendor/github.com/kr/binarydist/encoding.go | 53 ++ vendor/github.com/kr/binarydist/go.mod | 1 + vendor/github.com/kr/binarydist/patch.go | 109 +++ vendor/github.com/kr/binarydist/seek.go | 43 ++ 76 files changed, 7610 insertions(+), 48 deletions(-) create mode 100644 modules/api_rest/api_rest_record.go create mode 100644 modules/api_rest/api_rest_replay.go create mode 100644 modules/api_rest/record.go create mode 100755 session.record create mode 100644 vendor/github.com/dsnet/compress/.travis.yml create mode 100644 vendor/github.com/dsnet/compress/LICENSE.md create mode 100644 vendor/github.com/dsnet/compress/README.md create mode 100644 vendor/github.com/dsnet/compress/api.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/bwt.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/common.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/fuzz_off.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/fuzz_on.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/prefix.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/reader.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/rle1.go create mode 100644 vendor/github.com/dsnet/compress/bzip2/writer.go create mode 100644 vendor/github.com/dsnet/compress/go.mod create mode 100644 vendor/github.com/dsnet/compress/go.sum create mode 100644 vendor/github.com/dsnet/compress/internal/common.go create mode 100644 vendor/github.com/dsnet/compress/internal/debug.go create mode 100644 vendor/github.com/dsnet/compress/internal/errors/errors.go create mode 100644 vendor/github.com/dsnet/compress/internal/gofuzz.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/debug.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/decoder.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/encoder.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/prefix.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/range.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/reader.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/wrap.go create mode 100644 vendor/github.com/dsnet/compress/internal/prefix/writer.go create mode 100644 vendor/github.com/dsnet/compress/internal/release.go create mode 100755 vendor/github.com/dsnet/compress/zbench.sh create mode 100755 vendor/github.com/dsnet/compress/zfuzz.sh create mode 100755 vendor/github.com/dsnet/compress/zprof.sh create mode 100755 vendor/github.com/dsnet/compress/ztest.sh create mode 100644 vendor/github.com/icedream/go-bsdiff/.gitignore create mode 100644 vendor/github.com/icedream/go-bsdiff/README.md create mode 100644 vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE create mode 100644 vendor/github.com/icedream/go-bsdiff/diff/diff.go create mode 100644 vendor/github.com/icedream/go-bsdiff/go.mod create mode 100644 vendor/github.com/icedream/go-bsdiff/go.sum create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/magic.go create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/diff.go create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/native.go create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/patch.go create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go create mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go create mode 100644 vendor/github.com/icedream/go-bsdiff/main.go create mode 100644 vendor/github.com/icedream/go-bsdiff/patch/patch.go create mode 100644 vendor/github.com/kr/binarydist/.gitignore create mode 100644 vendor/github.com/kr/binarydist/License create mode 100644 vendor/github.com/kr/binarydist/Readme.md create mode 100644 vendor/github.com/kr/binarydist/bzip2.go create mode 100644 vendor/github.com/kr/binarydist/diff.go create mode 100644 vendor/github.com/kr/binarydist/doc.go create mode 100644 vendor/github.com/kr/binarydist/encoding.go create mode 100644 vendor/github.com/kr/binarydist/go.mod create mode 100644 vendor/github.com/kr/binarydist/patch.go create mode 100644 vendor/github.com/kr/binarydist/seek.go diff --git a/Gopkg.lock b/Gopkg.lock index 86f30cfe..53a365ec 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -66,6 +66,21 @@ pruneopts = "UT" revision = "61ca646babef3bd4dea1deb610bfb0005c0a1298" +[[projects]] + digest = "1:d052bda13fd17bd8cf52ccae57b0a03ffeea80cffc2cffc62741235fa34f92cf" + name = "github.com/dsnet/compress" + packages = [ + ".", + "bzip2", + "bzip2/internal/sais", + "internal", + "internal/errors", + "internal/prefix", + ] + pruneopts = "UT" + revision = "da652975a8eea9fa0735aba8056747a751db0bd3" + version = "v0.0.1" + [[projects]] branch = "master" digest = "1:6f9339c912bbdda81302633ad7e99a28dfa5a639c864061f1929510a9a64aa74" @@ -168,6 +183,20 @@ revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d" version = "v1.4.0" +[[projects]] + digest = "1:e5cbd028e1c6f49057ceaa5012b504e7a311e1f49d411a38f2d4d02b718e423a" + name = "github.com/icedream/go-bsdiff" + packages = [ + ".", + "diff", + "internal", + "internal/native", + "patch", + ] + pruneopts = "UT" + revision = "a1d297ebf5e610377602c9b33a70d0dcee9cd4f6" + version = "v1.0.0" + [[projects]] branch = "master" digest = "1:6480de9b8abc75bfb06947e139aa07429dfed37f95a258e90865c4c84a9e188b" @@ -184,6 +213,14 @@ pruneopts = "UT" revision = "f16ca3b7b383d3f0373109cac19147de3e8ae2d1" +[[projects]] + digest = "1:7ad278b575635babef38e4ad4219500c299a58ea14b30eb21383d0efca00b369" + name = "github.com/kr/binarydist" + packages = ["."] + pruneopts = "UT" + revision = "88f551ae580780cc79d12ab4c218ba1ca346b83a" + version = "v0.1.0" + [[projects]] digest = "1:4701b2acabe16722ecb1e387d39741a29269386bfc4ba6283ecda362d289eff1" name = "github.com/malfunkt/iprange" @@ -331,8 +368,10 @@ "github.com/google/gopacket/pcapgo", "github.com/gorilla/mux", "github.com/gorilla/websocket", + "github.com/icedream/go-bsdiff", "github.com/inconshreveable/go-vhost", "github.com/jpillora/go-tld", + "github.com/kr/binarydist", "github.com/malfunkt/iprange", "github.com/mdlayher/dhcp6", "github.com/mdlayher/dhcp6/dhcp6opts", diff --git a/Gopkg.toml b/Gopkg.toml index b2f07827..fffc8500 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -73,3 +73,11 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "github.com/icedream/go-bsdiff" + version = "1.0.0" + +[[constraint]] + name = "github.com/kr/binarydist" + version = "0.1.0" diff --git a/core/banner.go b/core/banner.go index 3374ee60..67ec0563 100644 --- a/core/banner.go +++ b/core/banner.go @@ -2,7 +2,7 @@ package core const ( Name = "bettercap" - Version = "2.21.1" + Version = "2.22" Author = "Simone 'evilsocket' Margaritelli" Website = "https://bettercap.org/" ) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 6f7fc11d..3543a7f9 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "sync" "time" "github.com/bettercap/bettercap/session" @@ -26,6 +27,13 @@ type RestAPI struct { useWebsocket bool upgrader websocket.Upgrader quit chan bool + + recording bool + recTime int + replaying bool + recordFileName string + recordWait *sync.WaitGroup + record *Record } func NewRestAPI(s *session.Session) *RestAPI { @@ -39,8 +47,21 @@ func NewRestAPI(s *session.Session) *RestAPI { ReadBufferSize: 1024, WriteBufferSize: 1024, }, + recording: false, + recTime: 0, + replaying: false, + recordFileName: "", + recordWait: &sync.WaitGroup{}, + record: nil, } + mod.State.Store("recording", &mod.recording) + mod.State.Store("replaying", &mod.replaying) + mod.State.Store("rec_time", &mod.recTime) + mod.State.Store("rec_filename", &mod.recordFileName) + mod.State.Store("rec_frames", 0) + mod.State.Store("rec_cur_frame", 0) + mod.AddParam(session.NewStringParameter("api.rest.address", "127.0.0.1", session.IPv4Validator, @@ -93,6 +114,30 @@ func NewRestAPI(s *session.Session) *RestAPI { return mod.Stop() })) + mod.AddHandler(session.NewModuleHandler("api.rest.record off", "", + "Stop recording the session.", + func(args []string) error { + return mod.stopRecording() + })) + + mod.AddHandler(session.NewModuleHandler("api.rest.record FILENAME", `api\.rest\.record (.+)`, + "Start polling the rest API every second recording each sample as a session file that can be replayed.", + func(args []string) error { + return mod.startRecording(args[0]) + })) + + mod.AddHandler(session.NewModuleHandler("api.rest.replay off", "", + "Stop replaying the recorded session.", + func(args []string) error { + return mod.stopReplay() + })) + + mod.AddHandler(session.NewModuleHandler("api.rest.replay FILENAME", `api\.rest\.replay (.+)`, + "Start the rest API module in replay mode using FILENAME as the recorded session file.", + func(args []string) error { + return mod.startReplay(args[0]) + })) + return mod } @@ -205,7 +250,9 @@ func (mod *RestAPI) Configure() error { } func (mod *RestAPI) Start() error { - if err := mod.Configure(); err != nil { + if mod.replaying { + return fmt.Errorf("the api is currently in replay mode, run api.rest.replay off before starting it") + } else if err := mod.Configure(); err != nil { return err } @@ -229,6 +276,12 @@ func (mod *RestAPI) Start() error { } func (mod *RestAPI) Stop() error { + if mod.recording { + mod.stopRecording() + } else if mod.replaying { + mod.stopReplay() + } + return mod.SetRunning(false, func() { go func() { mod.quit <- true diff --git a/modules/api_rest/api_rest_controller.go b/modules/api_rest/api_rest_controller.go index 16738d89..72e7c43e 100644 --- a/modules/api_rest/api_rest_controller.go +++ b/modules/api_rest/api_rest_controller.go @@ -36,7 +36,7 @@ func (mod *RestAPI) setAuthFailed(w http.ResponseWriter, r *http.Request) { func (mod *RestAPI) toJSON(w http.ResponseWriter, o interface{}) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(o); err != nil { - mod.Error("error while encoding object to JSON: %v", err) + fmt.Printf("error while encoding object to JSON: %v\n", err) } } @@ -64,8 +64,68 @@ func (mod *RestAPI) checkAuth(r *http.Request) bool { return true } +func (mod *RestAPI) patchFrame(buf []byte) (frame map[string]interface{}, err error) { + // this is ugly but necessary: since we're replaying, the + // api.rest state object is filled with *old* values (the + // recorded ones), but the UI needs updated values at least + // of that in order to understand that a replay is going on + // and where we are at it. So we need to parse the record + // back into a session object and update only the api.rest.state + frame = make(map[string]interface{}) + + if err = json.Unmarshal(buf, &frame); err != nil { + return + } + + for _, i := range frame["modules"].([]interface{}) { + m := i.(map[string]interface{}) + if m["name"] == "api.rest" { + state := m["state"].(map[string]interface{}) + mod.State.Range(func(key interface{}, value interface{}) bool { + state[key.(string)] = value + return true + }) + break + } + } + + return +} + func (mod *RestAPI) showSession(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I) + if mod.replaying { + if !mod.record.Session.Over() { + from := mod.record.Session.CurFrame() - 1 + q := r.URL.Query() + vals := q["from"] + if len(vals) > 0 { + if n, err := strconv.Atoi(vals[0]); err == nil { + from = n + } + } + mod.record.Session.SetFrom(from) + + mod.Debug("replaying session %d of %d from %s", + mod.record.Session.CurFrame(), + mod.record.Session.Frames(), + mod.recordFileName) + + mod.State.Store("rec_frames", mod.record.Session.Frames()) + mod.State.Store("rec_cur_frame", mod.record.Session.CurFrame()) + + buf := mod.record.Session.Next() + if frame, err := mod.patchFrame(buf); err != nil { + mod.Error("%v", err) + } else { + mod.toJSON(w, frame) + return + } + } else { + mod.stopReplay() + } + } + + mod.toJSON(w, mod.Session) } func (mod *RestAPI) showBLE(w http.ResponseWriter, r *http.Request) { @@ -73,8 +133,8 @@ func (mod *RestAPI) showBLE(w http.ResponseWriter, r *http.Request) { mac := strings.ToLower(params["mac"]) if mac == "" { - mod.toJSON(w, session.I.BLE) - } else if dev, found := session.I.BLE.Get(mac); found { + mod.toJSON(w, mod.Session.BLE) + } else if dev, found := mod.Session.BLE.Get(mac); found { mod.toJSON(w, dev) } else { http.Error(w, "Not Found", 404) @@ -86,8 +146,8 @@ func (mod *RestAPI) showHID(w http.ResponseWriter, r *http.Request) { mac := strings.ToLower(params["mac"]) if mac == "" { - mod.toJSON(w, session.I.HID) - } else if dev, found := session.I.HID.Get(mac); found { + mod.toJSON(w, mod.Session.HID) + } else if dev, found := mod.Session.HID.Get(mac); found { mod.toJSON(w, dev) } else { http.Error(w, "Not Found", 404) @@ -95,19 +155,19 @@ func (mod *RestAPI) showHID(w http.ResponseWriter, r *http.Request) { } func (mod *RestAPI) showEnv(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I.Env) + mod.toJSON(w, mod.Session.Env) } func (mod *RestAPI) showGateway(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I.Gateway) + mod.toJSON(w, mod.Session.Gateway) } func (mod *RestAPI) showInterface(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I.Interface) + mod.toJSON(w, mod.Session.Interface) } func (mod *RestAPI) showModules(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I.Modules) + mod.toJSON(w, mod.Session.Modules) } func (mod *RestAPI) showLAN(w http.ResponseWriter, r *http.Request) { @@ -115,8 +175,8 @@ func (mod *RestAPI) showLAN(w http.ResponseWriter, r *http.Request) { mac := strings.ToLower(params["mac"]) if mac == "" { - mod.toJSON(w, session.I.Lan) - } else if host, found := session.I.Lan.Get(mac); found { + mod.toJSON(w, mod.Session.Lan) + } else if host, found := mod.Session.Lan.Get(mac); found { mod.toJSON(w, host) } else { http.Error(w, "Not Found", 404) @@ -124,15 +184,15 @@ func (mod *RestAPI) showLAN(w http.ResponseWriter, r *http.Request) { } func (mod *RestAPI) showOptions(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I.Options) + mod.toJSON(w, mod.Session.Options) } func (mod *RestAPI) showPackets(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I.Queue) + mod.toJSON(w, mod.Session.Queue) } func (mod *RestAPI) showStartedAt(w http.ResponseWriter, r *http.Request) { - mod.toJSON(w, session.I.StartedAt) + mod.toJSON(w, mod.Session.StartedAt) } func (mod *RestAPI) showWiFi(w http.ResponseWriter, r *http.Request) { @@ -140,10 +200,10 @@ func (mod *RestAPI) showWiFi(w http.ResponseWriter, r *http.Request) { mac := strings.ToLower(params["mac"]) if mac == "" { - mod.toJSON(w, session.I.WiFi) - } else if station, found := session.I.WiFi.Get(mac); found { + mod.toJSON(w, mod.Session.WiFi) + } else if station, found := mod.Session.WiFi.Get(mac); found { mod.toJSON(w, station) - } else if client, found := session.I.WiFi.GetClient(mac); found { + } else if client, found := mod.Session.WiFi.GetClient(mac); found { mod.toJSON(w, client) } else { http.Error(w, "Not Found", 404) @@ -170,42 +230,72 @@ func (mod *RestAPI) runSessionCommand(w http.ResponseWriter, r *http.Request) { mod.toJSON(w, APIResponse{Success: true}) } +func (mod *RestAPI) getEvents(limit int) []session.Event { + events := make([]session.Event, 0) + for _, e := range mod.Session.Events.Sorted() { + if mod.Session.EventsIgnoreList.Ignored(e) == false { + events = append(events, e) + } + } + + nevents := len(events) + nmax := nevents + n := nmax + + if limit > 0 && limit < nmax { + n = limit + } + + return events[nevents-n:] +} + func (mod *RestAPI) showEvents(w http.ResponseWriter, r *http.Request) { - var err error + q := r.URL.Query() + + if mod.replaying { + if !mod.record.Events.Over() { + from := mod.record.Events.CurFrame() - 1 + vals := q["from"] + if len(vals) > 0 { + if n, err := strconv.Atoi(vals[0]); err == nil { + from = n + } + } + mod.record.Events.SetFrom(from) + + mod.Debug("replaying events %d of %d from %s", + mod.record.Events.CurFrame(), + mod.record.Events.Frames(), + mod.recordFileName) + + buf := mod.record.Events.Next() + if _, err := w.Write(buf); err != nil { + mod.Error("%v", err) + } else { + return + } + } else { + mod.stopReplay() + } + } if mod.useWebsocket { mod.startStreamingEvents(w, r) } else { - events := make([]session.Event, 0) - for _, e := range session.I.Events.Sorted() { - if mod.Session.EventsIgnoreList.Ignored(e) == false { - events = append(events, e) - } - } - - nevents := len(events) - nmax := nevents - n := nmax - - q := r.URL.Query() vals := q["n"] + limit := 0 if len(vals) > 0 { - n, err = strconv.Atoi(q["n"][0]) - if err == nil { - if n > nmax { - n = nmax - } - } else { - n = nmax + if n, err := strconv.Atoi(q["n"][0]); err == nil { + limit = n } } - mod.toJSON(w, events[nevents-n:]) + mod.toJSON(w, mod.getEvents(limit)) } } func (mod *RestAPI) clearEvents(w http.ResponseWriter, r *http.Request) { - session.I.Events.Clear() + mod.Session.Events.Clear() } func (mod *RestAPI) corsRoute(w http.ResponseWriter, r *http.Request) { @@ -227,10 +317,10 @@ func (mod *RestAPI) sessionRoute(w http.ResponseWriter, r *http.Request) { return } - session.I.Lock() - defer session.I.Unlock() + mod.Session.Lock() + defer mod.Session.Unlock() - path := r.URL.String() + path := r.URL.Path switch { case path == "/api/session": mod.showSession(w, r) diff --git a/modules/api_rest/api_rest_record.go b/modules/api_rest/api_rest_record.go new file mode 100644 index 00000000..ab0be552 --- /dev/null +++ b/modules/api_rest/api_rest_record.go @@ -0,0 +1,107 @@ +package api_rest + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/evilsocket/islazy/fs" +) + +var ( + errNotRecording = errors.New("not recording") +) + +func (mod *RestAPI) errAlreadyRecording() error { + return fmt.Errorf("the module is already recording to %s", mod.recordFileName) +} + +func (mod *RestAPI) recordState() error { + mod.Session.Lock() + defer mod.Session.Unlock() + + session := new(bytes.Buffer) + encoder := json.NewEncoder(session) + + if err := encoder.Encode(mod.Session); err != nil { + return err + } + + events := new(bytes.Buffer) + encoder = json.NewEncoder(events) + + if err := encoder.Encode(mod.getEvents(0)); err != nil { + return err + } + + return mod.record.NewState(session.Bytes(), events.Bytes()) +} + +func (mod *RestAPI) recorder() { + mod.recTime = 0 + mod.recording = true + mod.replaying = false + mod.record = NewRecord(mod.recordFileName) + + mod.Info("started recording to %s ...", mod.recordFileName) + + mod.recordWait.Add(1) + defer mod.recordWait.Done() + + tick := time.NewTicker(1 * time.Second) + for range tick.C { + if !mod.recording { + break + } + + mod.recTime++ + + if err := mod.recordState(); err != nil { + mod.Error("error while recording: %s", err) + mod.recording = false + break + } + } + + mod.Info("stopped recording to %s ...", mod.recordFileName) +} + +func (mod *RestAPI) startRecording(filename string) (err error) { + if mod.recording { + return mod.errAlreadyRecording() + } else if mod.replaying { + return mod.errAlreadyReplaying() + } else if mod.recordFileName, err = fs.Expand(filename); err != nil { + return err + } + + // we need the api itself up and running + if !mod.Running() { + if err = mod.Start(); err != nil { + return err + } + } + + go mod.recorder() + + return nil +} + +func (mod *RestAPI) stopRecording() error { + if !mod.recording { + return errNotRecording + } + + mod.recording = false + + mod.recordWait.Wait() + + err := mod.record.Flush() + + mod.recordFileName = "" + mod.record = nil + + return err +} diff --git a/modules/api_rest/api_rest_replay.go b/modules/api_rest/api_rest_replay.go new file mode 100644 index 00000000..787bc406 --- /dev/null +++ b/modules/api_rest/api_rest_replay.go @@ -0,0 +1,63 @@ +package api_rest + +import ( + "errors" + "fmt" + "time" + + "github.com/evilsocket/islazy/fs" +) + +var ( + errNotReplaying = errors.New("not replaying") +) + +func (mod *RestAPI) errAlreadyReplaying() error { + return fmt.Errorf("the module is already replaying a session from %s", mod.recordFileName) +} + +func (mod *RestAPI) startReplay(filename string) (err error) { + if mod.replaying { + return mod.errAlreadyReplaying() + } else if mod.recording { + return mod.errAlreadyRecording() + } else if mod.recordFileName, err = fs.Expand(filename); err != nil { + return err + } + + mod.Info("loading %s ...", mod.recordFileName) + + start := time.Now() + if mod.record, err = LoadRecord(mod.recordFileName); err != nil { + return err + } + loadedIn := time.Since(start) + + // we need the api itself up and running + if !mod.Running() { + if err := mod.Start(); err != nil { + return err + } + } + + mod.replaying = true + mod.recording = false + + mod.Info("loaded %d frames in %s, started replaying ...", mod.record.Session.Frames(), loadedIn) + + return nil +} + +func (mod *RestAPI) stopReplay() error { + if !mod.replaying { + return errNotReplaying + } + + mod.replaying = false + + mod.Info("stopped replaying from %s ...", mod.recordFileName) + + mod.recordFileName = "" + + return nil +} diff --git a/modules/api_rest/record.go b/modules/api_rest/record.go new file mode 100644 index 00000000..1cc734a4 --- /dev/null +++ b/modules/api_rest/record.go @@ -0,0 +1,233 @@ +package api_rest + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "sync" + + "github.com/evilsocket/islazy/fs" + "github.com/kr/binarydist" +) + +type patch []byte +type frame []byte + +type RecordEntry struct { + sync.Mutex + + Data []byte `json:"data"` + Cur []byte `json:"-"` + States []patch `json:"states"` + NumStates int `json:"-"` + CurState int `json:"-"` + + frames []frame +} + +func NewRecordEntry() *RecordEntry { + return &RecordEntry{ + Data: nil, + Cur: nil, + States: make([]patch, 0), + NumStates: 0, + CurState: 0, + frames: nil, + } +} + +func (e *RecordEntry) AddState(state []byte) error { + e.Lock() + defer e.Unlock() + + // set reference state + if e.Data == nil { + e.Data = state + } else { + // create a patch + oldReader := bytes.NewReader(e.Cur) + newReader := bytes.NewReader(state) + writer := new(bytes.Buffer) + + if err := binarydist.Diff(oldReader, newReader, writer); err != nil { + return err + } + + e.States = append(e.States, patch(writer.Bytes())) + e.NumStates++ + e.CurState = 0 + } + e.Cur = state + + return nil +} + +func (e *RecordEntry) Reset() { + e.Lock() + defer e.Unlock() + e.Cur = e.Data + e.NumStates = len(e.States) + e.CurState = 0 +} + +func (e *RecordEntry) Compile() error { + e.Lock() + defer e.Unlock() + + // reset the state + e.Cur = e.Data + e.NumStates = len(e.States) + e.CurState = 0 + e.frames = make([]frame, e.NumStates+1) + + // first is the master frame + e.frames[0] = frame(e.Data) + // precompute frames so they can be accessed by index + for i := 0; i < e.NumStates; i++ { + patch := e.States[i] + oldReader := bytes.NewReader(e.Cur) + patchReader := bytes.NewReader(patch) + newWriter := new(bytes.Buffer) + + if err := binarydist.Patch(oldReader, newWriter, patchReader); err != nil { + return err + } + + e.Cur = newWriter.Bytes() + e.frames[i+1] = e.Cur + } + + return nil +} + +func (e *RecordEntry) Frames() int { + e.Lock() + defer e.Unlock() + // master + sub states + return e.NumStates + 1 +} + +func (e *RecordEntry) CurFrame() int { + e.Lock() + defer e.Unlock() + return e.CurState + 1 +} + +func (e *RecordEntry) SetFrom(from int) { + e.Lock() + defer e.Unlock() + e.CurState = from +} + +func (e *RecordEntry) Over() bool { + e.Lock() + defer e.Unlock() + return e.CurState > e.NumStates +} + +func (e *RecordEntry) Next() []byte { + e.Lock() + defer e.Unlock() + cur := e.CurState + e.CurState++ + return e.frames[cur] +} + +// the Record object represents a recorded session +type Record struct { + sync.Mutex + + fileName string `json:"-"` + Session *RecordEntry `json:"session"` + Events *RecordEntry `json:"events"` +} + +func NewRecord(fileName string) *Record { + return &Record{ + fileName: fileName, + Session: NewRecordEntry(), + Events: NewRecordEntry(), + } +} + +func LoadRecord(fileName string) (*Record, error) { + if !fs.Exists(fileName) { + return nil, fmt.Errorf("%s does not exist", fileName) + } + + compressed, err := ioutil.ReadFile(fileName) + if err != nil { + return nil, fmt.Errorf("error while reading %s: %s", fileName, err) + } + + decompress, err := gzip.NewReader(bytes.NewReader(compressed)) + if err != nil { + return nil, fmt.Errorf("error while reading gzip file %s: %s", fileName, err) + } + defer decompress.Close() + + raw, err := ioutil.ReadAll(decompress) + if err != nil { + return nil, fmt.Errorf("error while decompressing %s: %s", fileName, err) + } + + rec := &Record{} + + decoder := json.NewDecoder(bytes.NewReader(raw)) + if err = decoder.Decode(rec); err != nil { + return nil, fmt.Errorf("error while parsing %s: %s", fileName, err) + } + + rec.fileName = fileName + + // reset state and precompute frames + if err = rec.Session.Compile(); err != nil { + return nil, err + } else if err = rec.Events.Compile(); err != nil { + return nil, err + } + + return rec, nil +} + +func (r *Record) NewState(session []byte, events []byte) error { + if err := r.Session.AddState(session); err != nil { + return err + } else if err := r.Events.AddState(events); err != nil { + return err + } + return r.Flush() +} + +func (r *Record) save() error { + buf := new(bytes.Buffer) + encoder := json.NewEncoder(buf) + + if err := encoder.Encode(r); err != nil { + return err + } + + data := buf.Bytes() + + compressed := new(bytes.Buffer) + compress := gzip.NewWriter(compressed) + + if _, err := compress.Write(data); err != nil { + return err + } else if err = compress.Flush(); err != nil { + return err + } else if err = compress.Close(); err != nil { + return err + } + + return ioutil.WriteFile(r.fileName, compressed.Bytes(), os.ModePerm) +} + +func (r *Record) Flush() error { + r.Lock() + defer r.Unlock() + return r.save() +} diff --git a/session.record b/session.record new file mode 100755 index 0000000000000000000000000000000000000000..55ef2d96a70032224db34b2cad6fed778928d2a6 GIT binary patch literal 65748 zcmYhCc{~$-{KqRvpZYLKZ@9Gn?clSFT)B|IYbVyJdF%09x|c7Bla&Cv$%m2fuYY z{Pu^J+qQ7^J66 ze=G*7aUu@_-0GJ`<2oi~MVS}rgB2eS(2k}g4+0?5_6h8oPpW>iXy@9T-+K3+z^KGr zU-z?IG@%wVZ;_l)qi8hJwI*jt#?E4&FJ(Sor(BmyG5^s?)Lf2llVp^akMd%Vth-CH z%sbB^)ymjfINtGbLb%Dve7tYhPi&>A4CNSHUg6;|+wh44#Vvz3fIf4$d)KzPWiz zHV4xRCy&3}cJU8YBV1~ex941$e(_Dj6Tj!$l_;q~m1&m=7Y;k4Ii)$sMPndcoTFVu zyOMj}UypJ^ECnj34_;f@@sF`ZfnvptG9EuQ5+;$e>tkJg<$uY~NC=bS*O84IPVnT1 zVexO+6@6{zm8OgbN7Bqz9xOZlCM|NU=Q{%R*8shJd6N-f?PVuy#9egajl{g(tU!{Y?Ziddtb?p<;a)G%MgY0upBGWXEXy16=(x2oC;bPIbnNQWuy#6WAq%|pa}~KGR(sQVZMijY zigXxQ!PWoX$y4*`y^hl#)q8@doJ;REuZzR>dk3f=_!!Hy)-Dkql@!15Y1%x8bw{-I z9>Fel2(m6qHA=25{LLSEu30OLR-pI@qs`|>0+5_?s^XlNFoNcPH#|i{Cs=C$eum;=Ce%q>P>J$dKX&MC%~rhJq|16+Y-$Nu9o%?*bXxn&pBVduJk> z;|rJy;lC`5_ZMY$oxF|fx7cfi_eAZEB)20}I`WSW7VRss;%^UkkZ58GXde;y_Eszh zr}HLkhZDbBDy|e#&jAS#F(WkyrlJGYy$Df~3!+c2rld|+6|Lb5O;ggk5YR9*a zFvwY#n9FIJb~#YQLBPFR`N_z|VVWR_wfSKx2|iKBdZ0z>ZBtWAQ0Bm^cA z<2kRtKex7D-;N%+iw7@pS0i#|&94DjPU8G5r|@tRAdaNPC`66VDUVR*(q93c%fJX< zWx3c?3{qa(^Hmr1I!OU6#Y}%Yx84FZUQ=O9bacOZ!Uf!v@PA4<*_&LshWslbdy1ubR|K9XpcFLG=eWy7vvbI)Bir%$QjJ^dn$Xxx~l(2sr$Qqd}!V_Onok; zJ!0i%Ggc`wYf<%d?m=FD=pgj;T3a>0I&dOp-7rtq#VD}CtKrQtvwg|npG=!=YtP?G znRJXP^1HUrdGOvhV@u@_AD$6SMgFr?7l^x#>%JmwIb*C@7$f2MH(ALy2ZS&#LE)6T27?fHf0P} zXSL4ljxdW09v$`9wedF+nUwnZ`h)$?C2m*9O-WHfi`lGwD6ypP{Fgkl`>b5VxRm&B zg4_#}?^P#9DXo^#gnd=R^OI>+zfVVACCrG!!E4=2-8@%>vg35tXie+WcoyWBxOwRv zFTxA~IOAIPMJ{bCW*(gIQvHG3p)<=@RcJ|Wn$w*_y^pc2;LAYnYwfS^zOcRcZc~Dd zBXtLKej{dI`t}LkP_AvsZ1B(Ar|5}UZeuJkr?1)XdW4+EvY(T<}1AnJ;I&wT2Y2? zHdfQ_*0$xoWR*qKTMx69hcbViZF$hVBr8n(MZ_P1eAId$Uo(Fq(^l)k#Xx((lhXav z4tMo~MhHalz?!?zyu{2;#TV3Ds)J^cwG=I*V_Cm3xcP1(TVyY#6e#`E0F!&5WB>xpIc zRok=;llVPcVXm~tjMp(p2F<#*5W-s>$#ARhc5%&GG`vy0?Ykpxd_e6mbEqKsCCp$o z*DxT)V?ouqb1|FTku~>Hru_Vo7cW+R@Gm$Pl`mZ zJO=2*`aXlzFw-Z|0gHfLs?Ut{yPWyB)e1dZv}1j*+lUI_WfEk9wO-``SHcLKwGt~# z?wa1P#|F>}dsR=(4A{*8B#t8m_hv;{&qo64)h;~pF)}?~nuPndk_>Upp+8u(QQH*A zPxd5j)UY@BzIaXQ0A*DPxyPpAhY{oL`h({_{BgmV`uw3S7$+!y_7v zyWMU|`|V=>vmwpUiLaa7?Uz9N(ri+kv5|5fxgq_qs(8^GeBXPKxWuqL z&Gh|jHo|SH;)z-N-?wSa^|8^mmbrTYXvyX1oQ^{?`=(Q~V#B*Y;eDoypkw=#wD5;G zc!wp`|M*vPXZxh)Pi2}phWydqPn|TS&|2JkidpSmkmz*oLY&kS6{y2*$ly^k+CHt5 z;huYFF_(r>x<*2Ly&&?ohSkQzC=nObupdkg{#j~s;p7D6l{+g~E~-n_!r6awKEFV^+(Q+^{@bnoDwf;^A^hzH%{312=M)c)^C6Cf z`;&!(kBo(9Uu0FFJX6vnC%AYIRaudz33#Dn9Zsp)eoBavaW{JpKr6mDBJ+^;ecH6Z*6>9LO=H9!SMu-6391v_sCeZB{F{G*z^9)LN0bt&VZ|hwx z;08zLL9XZ#4yHe*Jv$#}{u?Ds>@SlSFgIwAP$GA}8PPf32Y$^HXQ>euc%s`j7c@%9 zulLwIl8Ryhz@gh_iPiRwP*h~Qbt`n5)R=s)IQflT|AQOw173KX6v3;dh~wq9>Ilbo zc(oaibyTPJY`gnHApD8P7Y*IheZCK01lfXuQ!UN;%R4f5-#YUN{pxikjo_v@DXY9` z#@z2W(>E0X{V(73zqBH%MhF!jSbnX^2}U2ku413L3=4A%@Ve>)v>!h`18CdKb0$Z# zY6a{rHfrAg+26D@2vJVsRjUcvP}W(wlp{&W_I`KGYlOWFz@CA=2y;q}D3Whmc1a=wf3u%|Z-SAg*jN9t#{7wNGdh3;a zmP9Q_1v2EngzlE`OFvw=9o^TfMyac)>dtb?qgH1}tWVRE3Y&u|oUYohwBa9!OdydGNIdo5f7@a)jGGWqE^ zCRyb0WJ`8ximpfdn!9#3?)FEVs$dl!yicZ%d*Tu^MnL#{Zy~fC#dqY9= z$<1q5;dC|f(L|2Ej|rnVCmivhzMKAr^(PB@h<`kTPgEazbk;>#>>QA{@7QZ$c%;tV zJ!=I^NBn_pmK#Zjdd@@;^B;fQIP_fUAuC&kMe~a3gxWAN@c+3AwdM?`C?azW_0f#; zNc>>u!Uo6@5h35%UgNzYo2%I2Vi)`am+eW#PMOF(^jJ(2wFHHMo+^2 zHukqnc=olddG@fy-oZ%RVbl_QiIk}kO|+&vIcORf~=z1Q%=VL7}#C0d1rmTfLhQDvL3Qpx!8X+JX zk9jWt(bO|l)*&GiyAlJa6Ic_zGgcq}mJ?sZcwo%Al(xDYyYYV8B1r95P~@c}Q>yRn z%$?)#$PwUw1$h|#4*CVExeJ4I0FMU8;1Q0`pS_>O6BcO+R2vpe#z4cMBH+L zzYbvfy!Ebl$J4TxKRM&S70Xq9&-WgZ!Xza!-pxhkJG_8vzjRB^S@~5EVsULVsiXPD zuLMB=V_6#lk&|PE5J`T1!-S4jIUc( z=c3$t&Uvl-&K(j2p7FLyl876bjmnQF-)!St0|5kzonDgHPX2ecdOPazJFNlfc*O`w0!kn;s%N! z9}S$o1(&;>uUkoK4zsA-g4q1W#Q{ydNjtecb>%GLa~ zO!p6gIpcQ@7XH5f;6&>;Av%9Rs}-`ecM4L*=toyynKS!!&0B84zLpB{zVf)0t?*9% z3QEBj%pjK{j+OPs%H9nTnL<4mos4e3l~5rx@aIQPR8)8GxTx^Yub!0 zIk*aOx)5YiX|^<_-oI&AhQD!K<9C`E^D>->a8R2+=H4*>G_nfW(>D%$*Y`e?!pq>9f z_XTdw6HPNJO8vMcD;&SgYm4MiOb?;I^D>Cj|82D@B&65luiEMscC=e71V2_K=bx)7 zp;bt?L?vx;PXjP3j4JnbBi$vgAFa(2(}+ltj(zT})($vg80Gg+VUP_J#CBbMmeuB! z4ca4AJaeG1Q`4nFoGp6NrTxp`FkdrUf=rs`j$CFb#oZ5A_~V$O~59A)BUyfF;k?QXM?rIN>pp0JKhd-vf#try<I!Ca*px1=IR@HTHEG=TZd6% zwoRtw?x6J0U%8Q8)gJXyD8AD1n>=V}=yi^}YG zf))tP2FRlw8f|-@-~!bXOH{9<3yfL&t^Pd@_7M&!PHW?;x&;h`5qR2& z)|B~03toA~T-xR;81;Z7i2gsjDMz`nm1~mScHwRXghr??^cDFwlFY-Au?=Ij38`u{ zT!tq@rsN%YoqNoTYSdp9*JY~J-Es4pR(>ZD{3c2hX47a=RYu=DIlrgAe>=aNxT0|c zS#|M0dAUGXk8eq?k`)S&7;scOx_ADx;=eGSMD~;)&$q1?Y1O7kIO?|=ew2%*T(`|X zP^<$mgxJ-gK-^jhnjg4U4bQI<*@l9H9RyVQvIZTNU;!=8Z){EJvAwETdH)`d`0L8k zy5yQcrI9UDktNNFfqALMf0*D^dJSp03-n@G*`?52+yhiz0C&IZv6MRtAk@P&gX#V@ z3zB?V9aJ^et*UK0ta(NS)$^tJi%;xu%>c5AoU7MM)M-4X^ml~ymZC$Qy~RT6^pD0J z+@)re^pM>lYM@g+`7Mw7_s>U3^!R7eB2Z#Bg(sWx>?=O5_ZBuWn5mwmFmmsarcSE* zD>Ic#`%Ez-Sk54Zp95Q`3JZxge(WOqy#Yu6#&@5LQcm5$SPfzY3c+w)-98OC= zoV^z#gTD-N+&Su^?zHR< zibJ`nS2qw@lcfbzQZ_!D=j}?{x>=}luUd^(2Y#v_Lu}v;F;So6$;m#~Y(zcV>`f&N zB}LI!gm50rcKNcWX3v40H$@!GKhH$K=RSXMkk(J=JsI{q3vcmjTz!7Z^}ZdDMgEwx zY%5CF|19_*0>d*Jlp!MfBmE(lvO&749oj9cB{|SGQN+jW`>kJ1a!1tB!%a8qWx53# zUA1529XC0hDxV461OCIle}!tT;z&tQf@7izl7FSgdcO{99kwaauMy^ByqndG=G=biN(;jF|vc6BZ>o^G(TJw zExvzyn|Yr}diSy6d#-0Tp<9}&@wHt2`XRTHJSW;gUjV!1?$KW|Rr-E6OWY`b3w7Oi zMonmQ+$%L&%ET#A;H^Nsn=;Jyv&=IBnJ%~8EN{-)6z%hU=OkNn7iSWWA-z13Zr!wL zMStnFVS$6o=Hu}G;G4I=h0%+>#^@5}DmRq$$WNZ!DTs)k^2yGiTD*q+_F&T1(;w!S zwT^92U zt=WBhuZuvYamkGJ9(C=^rteKuW&M?CmPT+_C)n+^PcG;wS_-q#^M1BzEATi^&I)-tPzAEXW+_rK2yO zeL#W778a~7{apStC=B)j>P?Iz8UH(C~4Q4@08;RxE9cGmgzHYgjT=(=Jm z@W%26_Lu5`!ujgH@MxYv_XtMXQotW(k0ieXh0<_J_LGjJvo7RRHySJM*B@0qmu_={~n@~*zXkY`-S}|2!`po&|e_IpPVQ{QuV{U*iu9!qPMQB z$RWPSoumg~H(s-I{HNtsn~@TXZM95%Fuwc*yAg}4inrZTu@UK<{SfJpI++1G3kKf+DJHUFPlp zHxbHG^ZSPUeHpH97ATmU6Zw>kmpN8Z>>U2CAVqSgD#}qa2X6SuKlG0$@Toh$LRa9g zt?Os$e9la&FC^~}&@R4V$nlmnMd|{~ML?+J&f(L(BcIDN`qsA3-@-NI)4t%F4H+;O zR1esj%ai=R)MKPkU!9EC6fd8LoGGVNAEC0^<1Z8R2&S;1)0Q02P9xV@(FjOnm^s!b zU)g2qZMTVw?ej~ALiNN8#%AyGxY;I2X-6N&xx%VyPR7kQF4CI2^DdxQ{gy1tUuY=} z<&fiiLC~EW(7VCw-mfHG>>1@paslzQ#}+$R3Zld+{g_=H|Fi$?mAY(DvHbeg3+{mr z_3k;!43PKV{CgF}&pyvts82yu{FBw}4$*D#IpBbZW&X-2F!(jj{k8^vek^Zq*0;|M zePURk*cEbmCy5~dsp^pWHX-NjWJIP?g5ykV#k}$HB!5}lMg`=~o#Y%p#x-4&*XG;I z#R=kstgeEJgMzx81hQo%DtTQD*8aB5KSmD7-t<0hVIy9`Wv66E&!_Uzz)}6cQB%AY zm=O1-z|3Es?boLTMTIkK^}=B8I_YzyW#U;8NP}8_+KBdU@-1mBKlrKnuj~Xs`67+D z{kD*zCT18{tx{ZLLFWx+jOoTKkMw#={rsmZm_%FhJyuk z8CGrmY)N!0>jnSoO}j42#VRTDtvi$bYtU*;n^1E}6IVWa&FC zANzZ-;*7KnVxLZEPtJR$BSZ6MWX(eF)#3Ao&}qa$)z#SyubrHvzWwAmpM;H^RDJ=M z)dfHO4m|k|HdQPz3_1OS7}nuyu=;pe4E&9SL&$zi9ySv%Ix};9m5wpg;1s82*TaL+ ztO6u+&n4}y2#N%ng^2MU+(D}Wl9q)S`-Nzm<)~K@SC}|<(H9ap4nylFV0jNIFS6`c zI8Kekp*bm@{Z+;_PcQqPyo!d+%`?E-8`Cu!&MR!+cm~%<+Cbh?Zo)=AefV36G$V}* z)7#O;;)l;;Uia-Iv*BqT_JislEAQQ|6dEdifYRr`-rg43dj+%&MX6rVj=>he24IhC z=KsEIi^wT@U_+cjYjRfSR|H(6?@2@6`w^h^hiseFP2}l+F_VThDM$|PP2imkrz*wv zkIy&KfUw@pEVsSSRo8)5^}1CQFSaJnFU9fy_zozQd??kL$hbIEmJK+5_J!H0_oKq^ zqP?LGviX?J>X)t3rmNccPdf-3$kF7oe~`!S+lTTFjPc*12e9k+{YF|5Z6wo@w|wm1 z9hvYmhF4j2K4%G4PcW{lMnkXFK2?v@La?8|x1C@@+DOj)FUmspHGf7Y%ZCs!{D@fIU_GQ`3z0BZ?b- z+%tsMh{#v{fMhuy&|cE#TMZdo5X!Rt`EB`MPk!RbNof!uKmSe^eJ^Vin1 zRhs6$E{PGZxZY!b^;ePL4%#8P&eL_0J&YFvth&K+DRnR8C~#djh~rr3xU|(!>x*c% z23V4{A+0u5#ZIv!v;1t(%3IjypQ8Eh{KKfCu-h+X%SDRLTbAQLff$WM{OL0RrlY|! zq>&lqta|(THHU1Z6BzAdg5#DTW@O~mL5=sig`zvS@4X4p_!HZ78AG`kVmXhji0?RH z4(?a=2_1GcV>o9E8JpX#9xnCb_e`2EK{o#;YCb9uy>p!12{Bk>gp{ZF@m>V5$$7_q z%~QXt8;_fQ#;+xY1(XQzsdvVGM_c(&1Wh#J)wE;P+>Z&4XO4@#roM@KIDXKOK_j)} z1})+nJK0NyCymB+;)#gZdD-r7#>2IF)I|yY#hPtFx>G+G4~FN2`5-l9GR@5~UOc$T zIkh1mY>qwPflQpB%Mi&$r^N&jW@A0gbv zEl%c3^0SVMM?ZudF65B3~kH`4|=Gbj-7EDL3luaMvkq6c@uJFx}YtOx31Dfs{=^fvkT{-t=yB{tdU z-Um;sJ0=`X7g+7uRgLM8S1dP}rjx7Y`hu5FHk2mMcepz6tuL^8B;Je_m& zM4fBi(yO#Tvf&t*G0FWd()rt`lK`;V6p#2Ik5#{vTgYcY;CTn%Yz!_dgO(|6KBmSm6!Q0FWKeh4j#mLNh zI=%)$nYFS?2^GZU{<`Q2A;(AhFmVzg|5+rD{;nH}#Jc0MAZ|U^cQD8IsKs|zWSd&D zu$@fsabjK&?Y{4DP`)}u020#pt8x5`Lq(p2z{U!w%LmB~uEw+OoayURB*_^I%QNkD zP~%z8;eizufO&h zs|I==f>!C5C;~s#)E4>p*nBHJBX5ZBk0x@3E~_P04ahs*cJz2C2<-nbk)tg0a+cNi zo6Dy(oAy@H*9u1f`beGsSyh^hc%d=?V1_UCPRhy7)% zxG6Xi&3}ok#l6_=%+@p!QWm`Qs6S;cu}+X5<)bZ%0>+XgxtVfVc?9gKhLx*#iLrS3jewXWVm3)zF#5wz#j&s#OOJ3vmeph^D z9j+RkE&RqE*WMw%FQrS_JbjKaNlI`=%(k72KURSR{x$-L7>i*&1FO3My1gf zyav4YPo+Prv#B$*rRQ5KBkk#!O`f@Fan|oPr5YWH3xsMoL^!jhO(IXjbRvrLm7iVs zmU{RnnEGtV(t0!Op^D^^_BfSo)+|*eUz0E&EsnF|t4u3cdF;det<7ZJ!WDjYGtG0Q z@GuXUG(b_L|pb%$Nxhp|4e z6d3HcxBzO7%IRe?_meUm*VFJ(BiVv^InXVkAyNt#g4sefsv~hBCUNty7?VA8-b;_M`g$*}x4DJ^5)Kxxiv*sDJI%_T0`x~qmR5^pbcY=BZ#x+M8 zYx^C+nB*hBFZTMXR-Uis%H4mrR<;I}4bNt!&{u_v7Db09uWyyOC&TV0>WfI1l*AiC zE}%Uj)^}FFkZ0Sr8|RgEQ6bC9k;auCUjUjALHR8I%@S6pPJ63v14i|+Q*!lGPke=A zs%%>V)5?qNhXOo#$GxII0Eg(Ut@_5B2RI?Z9QH}>tsJj##j?X5#^24`%5y<|a>Qm0 zM+)5SRHUDJ(eW2FNqQQYUe=MMvz34mH-1F}AdehwUhoinsIsbXUje@K31fcj=fLU> zZ|ba{c%f==98Hm#(DgpvfF;5il}+CW9ir=?x~tceNpzlBx_8|!)?L`Wv7~k@<^%ev zLg>yCH@KF85Y$K4TDrA7mmsWsp6`BmpYoKtI$ZB+2rO-1<*&*J zrjV*>r26JK)z1hjI7}IX%_hhfgD*?@T+d>Be@Anh$zt~9ETf0h&U?PuB ze$H*+DY~_2tw?f>8P$$oR(NkqqZM*GQSRGCXoxiZJ}GU%~8boF~SamE0Cl zB}ox_LsENfkr=Peo0Rd#VeY}&S;u7D65JEa=>Y)>9918b3c9s0y+>TliC0vkhFG}| zFmHd!C0cd7uORf4Q{>m!P_P~Cv(??gn7XHw zk1-@p&4s;-RVd=jXCt1}eTqh-ydLWgc*rfm@tdPrx4!##cOIbYtgP%0F8@~zuYd(& z(}WMcz=7-@{wIkJDj|?F_rO{ssDt-6LY{z0&L> zJ6gJWl31Dq=^2*4p^y7b|1Y?(?1zUqRk&itv2#v-eVkV7)gABMRw8(ZvEn48r|O@i zrtMSo+D+Yj)j;hYDMP^V^(iy?sv_a=0e-Glr1{cBol6+q zIc{^Z(mPh4G6n?#%EDc(va;l_?*olce)e~OF~uy4=<`SkOR}3uvUB4!yDye{3Fhxd z6o)lk0q^xfpPZ-omB&gP7={?C1|fY~M!7W}iV(Jw)^4*RVl@MDSLW;L;C~vO{WY^; zK44@ygwfN_<$EvZua)yssI#KmfeqrK?Qb{9a?W7?g9TC0X>&!G^vepI#K|7^&vDp4 z*Ny0;D(UV8f;svI1(03bWLkmm*LWLJo+O)_&lLPwK6s`)-U>3DeO91s!$kNA?y#wD zcMdCeX0xRlgxYY3y|k!OUR1?#8~R43{Uzd6spX*20{hVO?_aTSTEgy~pvsnA{ zi)WHx9JtR4oug=_`d<3vofvb5`67M=3~pGQAT%`E&@f`B8lx~}$wMiR!}8hrtD9x| z+;pc!R$$cAk>i&FKhJ?SdVY_O+a;McEZ8@~7d(PtdBXxxS3KI?3V$QV4{o>^GoXxJ zI8mUD2F0&6?fpT4+nM;IQZ|7Wo!Po1%IH7ub(Xl&uej33*@GQ7R#B-9|9$Bm5rsVe zP75j2eKWmMrFoVaR=;a~_Su7lHbxb)my+;HcqEa+ohh?6Vr#+jAy zN~W5OHBNIjWRpA3s{h+M?~%JnGdd64OC8)frO1Eet25>Qb7}$VpW-R@7^0){_{j z=i)WX*MCXPha0$e^Ch>z-hoz$2K;ikB8glxuWLmh=^lP?;!hg0Mx)lQo_^MHGeFcM zXA`R{##fqj4GysR|q*Mm6_9S3f#3H)ciJp z&UCf*(N;h5IB{=Rnx~#YB)eOiJ5b193n_)CsHG4qgBKc_xLzB1OUWfS&q9Bjy~dTM z@#=K@BiWes7isDX4I^eK^4 z?77z6xA=~FE^t%Gbp+4$?2dpspk`Sz`D4*uF6XiGEPFc3YM_Tus6*f5z~QM`YQ8WR zV+b9`#N zjZ}fCd|u7zt>C%(*R3X>Fb5!}c?+q+=R=Nr1CwpK4b-KT&HG4P4=;iQNCK&+2z_|I zy2gs0mq83hcscO4K*0wp@lf5MgW7GwSHgI5?x?e^-Z@kqLMRS;)^9B^LkhmNUKs4y zC_B=aXyw7X{NvaAIj2wqH>4=4wX2`0_yS1ufcMSD&91xlm0xFm4bf>-9jC1}5ujN{1;2Wvq79f39faTiau z@AK!n!LM?Em(sG}4>@_yD)H){2m$b1b^AE*!g0O|K43)=M0*6;frTJ68_%jesfkn6 z+p9QK57c~L><5Ih0Zgn`+Fr;}{c%k! z^H%F*5B1BtrP&Bp^2|Ga5Hqh~oqs5S&uWvEecg}9P55=!kfs zcKhu=pEF^*0M}BWaYa%)zXdD^3_jnm#Q#cHJuxB&Znd+=3-pJ7%oIvw)k1$hdsioh z46ebeDZ12&@6N9whZ~j&q^Zf}?s;3NN~>FkL9(NCp(>9@H`9q%<9-O zb^^UF0}gI4*M91O400mm#&oqj4JBKyaK$v9o(iA>66m8yar|#y_T4rwrPCztz|C@Z z{gD^@S}1Hr2{k}d$hrJhCDX^gg^h&0Nk1ZvrFXUzO}k$9V#hjierS`jE6CA03~|q^9EnrjZLJxTTpM&H1^D*a<@IKJxa_L| z^Y&&qy(z?wy#qe*GH-|9!*?tUHS%4Zckts}&4_D%qd#};1_sgh#8|K*RdxpIF?x1T z#44kAp?t4c&CwX{fvsO8<3Qb~ta}_$ULp!tlUeU|mxkHdJXi}4{$J<#*>doSihCb- z2b_p}xcm+EOMKjLPH4MG(Hb1WRln&|`%5@3h;=8JSz23zrt3BGsp_CLPV*aglq$I3 zDT51Wvh~0c@J}#qkg+~k`>pFziKcFapcWj(#sBrD9b{ymici;oQlAhQ{UzCtd``Y~ zpXb|F$5l#n&=|w%oq3aQIcc9Dvs>+cr+w^=sW~@4R8`J=#A1C4gv^_}Set#Wljc>H zuCyQOrPysWzuso-PZnmxg>ZRBqr{ET|Ax7dyg z&krXf+x{k;8XWQXLPQ$g#t33tG(4pE?bj>AE~X61-U@lhIS_yJ!ZgkqGG-lN9a1=I zy48&xOMDB-KK!!ivp_G=0nwJMdi$ytDAHgPU$m%HwW!42A==NaUP$4tSI%Hg=m>r1;!OkCW*wXTSs zoiiX!9EEOSChngd+xNBh6xZA%LeL9Ok?d6FGZr5QN1k$}Y>!NV@}2|-K+9B|{R=+k z%QGp}B=aTmtz_dA9})o%)#f4&Tf|d= zPga5HEqtur}`^s4{utRWzM}3i+xbamowhfTd~2|ez3CmR zqM7O7Y9BMp>gF|+KCZg}YNYAV@csuY2sAe_U!v0({&%OoN?uhrkk74qE+?{;2lFPw z82hQS$%9j6(Z9UvX-q`QaUl6 zxVQ@ZKTEL-9?S)U*bP5Jplfw$%BQ{tU;aC;^a$~dS)SjPwN0HVsysYI%O=D(S0;{} zl8?E~e^Yr2nxmTzaN$M!{yeOa45qjtu7agj;oSh8r0kuKBN}$ zpXz8lpD90RZN3PSF?s)snb(VJX13 zPWl1W6@|$WKl8`wT5|fzJ=CSr6nCn}d*>pve3~j>ml9+2I#t*E_X_RizllLslO#Y7Dw{x?>Fv6!Hhjr|n>~CF#UYWk}F!gb{YCUoO)szVgKWuy) zxbBL7p*+0LJC8~q-0;$I^9%`ak5T8m#wFG5M!OS4AFn-!zW4px1nvFm#~Z|{ps{XA zj@^hYp-w5mpqDX{KMIwClT670B(2%R!<#0rsBIi3EqAN`$>d;pDV}E?e(q0wYfRO= zW7jZEgs!tniv(u70p>54@!~-}l4vjtU)%&`yE$7l&@ZKG;CKGspt{@m{K;rW%XRt< zwkArw?xrG*VFunDo^QHkrCk;6u7)5!r-s$Cew}F5I!GjAKG#+Vw_X9N0SsXGq8{!L(Ez>XteJEStVqV-NDh z!O!YQD2z_Np|9c}pGKMJJ>e~+Q2lagyKdhL3iJSQblcTDhqFv|4ydE%IE!_0ZuQ}}rVzHz=vBu`|m{~nudU4BXj`ZyWm~l7CjYWC6u7kv<-Zcb8PI{p-Ghn?7_gA!R}~rF#ecb% zLbw|jjk!v^NXs$_t1TXgp7=TtebxIn-)_wn{y!~W|C->$xsie1SG+^TYOD&+<^$9C z9uCdV2RKXtl=D16=OBdmwCSTGGxBNMrfxNAxG|V!iOuiW-Nvf$J{GmnA8Rd@?QigB zxncQ9jV`#X#r4Ng?Ut?AcH@B}S-~9{Q5m0iltv9l>4j&71G+1kXT_=kq)T?!38ph! z5!eF*3sLLDpOpCUzn5PA8V&M5eb_em) zzQ8?XW&gj`_=>{<3*itBmkq^*r25$Ppi{i#*@{`h6z1YSpxV1>-ghgx=7EBBF1D&Z z7MCe!bT4_N7$xlPl%^i4nw!30u)=Mok2aBbCRn75mz=)4onF89qWw`dq_vV7>`|SM zS0Mm&YF$}fPanT$f~-CXnPJ1ua9f!|oH*4F^c>os0lke8)gMJU2XKi?vMp3C8;xFo24IB&m}%u?|?0SudKXN=^H zPWKLyMJ8&$q@WJ%vJ(@g*Iew^5LUtnPFj>c0!``jIrrMmDJCmF)-9=l@hM}I47>`| z|H1km`a|co?KQO)P(2@#IQ4A`XJb^AA@`7ap!h>1!y+e9Y6MK&g9#8iDX&;sVc!E^|2UNg*SiZGjet|-GI}HI`U)kBrrajR zw)eECVJD$w_&try?3GKuV*Db_fAWus?|`%!OBFKAcOA=ub5)pmlmSq<@rmVmqQ`=u zTa-=xx7+n7OkM;kDf6Iv5gi zS_nh$8vX~MqZq!vHNHI@^l6lm%f<9>Lve?8opNcQV*jjIIyp8EhE8~iH3OXoY`X(# z`JNHGA7aC`|LcDsttIQlLAXy@m7nU9vJUTi{5qCV@tM+4b;G=dGq{NmmRI_#f9HPy zk3ew0@K17QJI=s=^(yX(Gn>gv-?K2Q?My@MT;d$TJ06@v$Jn!qu|V@J#-g?HJp(tf zaej2y$coM-TJ~5zNAD6qJ#GV!rxjBNV%ub%xxJ6tYGrS(y^zy}Lr*?NnOamBI)F{; zT+oczB;bEvhhlty{4~ICl?mPWNF(EVRQIDFV~szJMKY%CwmT1|<-V``UHM=JvFb~S zv)-mZM;Sg;@!wL1(9TkA{V`Hte$~64?{_k8=Y%wd&DdI3&=bQ9asZiMh_r@jGUyY#`207mCA(J_%D?qt zy!Bma7rR!f4~@ci6%-Yzlu57s3V z%mbzWb6!{o4w2nyBoeubOFgwfp2{Q7DsxMZk>(t(wHNzGb=mBk>74KN`b%}oL^o~D zt;#iTUtdb2YtH(Y_Cx48mq<`^d~bT;$ui{<`bU zeMhtX_LY3xXZ;4sa|(Qn@Xkt~uVH`CaRn~R0VFi?92|-EBfb9X`M7M3NOQpN^Jrh( z-t!dC0$Z5Xt!BYYUL%+2D#>lm^;h=AcjFZ>9~Tcv@o#Ip*!qDl z>k??{FXWjAnj80=7BX-HH;XFX$WzOO(@DIhMRHGHHM?(PLmJK z*+(N=B4-Zy4y#o@T%TR<%Gw#|>%#ngp}T-QtP{Y@p9=b+psKwADD$uyHwqOI8 zQ^H%-AM5&!{wDCId3oj`t;Z946)Ex-E#+k>+8ceKA1s!KZ_VCLx@M#_hlt|A)$H9z zF_^-dKd9|nExw#{$o6@??xB7C>M`CgW53mO+)K;4Tc0A-Epe>?(F<5;; zf6s776JiU;E)Tgbp{OTFbGWJw+;g~)*svOg+||$Tjp0%LhgzQd9)*my1#;upFyFC- zt2iC#x^nq!9TTs_J6a+a9VupQnWF$c!Y|;|{=V#thBg_t&4>m;5%&@IH3A za^9D~jiEPSB-&(bIngU1FL`L}AIxWOuKL|^$|YBrU+iUnUE_ziF75Yx1?xf^CQ;_( zv3pLVrHntBZb0ip&^83@P>NZ>_=@6$d}sW<_TCO`m&QCp4exI-_B+=1ieJN?p>-Lh z%zlhf^KEhG+qDc`^$-&oB0@gd!!^CT_iHs-^>>x$Nr?eUe*@du#()9mL0wuNHELu;^G(UFYu&MM548=FkjmB ztL8oTxG-@2r)Iz$f3ms$)Jk95`A5TNm7lA5{si0`=|?0uhj?^ma|EoBi!YwZytWNl z4SF`ejI!RZF5B<+p66JG&^wRVV>?~5kMUr5zQOGy`g05dH=AN(A>(cl&wU4iP9e<%#HK<|L|J`SNJW5ZY2z>+E=)!%D zL^*@l92i(Le=tj#uh9xuHe-;?~Oe`$Oyucj+j2zfgU zA1(C-i`H3aJz~RLASS4tu`fc-{_=Cy(5D;tnPq(St8|+$)Ik^B2%nenJ6h922K4A#(cry;gejTwD#@F*XVmT z=eyRgF}1bHd2_=P)5_;W?9OOp9ODeH5!EkfU2ui%U-QqEJ}AvO#I#l&BjuD3TON6a z7(nhy?@-}Zjx4f2R@C2B^$JKFa-R~Xp?1p0N92qC%3xfVIWql9Iz7xZwT8A>ei#c+)vYSKlV%&w6ZfzKuUG!}EB=ZGV#eIyy(5vf19&O;<4~(K`Bt ztSorTu$7M7Ru+pP>)aKOp?NeEBLumXEMV`wt6?Vl3qw7lHcS@Sn&_%*%?Lk2f2+T9 z4I6c|w|x76<}A*lB+v`NpYhpr#%rd+NfROy=EL338UEqB=iN{B9DOW1_J5{!)qfeZ zCq1##)L&>;=Bqh-h$(3X_K8f&P3l7vjdfKs%W$2kJ=@D;w<8w#ioB(`kzzD5OWHof zx^sAWt`q$r>lL9}VJj>{(V=`UU;(*rHF@0M#9v%niYGwMo_j2~YJI?NiX%^T{&Qa!cW;Z_?+3sJtkvk%@=6+KH1)xM zXr+!ct=nMaoWwK_qpf#y>wabDQZ3hDXzA01G_e<_j&+K+c2@J76525QQ>aPPRMuN> z4QpTs^jtESJbFo|0GeS!dpbT$_Df|`5nb-$&2}4FJHq_}nN^$;Z?>mBMP|G02iRIp z8j*eTN*-uoR5TzuVVFC+aCDm+8Xvkn`{icnD3 zzMw)bG?Qg%4ZC7RF$@2}oNw_--mlQxjWdYrSkpxpb_$It`ZL*^b0?ZBF_R15ixT9) z*?S;5))l`4;wr=*OUzMnnhd6*uB1d4ID`d!ho1YQjD5nluCpF1a)wsh`#;Nie6H5h zy~$t4P?2z~B^$y^ps z?`ussS!2gm`>d9&FZ}~^CG%*^^)*aO$gNxXp1eb}_j3q%9YoJ~F%Os6!lCAHQPG2( zCf57+5_ns}PZx6#5xee)h%JtH%Lw>EJ%McUEN+}T7wF9d{lr+mEAnr!`M}qnMjGh# zmspWl-<)27l&TNcp4$6&_g?k*LH_{Qjo^y}o8^MMY0`=fH>%>-_xD^ep^}=vuH{{o z&2YuAqq&9f|Jb!p&@&{TCf3|6$8@bp0Dro=zhd`PI8RLGHRn;Qhv1^jNyAx%Tb>a9 zt?MfH_cE*=n#WkJOPTk7gC%s(G2n-lItq(BGWjrq{L&con6d}wOS#~^iU-F6^&5r- zzXz=GWsmvSe(XaRA84%#)rlejzmvUet$cy-aVP*^hx*bieH4|gBsVDF)Rezv1r_8^mvA!x_Y{+lJ%qb>N%YzRA^q)K>ET``nYP};n-{^IhXX^vl zO>{j4`i?Q>ox~S<+b~PFakgN*tgk3;9P=9qj#P`sTa*{7ea>jnV>}}LLf$$t=laDO zEClP4NVR!wOFcQr&Dy%sy+d)#s#?>sX0n>w*e%Dsz0}^xhqOmK*lL06H~NZ2qO~J? zIPm&p|D`m}O!k=7`prl(WP2rxG;q2}O7>7Q$3qWnCvc)*&0J|du>ZlN|k^FbdV--24E z?7a1%uSLA2z5wGy^q)}ZWsoZX^)be2oYUW0WXvt!A8*%&@*36GN`GGjwXwu*>G6@M zA8Ft8(=ig#G|CpoMz3Lk?Xi)0_wIXnB>jihzggSQ404X%tbyJ*l}2Y zpZqzKBORY3b6{TFi&=yY(Y*;j^b?)toZ?vNC;EU(JjIB-LH}a%yA!#uq&v!;Ly40g z3X)0m$`56od=}MvcA*f@*b?D0crP}v@7fCa?L%4r(WD{$zMu8h zpEF`t+4ytV%RZEt<>#BnKyy;0mNf>^R#%2SUE$7pgD-1;MwS?O_9{@)lHRv+9P~g> zG2)Mz{737Wl0O>o3|WKgdE}gku5tg(+8;_wB{5h>7UN| z@4oiDRqaiP%gp&m&)HhmbrQm>BdhQ)d{p)TpDceIYv(7}d}FdNmc82f40#=gAh)8j z=V#el*Drddom1AE~ z8RYCIJV5R-iuE7md&fEThXSvvU2}1|)~C52?*82J|Lq@nN%ZC5vg>>gvGWm;y_Nhr zA4AcOz?ORSZP@#gQtnN*=L+P>JKweNhthvCcKDN*^+jVLCHuOzwqJX`JWH9)-t!cL zLte^_h5DIdNBUB}M43sh)86Qp&9Qb~?n0r*K^@Ypw+meTdM3E|K=dghLvPot{C4TX zFpAS}jlJ@LP`StCOTWkZKMt7OF=34Vck$*Ca8y0{|1Q21V=*qIEXo{+r^S#vE6KV1rzBeVyOqvw-S059Y5K1#;D2lU5yoyU|Oo|NjYo9B%1({9p7T*vD9q026f# zG05I27T%8NT4%h6oKsO2!{wdNs<7t1cfhzULe9Y!ruiW&p{H*s@^No=4gMYZ>nJ8 zm&N#*(*Hw47`Z1x8_a6gTi1;G-L7k2lXX1#j_<~B&Oj|o*4rYbpG>QNKc*Gzzl@G^ zoDmw8G9OQU*ZvVW@`mh%pBkQqw7)&|Q9s{{jMkvfFzoZMjD=5qp?86?452U4P9O5A zU&wzh3fb2O!;@Eh;eEM$vC1<_1$_BAVE?ar|Usa^c@?do$hyuC-~GEv+(_<(iasa9O`W_fJ&y-U6KY#yL{$e!-;(z8_Qzqod^>*3N}g78pV$Km_00pmtfsD^mX~7iNDz0UvQYG0i7THCUC4X5 z#GZkxx-B*iI)5UYoVUQEII>iJvi;|+6QV<~95s+ZNRcC=bI|jS=#l*m^OwH)E`JKO z#e3~MB=Q_Tl`Tz~U&$98VpY3cq}kbcWNpSHIr~)3?*Y?AI(aK|{$K9?tvgZW`5(t0 z(Es8M7-zzzpdffiMCEiRkO&IuY>)8h|EsIJp*g8ZX7Ba+eJ`?RVzBA1uCA`btEyg* z?h18Kc9ZkUB0Mkm)+<{IiQGE)s&-u2QWl{952r6bLIm>iK#_95T?IJS0ltnJaUVs1Q?GFQG`UDY`@%nQv4@jlX@L+=~(_Jcf z)4eB%q%IO*@=~Tg^9$3@6Z~*U*T(TU^c$m2$HTHMuyDOStPjKqEC00N zH1md2&WE11d7dR3Lj${)sVQ1KpU1c1SyE#kKZn*^o-Y#{MspIByyr8{uf%n@U978Q z1Otk`dv$I7_JH&JWPZ5N0yi}jo}nZ~Td zGt3KOI2!yEaU_a^aCt8wIY_E_UV#3yK4?RT>G%PY1NJ`8eKYrM3y2|gGhe+O0y*@x z=;mwDmW+41xcrm@Ul5uv#&-AtU{8B{O#AKnecrpirBZjy;V>@)n)@ zKV*{TRvC+~v<_WC@)4{C874xx6 zKC@CB6VX*W&9F_7$1-d0sO%TK2l01|Xgw?WJYkjl6_t*0% zzWL)++>H^>#n%XIk@&_T-Do7Ub2L<>KP}*HV`90=H5$YBdg1%Lcbd$51w9b|!`%UV zB;j-16MzKQh zT<~2X%dku&K6DP<8I%5T9muqP>SVnTbij9wlT-X*Pxd+Aq}#KnnrF9T?B7y}8>Pqf zs3H@`>Fat%Y9S&zn%Lh+)(;q$FuzapwIR8IeLZ>XQ_*9WGhci;EY42HfV0E;8E~wd z83*Zl@*4X&w}o#bP^bG5dB>=-PtVbCR`)Z~cPcjy>|8knUW9wI-}|kp%RF{a=nzvc zP9r+2pVVQ=`~5-AJ>5Ta)*AVT9b0A3y?J(rQ+s6_ z`;oj|JA%l2>I{7L&Rja9J3<>y51uU@!KWe6e1tIe0hh+mS9y4@>Sr@z&t{kNxEthR z&L9@I##oIf=xPaOkYg&CJ;VG`k}=PZSEU{0eO9Nfy{n?96f!tx-r$O3b~E@ zEiw9QA?sw^JDJ{LP9h#%1$4oGh@7D@u`R39qu^RWL#$$yZ8FS6pt`c^N^((%dL2qm0M_QM*;wK>3-#Q(bXW;I%6nvk~ zk0V*&BVKF83idn9#M8urx4@qeR?V{M>+ z@E%G_=z*3f9VGKU$}v2~ZyIOKdtRAWozJ5W+Wb*f?a_HF`>l;oVVIzr)@k|U90&mUbBa@%Q%%JB!j$UwuW3URqqB656F4v0R71tzmNH%jdlf& zqkJYB^_}|N`45&$6`89rJYSE!pff`3O8w%g#vJOZrS&~rV*hAO-hzL}o}9-K!K^my zuRS!tFKei{_q1zuH=ql)?h7NElsi0(c&FHd1&!Ii&n3y<>EXvbYUmjX%%*cKu0~_Aia>lr(&&4)5wnTItP)zRZWT-K+EwdHLu?K7ywiPd+gC=4M zG3}(%F93SkqrF3N580gYd2H#a*9kJ{4Kmc@Ac^CkN$w7|%1Djz9J>W0C z5Q9!jZ8DOJzX#td($f>DL5pln(N=Dtt5V&g$sUFC7vFqa)(iht{p^{fuKaqIW3Bi} zUL%asr-8m0mpqMw0bs>qKMH+qThMwz>^E2s6N{sEU85YlbM~6%P{qJi<=j#*(*Wl@ zG{!I5^*dT~pmsjtT;_Q4runB~$f~+jpK6@ir3wn8yCjjs2*L=R6EL zK`M3vsQ;#HTuZ-r!V%-MD9&Zq7(b!;VeGs~tfnpRMz734x_FTujO8NN)=jjNuZ9O- zSYMd`11$)a@uxYNZ8WdM4c6J){2DoexOL~+FDuo-53t;t3&WgMeErhNmbKqac@4`b z%u2X@B)^7c>P4kk0r>ynOr%IR)s-b@E&(gJ<7i%E5?I|0!P&{9A@$Sc{(Ua6^>Zl( zkQpa(?lLqsnfvbtK0#uKKFqdJaOiv^ zn`5`KUe3N6ULo#K?e9SCT_V3tsMSQ?l_*zVz2*Liac6xBIzeX)b(gqPU?bhQ3wT8r z^s0LP1&$%$zPpT1G4%yg^*Z{gA^!vWf-J}@N8f>-7~uPb&u(Dnd=9T@eg6Y_+>}Ew zltOeyksSjyAHX*abwR4%&v}ZO1LV|bincGy&n?_#T8^KV)Y)>X%(mkDJeBgC1Tyzu zFt>rWC%+%SCj&OmG<%BztXB=Ry{{@6+J7wP;q${=j(;oGMDH&@CAv}CC3mo%K^~hL zb0h2@Nr{i-P+1qmP)FE@d7wVbq6gH3c}H&Z{npg&KcloIw?-xRWh`R((&vgXAbwHu zPyHH4`%B4hT-!HT7pW^`<~jx)(-uGgY=49SzUyx4XB_@_H@ihWEB-5kUnZ~6t{@&fUTiFK5$z}gv#~x1> z&f~ie$ClSE<3s;2TL=z;BBDLNFme6NwRHjMmPKypgFM*BYoK>YesB8e55~E3mlore zaS0FdM_b$hVVb8tL1#Ov-?Od|a|EC83HWP}E9Ti%%>?PGZvRO4 zipGkzT0LH#RpaF!7P5cDPVG6N{2a}LtUZ^Mp95RH{@&YLHtDqGv^MAtO?Ujo+k*bQ zFX<1tUxuM63}A(k0KS9mcVz-Y!83 zG344jiC>XCi*1O%lWU6o9phNxp=Md16E!&id9kvl@wr*%8+>gZl8`6Da!q8Vze?@7BQ^Ey)wV}N?$ z`@GIy^}N*Q?zr#h4An8%7~Lw5(G`8gavcEhnR%S^FiMa0eZq6!lR12nHH*aril1pov-}M%~ zUWo2rmj~FszO1Zn;-zBhT5k5<0h{dL2ul(1<=94Rgd78?-_%6=ihT`w6C)QyY-i}f zr-^GyeYgIA+mW(yLi;)7H3qH~`)ubAC*}GMYLOcZw!tn4gTy3y39=2iOCZGmtmfiy z$#EBhm*&e<94_MN!sbHB1lg^`ju2jue4ClGe`Z|pw&W#yx#4bzCyURFh1utBg&i}E zHCf!cwsQ%FM%Of_@ip)$ zQW-3k%d>~EnvCwnKG#KeOf9GGi~44n7M*G4$uMr)I}fhVg&Mx4JU_-CQW|ep;0alq z2H6%)B3@Q0@Y`_|b*rDXy+5~2z%NP0#rnNb_-N$)x%ydw?+vqHINh(TL+GHtK|QBa z;YWeH$qgOBj66a72k+nY&p4@m*0SbAME7cjY!Y|Qu9@}#Yx)l0T(`yZ@SPlA+%)gp zpI)Nv{@2g;KaW#nzzYNmjP%p--yB;w#pwU85EOM%9{SXN$lF4heyiPt!Si=BY8Q+X z;{ts)cN5A7+Xg<^miVB-w^#JTF2)93uZABk&8s2{{-*75?>#kqF#pzfuI1cTw9eN* z+Q5h%;Rd-oL87E9ONk(VM&4ga7~tVF^Ar(e)caiWFoqNB6@d^!uT#jk>2vFk4p+8Y z@^}`5)y?Ba;>HuYoP+b1(b#9#_Segfk*=)nZdBM7exO_}NyOZ}` zoW%V^bhHt#tG5XB4FT!QJyBCm!5iQ5{j%>ea$^zfC%a=@Od-5@syTP9ApRKk_MV_8 zcE=6L`U^>Ir2psn0|TYkB?NGF8Y(and?l$$?> z7?F0sdRYSoMb-;ibU+@Rfd*=QE^LjKF^|T-p2mWI)D#?tqVI(MNy;@EnawmrdzSl- zan3!Wneu^aC;D34h3}*9Hmn_NFZ6dE-lfs^A6)+4TCdS(%Y2^LQ+s9IwvM2r5F>|N z(y)PUh*#U!+_aYWxzsQY>mwBLKF%?Q&BnF~q4o6y^oruY%u^xWzXp zi#VV)tUorr)7zX6KC<3%x>3aZGm|^Kr!R+58b9QRAJHaiq|NyuZFYHF1k1R@+!=Ow z9bta@z~6^lg*pw9Jp+F?N;Fzo!QHiLJHB~76+aAGH~0_u@;y7kbJ-*N7}>yg&NcRJ zGjlI&Rra&O*Ujt!#mauJRJ0!@JC>21#1G!e^Dqy%bC|JHA|Ko54tyIYTIq}tU013TND0+Qjp4GcF&*R3qgME(3 zi`lJFu9Vh9&)w8xXjX|GQ9eHq8TiWF!`JBk9)6AAD)PB(Bgz$B;hA+jF}Fv1%J5fy zPr$p_+zhpTPR6p-x<^mYEOl*z_mh`3^ZxsCtWDlO{9ArE6>ICa%&(PRRUdRAA80QS zn-8ap0r*ws6vKvf?)JNyIgeOR5}*e%+6QN`ylKe3EMoc)E8@Hk)QZA=#5Z?z;J7{? zY^xd<+&8I&MdG3rhB3bvI>xC~9DrKZiJZ$Lr|Z zM{+<*a9#|{?=-Gl`mD$uY?sTWm@|84tqgHgIYkD}RSHNRG5-S+<7SH07~4_THguzw}{<7NMz9l)1i zb*904M&uy2OD-1w@@t&EZtVH~@ScX=TlD%rjpxw(4&$^ozpv1e_Y=TjV0{nKrN6(7 zvF(9u6NbmId@6m`0sq3cWk%U{s-3~JV@p?WxZUHr+gGOFtC8K4Z+G<4ugexeIeMxd zI>M1X2V2KB!UsBQSvJq}nrsbnWqpUIs`u3Z%bf_@H3nxc@xBv2z`mtl<5Vo{&OFBP za>n;c?;m`>Y8N~<+Q;zI?@W*2H^{^$n-$hPCcehI^e^jlxA6{oJM3G(^8=axP`(fK z?ma(6jlNiyD||&|JF0d@S^0tg|GF>Pk|sxc_57O1Tz72pzbEolnAm*&nLXeBkG1ti z^JD%U+dkgLeJA^zZx7{tj(5+TcXlt3+<{o3TEr;E3B;eyE312UZCx`5{3N$O8g!0F z`ZOC){NBA`rru9%w91+Zzt`k%&E{g{aRWaPeN@{{Ozf9=diQ z8eckmbnfY%M*8tk&tcBP4#)hLf_ZeUDd%-`$y2Y`mzNBVfH$1qCy!o!GPbDInMKp)cC?*s=#t1=J5yU&5D>~MbRjykV(q-Ql-8VmhUZDWZbXx<`@Y;_ zKkZvof0sjh|2{dl;`x1v=6Ma$FVbwuP3Ln{!uRxM&(7iO2}V5LO;acB{jVycD!xtj z*}p`$SY2+FsnNctnV(G7?^XF+k*DmlZ&Bg9)HcVpv(sb8I{r*9H2Tm$uP3z(*B2Yt zG536)``=rP^=5K6dp6dWNzboBcYaUB!!v%hijIKsn&rJgyi|^3+?!8sW0ZAyG#~1F z)%(hGzK{F&9k~-q@x1-_=KZ=a?2zMO9PPhl8~t6`1oy%Q`jO)EAhn*cBurh)Df4n! z7Ks|8;{pDs!MBy`&uY!a`}Ucv$_ryF&KK=xGNk%&(^?AI#kJc5Ra?6J8R75m%D&p3 zEM~l}FBPLzKKmVgQ<2TAtouK+FY~24pDlUiD{{OfM}KB#e^a(T=YxEtm)*CwxwRuO zesLk*d5M_1^Z0jboU+|U&s^Kkq2Bb)4ZV^cwZ5Y_Q9o+UK744Slh{-k42{qcPM_9* zpUlBWlw(To+OA`{Crmd^tL%HIgUb5W6Q(@&K|_x@1^TuPZU?_(#9m^i#Qq#CI6lwk zSx}5Hu?rj5f7ecD?LcMg4scJng&Jo8vcGH%>%Cj!eX-2@#Xz4R*UDps&(iP?ecJiw z4$U&xx+Qn#jq8WqndkAY-^E8ZKy-IMARqD`{TqGK-|&Ncjkf_uvI*=d=L!60f&b#g zqP|MeC!vr5_b@J}-@agGP@@FdUPE4l_qU9P&Nt3I-Ink=D=RpA9X+P;svM8j=hha_ z9g94fs{c60y8))9oEN~vaG2+-SR#Vo`^Kh_n;(YavJN$ zz*;uGe_E;8=n36Vj2~N8-aqTSWs_^YW~7Mb(XGy5+VB0IWI<&fi1uVImE4s{fA)KV z^HSJ(Up#fzH1E5)F3oi*X`WIUV^{b}yCsYr>qGSu{Nnw)<(@l^5RM`1dex8~obj@8|I0#9ow?2BCcd_KU3 zju3|aZbJIcDe{an6k{7`^W|Z@81QckUNV;PbZMHdZ7K1}vGk3rnu!aspwrQ_x)*<_ zq<+WXgI)0QVIK&O=%l`p6a|)>Q@myB@Vxrm+S!?j3;HhLHZ$mJZR=7XhuDWGZ_Bk2 zi+#akwB;)gjZ2RG7&5poQX7KBSheBiih5)Or&QFvWU)^zSU-xnBUmSyFX}%G+we@+ zXr!Dza)0`~8+jVP+5a&GQ^Ml$=x#zew|T{%)grI}gsevu$6uBl9b? zZ^dkWlLxsn4}O6jwygab)7<80U81=?XYrKVlWiZb<(|ptBoEp-%w2Yh1ayrS{fzAILD^QP>dH4R29!_fP(p=ecK18L5X+G|IQPY>| zl*QkQT-cVppXra*^cLNHw`UU`Z=q`?R|waU(B$6#m>wozCo}XCwHEqR^lMzlv?5ph zQjOhO4R7@BUw7ZFoPzs&hiud7KY4B(-K+lVT|20SK9Q%2n-%T#_T9be?;qJP9E@Gkz@4D{h@q!@7<61j4w*WAgW!ecfOiwAk1=t^xers<^WSHw;weIAZKc&>Kp0IF<6&Y5hG@6)&2~ zhu&`cF!yn!2A6==k#CVJ#X7K#IGV#<3w~#$Uy z9@*+na^q=j(b9VOynY;8ukj6P09f5iUGI|#mF0)q;_LjR+alnklL$*_#6O_*79D9 z`sPD7#AD*Mk6PB##;YrzDU*=Xqw7|2*wM*xS2;$kF<%Rk~V29FQ>&q&L8?vaf}E zwl~SWTpp$-1?hKT*K$J+_ME{CuzSJeg3Uz_hQtCBl~_QF=Ke-M3(J>h>(=2v0T!Fw}~x&Jl%?J$arSM+^$ zj8A#{4VxGIK8I0CpFJ=E=ZseExwKjKw$b=`8Jy#+avK2Go7AzM4E`e4-(9j(L`_A! zHH|C00eN&(mxp>vfE8AbzF-1RF?}dy<=j3F?FMd6Y7(>7BvH^ z_sf9ZFClZkG}&^jEDEc#2-ghgg-*xJ0#!WOOzjFwP#WdYf5MZC{dttVZpZ1i){o}j zyGM$(x5$_V`c6H^Oiu7)BNS*i*p2wkNr{jE7poiTF_AA?nJvGh( z8Ep54`1iI%dp19X$;AO5cJcd(ez(LgbZ=O{NJBk+M%DqgfvLYSY330IcM9_Z{qW%1 ziWrO{e4l>~uZ2I0;xbC|N2Bs5s^mem&4raiQB@An^^|g`%W;W2V^2LN=HQ&Td=U>| zXwl)Aa$_exzdtUF6RQwI`KiIeV`T?^!lsu4C3U_Ac z^-rb0Q4?QPVl^`otI@W)Ji|Y5UZqkWftK@%4guT~ zi=Qg@2%vXLQ5(P9Tc5y{>3 z1sWub3}{~eI~WrOGMV9Lac>0tYI1n{n>d5Yfa4K)~YUo7Yc_wZ_8IeT~V(u?<0 z*FIwGjRm7aw92Tu?Nqv>89fBfc&R2@dG24fV~b9|JgtpxW2v8U8MEFNb>QB}mVpn1=QG4c(`Wn5y1wiq4VD>a&df$i#RItD6Pr z&Y}J-)!b?-IUi6jP&#;#v{-Uiu|(>MKNIHGNOuPe>p4o9Gl1j5yx=Qh&f=|EhDry{s{woaxafH|$N?1)4O* zi2D2WQV1_WDxEyR;5~J@#xIO$ba z^c*SIUZt_5CVnX5%_N^<{S0mjYWx$t`9E#+D&C7|wkQ1&V2!+&7_p1wsAXWC?Mr8b z%ZfN2QiEv@lu19 zVY~iI{&{~2;s#HB|BJXrw$FZvXBk-n{%vofaQAth^;yH53p`Rzlm4KtqZ7Zfb|>~af;kjD zEys(&fyCBp^lDg(U+d-7_EY6S<_ux~)9EnEl;8nx(Rnob7@Pc^wLZdacho)l=P}UA z+8^NzjJdr~ zSN3F`=Z8l<(i1$-?JCKb8uq)vzc;<(-2&Ha)2ISOcflD|cZ7C--ApvAp-$hT?ncX$aZW#*=OTGsFZt;N_p*}`$So%QpvJPaL31PL?cycqd?8z) zypF1m^?^^cihH5DRDR#!z55bP&VENauIw4>tNl0jlIGT}$I%Om3EFrVOf8NuHK8M%F9@H&54DAh=-@D+?F8IHg zQyvEa96(+p#ZRS^V;M2QZE!0!&M3|)5aOdJOIvh@y6ST6%%#Y&r}1<%%k5oPmi9&8 zMG5H-=zcl_UTP33vnAlC$X-FuO#BWU|G_=x)cB0_W5?DvE7D(it{BUuGGTj4X5$JL z?CW&OZ8cNf6TXaOxX|O9!M5>yIITrNU3X0;}wpLBz_{E zsP4`96QC~fOt>AOKS2E*K)b5G3Uz^!IChny4q5qo-D=$MA;!2hH>(L-jK`$?Z!vDF z{5R}(XvDuW^;?x|G{Q0cur-cW!DijEi9CNamzS-zEH7 zJCnp0Th?*Q^Mms+NA{BnJ)_Y%u8!0Cy(Oa;V&98cjxjR(T*w5bR!rHSR2qAhh+Fb( z-gC66<{R#f&&PopUYP#Onz2*{P2*Sj#U2{C&R+Tm!^gx*qtaayV#})2}Qq$3Z9#!8nh@~OdTfT4{r8#fBieMUrDgMp1yi@Pbmbb}l ztMunBvD#ecop%?G5XK(WjW_$txcJU(yk&F{`dLhULzUjt@_?TC^?Tae+p_k*V(o+T z67lnH%U~4>+}5{fPUJ08|xTF-GR*>=k63a+USl%wsZ}jjPX@^=wbXi?URl1Sj@Vx z53qiJ{dhlF-z3Qa_EjZyEIzX`sqXv7YpceM$Qq{J#C$4la$hHE$9+60JDOv4<^8%J z|9cL{-X32-_W;tlKjxIDo8(L*o(1a&O>xk_be5oY+-`Kow^ zeqTc=^b_Ue^T(?_ugBK0{D-|HcC%?Nw{hPS_^T&$k*=^W&ZrVRie*2{J@~tFq`0rk z`}(Y0V1F0?_Wgz4`8V!7(9Qqm8U0H#4(l6`=OufB(H-5zZj@-h@q}*X3AY$9Wobyy z#G$$e_#M;DN;OZAM~Z87QS8?}JHak;$V&Qd+4n?!J;TY`L-*kv`y_C0DT(;SkS%^) z`viCC3*Cd8fZqf6K8n6=e?Y#TlPuTJ^IMJM95eo%`E`toFH0f%KdlAqrxNFNg3wQl zjM^&>^~dD%S{6R90_%)>#654jH0A7-W#>=*z(Rb4n$+d|K&TV<|51Aeuj8(~mwa1& zDcupYcI|yRa;mMlj_WGr?9|WhoRM85N0W?W&j<0+Sl;v{-;5(D*&4!h?-HBEmNn=( zz}P%>dQ$&0#FtCHj4=02^k3I-$FlYjv3-~vb|&8ZbsDzFuDkd5b9?h+yXHy0socwj z-2rhw?2DqdPDvjFJ?@LTSZ+@|t&8RJsp(?BNd9qA7iz(+A*R{nKPCE!*5@ADNnh0e zpP)0VkLB{Luu&SYT?NL1!7CPe+UE=`8=>wrjq7p0Ix-Zo%pQL>ANlaX^m0 zVlpJwUC5u3j{t0bP582pAlGM~_Yvf_R*mJo4g zqnzHgS`=j+3CB~4ycBj!`Sqd=l&uSW0I-dLblepaPZ|xUS5FWV^toeow_`y}HC0Sk zR&ekVopG8IKKrwYP)?;jbm?T1PwBS)y4Bz9hM(#e)*{s3%2XEQyXO( zI~Uh#tU*|fsU1MRL5;;m-%ZS=%#n?v^M`D`!26!bKu$w)w$RF5p*nM3(l-t z=A}~TsJYF8hs^B2Lr00F9)*fREk-S|g$D;Y^wfvk+ zdR&m_YW2yUiNUAekH6lYa=%t+Je)mqP)+b<|RVls#V^ilLmY_78+jiZ!`F**}xwEz! z#;Mu?hItV#U>va%H7>|yD>vy#HLNV{g9O2Q-SH%lpMrv{q3ZZAEo z`}L?E^>0np8hNQgmoL|GZlqJ`F6n*y?sDRWyW|1yJcD2R`upzE?0dXa#tydC605u~ zY^$*v-9OhY`<~ks%$NiNruUV$+PzV!PGYIAj^2i6yB}L;@o3DwkI#Sg9@DG$SmS-h z>VLKKMg1`BzA<&@sQejM^<%khZF=md-Gyr#1plG!5e*lrNbA~A`alm_KP&J$D+_vY zZgZPOZt!i(B`gk{JxuG~|1|L1l>For<6G3(ZK#FE$l7ciPdUdnc(BF#lk24AS<)}I zi3WArS7mSUt~eMFe=d?Ii5>?-BleSN=rcW~XwRF5c~hGgYX1a7ov+$l6yJ$!dwipo zJfo*T@1Jkrm06tgGjSp6UkKFK8vde{N0{$9iMerX6P>RP=wSusIxN}OGK+_7#-BFs zb3+Wg$xn;*kG$4>o^QwKfq!sb<8)u(C1>kV^x9O9afWqp<_=}g!>9Lkw*Fql*DY>cOXuBh$s&AKTx+%le7WFOd@^A+ z!ga<9+e^*mYgVnz+Y(zLx-B`EjBk+j8G3@lDJAXWDNM z$LQ;K)dO4Li*pT`nt?UFx6Vt=wm}8UK=LFK=;56;h#r5F)wI zct_Rs5PH7b^@%1uqy<~f>OC}1j?;z|iv|82$dOq#=^GEQb%1Sx^{q*MS$t-2(QCxx zfv{Y!gzhCsiSKBL<}`ulOZV|(pA3^<^hh@-*|+8Xkuu%y9)n@Y^R{dC;ahrCKGXYj z{q;VzwL7XLn+j~klXh(7dyH+BW7iwc7vxPCoIjkj(TJ=KwuVTIulRQqC*AYf% zo;(k^oQ1wxVE=!{(>|3G=>6>YmV@tJSvvTslz1J;J#DNfTJ{gHvS%O`be>lGWUzNc zb6#ELypQHZ&8vHNHk+UFZ_7R-5qTtYz9FkBw3JL=4N}CjNRHC^BRNQNc5C??4`L}_ z&NXd~i}JN}?&<3nw+o$v7YUy)Y>TGm-z_sf&Pgo3oGob2sdpn=o|(w96=x=AfHfdhAD3 z=a5p@G7f%XZlI4P>t_I)CrANnQ>o9(^cLu^RCBXRrGpz${etBBCP1^!$~N}eAY7V? zv;ey@_Z^AnaZz7__yS*u4mvp-g2_>Pb=2%HEbbO!Uoqn_y{o#!uXyln=v(l5SwU0`$JW2)DnGnnT%7jn3M4xGf~0@X|% zLAjRfKU$}Yt?66haqz<<8E^>_Gxx>YvL3m%!MV1+aqM!=VyTwO&`1Nuj{d1`GSA0` z{k(cY7u|c`^$nj(wRq}!$3y?;y?&)lHcqSWlIEJ$`gAJ6{a&^Wy>On5EmMqv*=mos z>S7P#$+f<$Rew(Dp8Hwc?_ul$`Um$9iXi_O}wF%JY8WCLCAQ zKh5HcorvO#wI_QKw8oU%>YhA^Q{M48eOnLJoAl6odiyu(pR~NAX0!f0X53c$>DgFa zRcEXI{tf=C_fPyH55iqQ&$l&e-8NB#JiImVy81ZuLQH}r(sjys6$ESMaO?y z{H=b_(3_0$y*~9W#23|`^Ka41A4d0S;&qbq()#{?-Iij!`e{X5+!MH#8;izOJ_A+RS7H~ii0 z9@}lVY_z)`Xf5%v5bs8F>Ge~VJo{_dN0^u0F+#UWP0 zZ)-%~(QtCWeqUpcv90;JiGbeqUaqW7U|-Q&eDl$rFivq#u2z3)1rx>2 zzZ#!z9=pIEy+yB=Mt=^=N6dJ&cRhiL`f)r9&Z2a6tr>eB_kOyidD@F5cWsqH4HyoM z*zZg{+svp>zl8N+_P>HJc?O4hFXy4P=PHsx`5oENBkdsu4}3l^q*tZxBl&@Ov-%8I ze#WdkL-cVRrwxl<(Ky}|jAD=#=aE_99!7FZeN_kCW2nv#_b@R3fvxaNeZ;6aw)-Hq zTzXG?PDCTFv_VgJBsZS$P1AD~yNXzhlU6XD>>X0;dtK_A*NEkLfQKQu^xAgPUK^0! zo4GF(-|5UeYID43m-^7qI{2yE%O#w zvG6{h-DZM#4pej(@j9|m(HQQWpLxRV;B>aYhogBpe`TI1f@6|?LvruWIIY)@gq}pH zlsO&!;17)q_shezQD|(|CAcf3C$c#xk88W(UP(6XdGuVb$4ZV|ua=Y9I_`IB&OVR7 z!u^|gk}-D&4oFd-0OxZ`@Cp^ucMsK!0&wzAfkb(mJoPU>t%w z?_~Y;`FT@}uCFc`+Xwp01|y_*3WqRA6iO3_M^$QoSK~ha8%O)gYF@;J7@Nu;Cx=Xz z#Y5ZOSD{H$UD~RB`qLV~SUw2m7WdFZas;rLf77OZY(tl2mhNG{)SB@_wb-rD_dND? z#`43KV#am^z0aPX5a0Z9Y&WE5ZY*pM)!Oe>|JHl@@@Fh;i+)#(O=`>CKYm)`g*)~$ z;D7ov01OXqA|oK<@tI z+Hp2&@bCF)jP$)aHn%48wU*kS=w1I}eji{8Ykhm4$R}>WadQl`+Kcl ze}n&1{4_zwZQz5chx;6HXQpp?UeoTQb87ewf2S_WPy8H6hci$95Z5{j)YtXO>i&Uz z+oF#;*wHJi%Xz7^^@J%;<3|6MFR)%yBtFAgfF4cA-erF;+oN~rSL#nvs>}O-`L3$y z%5;AKE5gcBkfZ*wOdDFehTiu*Qb!Qe3+bnhuv~iUVa#Cr+!qrBpNdb6!A;~$Fg zb>>or>=;jlK9<`We(A?NN#~F5$^1Nl?*%YM@nr1+4(0FhU#??5<){7fJwvgUv6n#q zkZ1IfM>-nh4~=B0JdY0RYu%RgHNc&=;1%RK8K0l=Fnap5;&K^C|16^vGcj^0r z(MEh7nZ@2=p9H(lv+cyJ%saP}3f-6F+~w=0^?L}8;*#4Sl=hEfAwShMiCzaWR7PKb z-0_Px2$zfH^^pwo?_!GXU_+GW{1fSy zb1#5@Yx$1p*YX|j`+~U%Z2b@nw@lthA@^`zydHa&D7Z@|H$H~*z6oT0=_ktB??@MaH#;r!@#4}M zZyhJM_1W)kPfJxq`apMVB6^EyAoK5Da_g2cq^+3BSzzhMl{God1+zOg(OoosZa%TZ zJIU?fr2N>m(LCa_pYli^!TJ_L_xMrvL19^)m?y8}#_WAy$-L^rea63o-6M3fC+F{s zmjzwSVSN487-kSV+tVSC`TmlZo?QR>#Q3Zit~KTTB<4PHpY@ab8m~`=Dt5dC?7GvU zBf%Zu^H0-m=GyvcWgDDp%}%q9e(^gKH0H^%;xqNZuPpNm^avNrl%M&D$Wi_2$?qy3 zLn(GTXdXtnFC@ijptFg11@my+cvm&l;;?1?evHASbtjD2O%&c6&Q5+u-)QoFt#{0` zFDQvS=MP?TJ9x=*r^x~D?WUodEKjcOhMw&IE_{G|<|mW&DC+G3tD8BN8;`B{?5XkD zo%`%~agZoSJv*))K|Fil8@l`5-}0pYs$YxQsk*(^w#u z_wM|<_LA2VsS$P5RG#-cvy-2gf@9qd$QE;(WLstKh8ZbyJ@Bk-Plj`3(33KXsRh7S>~tR4DGRFacfU-6KZ>DskB|$(+!gMkPD!? zG0lzkQ=sqJwP$C4s9&69x%LFmMKVt{V_)vw&=_7~a^szpdV&tQsV+N5=#k1}Jkrp( zXsUU3EFIaNq(2_?MdUo;70-7F^uX_go*2$KNX*npM8F_A;1j@OgZ`zWJ?!GKqVCZX zriiY2Uv({&z&6HnoSs*fn~W3p;32+?B+lvd)>xnEant^U(2N~Zx_In*gyqgtuj{~` z#(A&4p?>)oZr6oB1+5eKwy1$IoB-X>Tu6S?nSAe@e^0IP{du*kd{6E|55SLkm--;r z4QJzzk8F@+dUP!%W%hdKrP7DZ!yaPZMElkWyAZFA@;qxgH@e>}f5K<>9PK;QdHqH5 z*B_i$Bpb-RH=IOo@#)`|510nK>ER<;MsFMEI@y%v1K6mFt}HooNopkfcE6WR_~X?x zuhCoDkYA{1`=ZA(sV!?8yq~U$^L=4&J$t&#&pA`iovho}@^?W+3w{Xmy7UMk1!(9BO`)M8!1Z>i66o%hv2cOK6B*aOYX6Iy>Ts_aSiES7n=nJ*AczOt93wr8EE zOMOrgV|;9hm*`GoItiHF(_$d0jQ>%2D)`Io*cm&Nix)Q0BSRn25& zbz_&FmyJvQbt-Ev`c>ufwBLL9o@dV^<{KmZVEL%}qw<-X7Cl{ZxirvF9@REbq&G8Bw2asdwd_d6eQI zKC)K=>$W95;0b+}YeZ+d=dNwUe$m%B_IGuifY75X6Du+H>Rj->S2`?~ck`+Vq#isS)`>E-7Ni500TJIsW zXWK_Q|7gjHZw>lB(6Z7aN?vfc!AQAA^LYBcVCtp9Ub6d{-*=#UM+^?6+B>%VJX80K z;A@`a=R5Log=iA>L~U?bEV!35I`_V?r21O>cVdy!0BAJc0S}#BAx=K#;U8ZR&tq9S z2iEO4mU~?Ft}gRs@Cm+n+&db+YYS+98fx*hkMq!;LHdC$+zwvS-W&T%=No;_i$`=1 z*6kAUW2D@CKiwZ}<@Q@g_DRdxVPn*@2ymH5hdkUr&kujfc4rj&_HgoNWZrx$on6Ll z-(z#92H->YAUAg2b}pa$DPyN!_K;s2ZIMj@IRsj?{iP{4|D!s$+I%#U<5KAxiRb$J zznp80j$fPWe@9n&jcnw@NWN}A<=Fw(WT^05G)oz*1FPGhzhx{AVoLkJ{JZnsk7eUk zm$xMvv-2i%OG_9H{$HG%a@$tv$jBeI-mg2#>ts`XahAuG|BW+k&(}wNc-@ZX4RWS_ zrh_W3<=>%M?At@GaUUCKCL@}Cn$6!Ac`G=95sM{yF#u?6=puV(H zJ(Y8mk=@VvB(IJ&+j~3nG)8C3KEz{LN>vs_z*G=N12e{7(^IonrejeW=&otg6xjo+!8yr?*QZd*mOdZln zEsf#ik6w+hG}DsvIH3Qzfp&Y>USoN@>D?xq2{&Chowt4*I>~hFT5joVU>Uo{aU?tR zMt$kU*c5(4O?}B_r<>*%Hg6y;F{1Vtlg&G%v;{wYA<-#(#n70sbJg?@m>A_j|0qkjF+xD)yli=W=q( z>y$J8*UEDntv^VocYPgDbC~ox0sjK7hf!W1i2;G#g3j!4MC+}yxCMndtMgnQOlHgJ z*yeS`sMKvA)%#W#{eUa-v=U#3aTb=ztyFPa9mn8x#hO=Z#ADkg&sPB7S2b5;TlBbA z6)cPBoSS-*k$AQ(YyT_o-TAWzd|!edTa~#WzY9z|j%_u`oa(-=~nlFJYBDrz!ntJZl-j8HRd~SF@_1hNftL133wN>?MKG)0G4mv@~ zCtgQCjcqeUverT8$1>PliH3U%bZixU?_Zb~_FXY`U>jsT-uHO=zKGa8C$>SU2W^#} zNY88d4xDQ}_%wNOJUm7|@U8Ysi$vb3@30^`sythx7t@y-ha$c_P1D$D6^gw|?(sGJl2s>)OyfV2oMg zvpHVOv84mpm7Up{(c2zAOYm?nll~g;eHdp4?>hT}nFfiOj*~*)pDeV_c+*2;wm-iE zy#D&F-y;~i96!BYS=}s9m7}MUyrd!E!k~u~#%!90JLAH$rXDEe>}gM zqjgC|U(2jV_^|hgXY(dJ(K)R@<0i}= z61;7Oa|-ke;ysE5MW5rIzIZH6MPnHUH;Fe6^bMF^2>tT@Ykn$qJmF;*$Wr21@$t@l zHmHjxQyDWd7D0b69XG88@U5l$@PV@77ia+j*u@cPgcqhf)yk&3D zIBJ|=*}YtUMz80RT(rpi6EuHxF3eNo-hF8vKb0Hh{lz^~pOEn#^H5-Dd!aTTtY=&( zEw+K8DTZd4Qw7VDocEAyKSM1nxyvJ(W8FxC|F`gP=y#?aucJfmyWg1`pBmLhU_?GH21mm|C$`pZWFm8M--qF?&S{mE#U6EUdK$e z0n}JnP<(SyDtmkCkxN+QzZSZJ({1r9zkvbiNg;K zUrV5VL+A_U(T%#lKRVOc?@SM#(EVIlhWLqeY##f-ksXuklq3&T+ks8l@I9#SoZEUM z`TjTou0xW~|I_A4pATdQkry1nM3Gw#R=Jun-sb#!t^l6fL z5m|+O072?n=DC7z&nAGgEB1oq%o2ZZj<5bb$$}H)Q89R6kX;RU`6f)UpD1Tn6;Dv7 z)#~m>Dz(3g+BQ6G1Y4R6z@G$YmcEbTsiB$J>-3R zi0j^#<=D2BmY&d^pm~<)So$uGGkJ+|@l1m8NZ0Er_$j5(-%aDW)#XOAG$l3C@U!#v zL6i6E$--GfT^URlwEkpe1-zdu_cSjM*DsCFZt}Wk`i}crQpoSd-1*^@b8MZ1-o1GZ zlkSe8TfJ9al}DO-<&iogrULk{&`sQf!}dPJSf_)RxR()?PuRy_+-XW5!V_s+)QO_l+Kmz4N$Ir{p_zlkV0?Ql6{#Q>)!Z>tZ%M zKVo|qe1}gGMR07YkFxx7_xokPTKxUGWlT(i)X&2`Xji+*7HT|}Yv|GX0YxrFk-J=~ zYsAE^c79^UUPS5}<@b!yRBQR5sdwtu4GxR{h}VkyiF}v*jlGjaE#^hr=8hk!j%DR$ zA3fuzLN=$Tnw+1~=XjDUi|qEAZN*J1wv}tUv9~iaj43ojZ!TTj#6fe4gTy=s;3K%j zj;*mdxB1`vj)Csz6O$*)+BkReieIQqt>@hC^1NiGFI>ZO3AR&;{%Yu5q)$QgdU2BY z63E?+CvZR~zhi{KaB^{WQZi0J4z}hIyGeXRHa)@AhW@UH{i*nO-W}$-z4i8 zloOBSz`=|7lViQmwVcsjYBH{Kf#d-3kEFib#eK@7q7SLoA9N?t%Wb+@pC|Jde!>^F zN6ANA%dci=8S$aBGc$KibETW?@p7Jf%{GsU_rg!vR%h1P4PC)Z^F0!p=J7Xe_^rh1jts1_}ZTODby`k7xV!?@WEBh@Qr>()-s+;?_LCQHvO1O`#|NvMsbDZk)DvjL*~}Yfw(Yb zDGRLWUQ?&3Ye8)8;8?c|&bTZ#oKG_y%hC>=buW@S?i{c-oS%qC;`fIBxkzR^mYb}s z$x#~yu`b+>xOOMnbMu<7t>|5TW@w{(JK*IQe13U6{fc9Ea&^7vbNzxo_tZG{b~Dfg zru`mA*9&#}v0q1WaQtS(*0HQ$;Oe%(7T3Tn#~@!XK;v|cz6-V=j@{w2l;aYJ?$LDK z!2gK*pA_Q`$ltalJ*{dl`(KSo-g|%94$>~;RDxc8Xz%L#wp=^bA^eb~*@B-g+5YpS ze%jHs6zGBBAo~I7rD_~{q^5YqW}egdN0W_3@qM)SHpM7FPggmB_v>jx-dv%}uHOM; zpf#@_*P*NDrmEr&>N>`!$$|^?MI=pU%dTW_=3E;nvT070Wla5f*QR`vbq#Zg_=VI7 z2?Qn6)B}%b=74|r6qo6^M7j{7-Iml!c>$eesE2{NtdHAteuri@mZhzqh*xqp(OEs` z^+|`p0nx{?Ue>O~UxVRxJ=yfG*q_E{E7%Gm&%kyZ3rH8fHQ;-h+PMWCs6VuVaVdOW7MA4(%iM2Gg-qQP41mo8`wqn?a++I$@JpQ#W+afqc1>z5w1L6 zp45usPo7}#o@!=eAjuoK$;X?F8N}natM^=^*JBxi;|CYJ+LU>ubwhLH8>}Qp(Xq>T3dpRIo zF+AX!*zUm>;t1KkxE6-gS*fpUx1)#7Zc^1{Xiq_+BTI}O@IvO%J86pazT{`fmfoyt zdaJ~FX|IJxu4`9_1=o53UBJm}8)E%yt{BUuEFt{|NA<25sM@?&H&=^ zNER~pLny55>A^{EIli031z&ozwb$(*kWuOeBhkQWafW0vVE&bBm- z3~b}#Ku-HD4%8)%!6ss!BSP|QPL35H=T`R`*kBhh{W!X4-@-sO6;H+U0cFQeD?F@1SX8ZE@w*}3H1*YaFbSp4@ zMs3Z^-{`upjy=r+K}_AIwKF&$y8edoSsE1MJ)lE<96W!JBdT)nFmCheBVBti-LI_M zme%jVCtb`j*H{j0Js!)y%K3&Gw578>4EOjBF~Az@9+il?8;$N zod819+V=<+2Yk|)7Y6RZwQjX}x=9>+s-e4b{F{~2h;I$+M*8+vWnB;l11jeX`C}O) zK7#S*){O8H&d65|p8&eK?~ z(0e*2HyYx20zUVKy0XlpD_^m#3;9Xr(H+Fw(c(yK^dMZ>Giy|2Y7&pF z?qzJd2Y)wPkLTFOt;rnL!{_|b>zG?@7#tnz&(Pvpr6zD z9LB#wyg@trs+q1VajW$Q1txuH92|khH()=ueu~C5Lw|EbXE9mLyIa>%_&{B+-NgCT z@KfN3hmhlQB0bzVFO@D{a_d4*7a6em+)6bwcP?#Ks&e)c^XO-!=JhZ*xCvmIH=giv zc7!nWl5XtDz1z629rJkUuJjy@qtSKJ-y7o4I$l>bzFOMvza3lU4}r=F^_c0Bujf77 z;y?~EZAjvO1ODK2^ouWuNgzw1m(13NAYNSI7HdlP(=M>ZTW>eXYkRw-UxJ@Xrp{F}Hc2pCP^z(0pj0IE{+PkPSyf3?!kO8 zodvkI9{5S{*k!shV`nbpS>exJYqmB9-394!esJx{+K$B!d@pyi#+WdvJMPSBA4m@B zgnQsCMR&J!@Q03&)2AY%93kHNI}YhX(Py5YD^+Ayd<`}((g`poYlxoLFP>_M*ME-d zLZ;#UB4-WfM&$U794|_8%Savz#(;hCtj3(T0GvtD2YYkyeu#lNyS8z{GN*i>O*Y$- ztgqCeKV$tnpL0nDmFt&x<3QyxF8^=JmyUD@)cQUAwm>?lj^l)zgNOEJf%`B5CQ?(C z2mpY!RoF}q`eQA+&@`UaoP{mYJdDs_xcj&RZ+26ql zh>4q+XCLGU|Nm$2&3YVFwtvCza$bEU1D^lVeOM6$qrnbFL#rRa1M_6C0k`_Qx9l7$ zv#R#q|8w?!xZQoTl#_+I zY=Wn)u$?J=U};@gIP>t;gH5kqyy&v`u+*7w_KKOu&;NU_&#|Om-pk?qgR+qMw;oI( zx5z$qA7;Mu&Z~cmlZWqeb!(zWAZ}*wyz|P9lf|bzSmfS8dCH>ry-t-qYmJ!PYc(ZMDpdfE`aq*tEX z<;M}~_dLLdcktix+`?Cnu1){Z>;K6sFTB3HxcTnz z*nRelc-d*=%T`PlpSG3ElU7UmmyY~?&3lee`CtEXpd1Ezrm! z8)OiNykZnq`^#DOb1lN>Gesr$inT2_v?MseiHo_lk&D_?CVVG zHNV5pXV9Oxh|e?YA9XJNwAaC()Y|$bK2-haA^G2{<@rzcDENX!_@%$wPtW<~-kAwz`idv`9Pb^=_L1vw64~}k&*4b@!}?&qoaz762l;D{=D)%N zeEI#77jY7ECKR2IaqO4RDWKTroXC_RerTU?qL(Q~#j6dTxL0lPNlFD6XRuw;4n6Gj8-%pT$4k)2L7R_q>$bNT^|jCS8?FAB&mY!nOkTLPI24E6$0+to5(?RAXk%g*@BQ8Vh3IOKnO-ap~6{`2?$ z&*jgrV;S`+L2)SNWtg%r+3Io53mbC~hR^GKTW60m;~u{A`)}mz?f<=U_V3rRIu@Jq z~@y=Ji)mt3gufyW_rLN46yqazKo|`0T zk!EG%HYMB+&xh?RkLicM>lOLU$7l?>6YHC(J0J8R#OR~e&s%TI*ZU5yUY2jM2l+?e z$hTO{E6)7FyGy6|1K)HkoUgS;-pBrJ9iiXW(t2OFUus`z53SdeKOQ%57jN^yoBSkd z_jf$U%P;zQ%-_YD-+cGH|A-sC{mm|pGogLuMZeXSn)xQ0d7p8NygNSi7`M5-Y83rA zdrFUZ@uhxo_SmLmuH*TrS>8>1&@VA2z5nGMlV8@lKH~QrmAAkDXzzX+*I?hynZC}2 zjQTi#tKsw|4(83^c9NV|Y~_#U<8$lJ*!}ywzxBWTi~1|?b7tS?%zm5mKNzbRPom## zXA0vN%x~h}fsuBWfu9OQ+NlbJ>mZ6{;?0=9rC9g zbt>%T8(qDvnf%HZGVU^-f0G$9X5ZrmG&+_>5x&*Ei?*)AdF~5$|8MZO$Gp|!_e=6rpT^}o@vTXr~ZlejsGr*%-G+@B47Se({AmlqdN>5e$N`&`!k z?%qFI(``bBpRwq_)K7lfySMt@|J}7K^6rIQ=IY*ePk&LH^MCs(f7?&V^!roay!n$K zJ?WqFX!BOj;Sc1oUS!0#T%k7vt2oqIfW z&Nuu)e3FdV3r;Yfnzxv;(JHY9}m)!sHr+JFQ>%8|zJ-{CroZr{x_@JTpylLr$!+Ou< zc(+ldANgG2_*Mt`N8IcS*5=K2|K&{Tmp%NnOMcw*R`WFKKd^3RQFx=X4=m9aTlg~n63R* zBmC)O>=IO-i@e{ydK3EX(|7z@xBCzCtA#!>#Qfp8O8b^Ke_95IH~n}NH@82C+%X)le(BG3F^Ym|?=MgLcm*E8RhyKv^` z$5N?%@o&|M`6jRbv`&ol%Ub8s9(GxuGcJ&8{CiA52Ayz57-=*O83ngy@AgCA!} z8}^u>`{}I)c^AIX%v%n+Isd(1`*zOs)}P?*jNqFu_4jHY{B%_EuQRfkYk015;`SE7<`0W$IZV@dsTkw@! z_gQmI@(6$wd3070@Bk5p=MWfVfB|#>c5k1Hy6SyR0k|C3OuD=_+`K;=Gkb0;ytD01 zmncENX9IPz!e)!96_+;vgf7eN#I`A5Y4IFYCIDapAi!WyK+_7aiS459>>$wfy~*o% zuS&}(v5RU10G+oCWz#mIfJb|BDI)0Bh%Tw)a%$CuF(TVaPLQd4aX(Zsng6v_X=H~BrLtoAY7oMB-V~cS3 zxi35U^Wy?Dzuls%9z`brsHRgowU$+lU=9JufRYGD6yOBpxo@BX1cJ3u8UPR}6G#n8 zf}(-oIp(a1j$2~`>?BahDHWMjTivaPH#zVEZ(BpvvU%@qSw=q0H&|U|5GB*}E_1T4 z_8OLD4UtUmYLct_G}>)<^;{nFO(A-Ng6F+j*Asjv{<^488M}(($Rtjff7F+uAlKD!rBBx~Q(>kEI|0*t zDxu@|yJU=>8cc;5HqV2SWZYe5VMXo<0R z_PS+1?0`p&u8XO~FiLWcd8#)&g`=+(aiUp9L_F+K6e)GZj5=rvF^eJr#^?YF`ANrx z(diar+|6_%Yy+pu6+ta1o@mk~X{|;r(a}R%*NGJ*$zqjW7hS}L+f@}UO==!kJ9EO* zU2^4-sV6Iy5vIzhhK+!=%-~UNNuGH^m8bL59ke@t`7r!&yFH%;*WuRbLEEN7)=`{^ zeZOV4!R@%^nZmd+OCFK6QY}-$UyNlddxNMLZ9-dU z`dW%&0howDcHcdYttIlJbXXsy<(X&J*9X7sleTJF6(#MhN_ts?7vCIGlTCr0T7Mp>OM_|2ridtFHc2yNqvYG`S~1w@#`;TQ-}xvk*t>TaqsyL945d zrITR1x$~P?6h?g<$J^Gcvtvh=ReNH&ttK!@WI$QiSXNrobW+nK_1uubsGTTpY@$@u zTQP2vs3L6Ht$Z#a#xUHHTQg2~b~T!{-6R1ytc_D(o`e(4^~WjIz2MP-^vGi&-1=g4 zWW3ucCI%h`RZ!U3YMn!x=SI4CY_IuNwGNEhWrxM4DR2CsTRQ_H3RZSg&;B)8)}h}& zwn=S`!F+nS-0{idOztR|VYX`cN^y#SI*B>BmXJ05Q1)q129hv4b8wQ^F7_e8kt}U@ z=xX|lW@eu*+A-X8hopeM6gUv80Fg`E82!X!*lJZ>W?jRq4*fOCAddMy4S4XjSGcFh zu^-w};8d4%vn#)HQ|C1DgmF|&)scFWk*Kp~N#{naDsjORQr*r%mmL{EB>ggn!@bvK6fMS!!8VAA~8FS?b_wFJO_BX1LU`@ z;@*!z?NtdHu(n!rY5s6+r#K5rEpRI#N@0v{VRS|}}aI00=xlEAO3>h2A8zuw$rKHf^Q(ki)Fs}P*>RlP;6MXVz6Y-FK2 zu_CV~;nCtuX~$0DK=0>xfob3WV?HkC+(6!-FE}0weDOi!VjBdz@c)a>F&?2QzXBxWZ zE_xWa?PSLF$lSlQJLyr(Z^hx3#G5k*%SUxd+qL4Wcc-iFZ7FfLt*~6wb7WYCL)V5@ z$aZ>A7gx~c@+=nB{n=c)(xN*=*fhgKPgs2G{d(CdJ}oDQn90~QJ}G2$cg=LN7%bUT z;#{FuyFP`;UO(~xMe}h<`oopK@Dg%P)eL9*lNc!zqa&hjL{Y?YT^41ofF&l>me^ql znW)=Xkj|-jQ%rr`om|$}mcw+mjyPpj9EfDxIg>dwGvKn7Jdjn$cWQCHz3cL*(C?eg z&sO90(^S;7A66qH|5lZ63$IfnfV8|Hru!6Qs3-_P07XD$p#qdZg31bc5g!1KK%`@8 z7h_T8QVxt5@sv5&okf%4T))f*4O*ky z6-Cdp+X)Hn)Y>>~J8jj(I5P*CULtE|QBbzHmoK%|mq~9evURpci13VHTLQf0jY4z9;0fC1I08eXjVe6Km z^K=xWKuhVl@_Z5L8uod@cv!X6Ak&meOpWZp?rr1T3w*`B6NA70#!0MgIvn)L_+Rd8nghgqJ z1kABZ_i|BL`J=51=_D+fPHrkP)y(teIxI!)d|0+)7f(LO;xfP)eP08UPR>sQ9~%>} z;WAyLE;A#TE;bv`0AD>_X?)$%wQM}sS)W=|)U6<5saOQXlj(U1x*;oJsEEh2|?qU0k;T-vGb`%`!?F(~BuNT8x(_m-8%=CDS z`c>mFss)06)$a8X8F)EG5$?*u<0~JhQI46b?U8ccQGd175{K7$wBfP6i&J;wsNN+z z6WrLtZL)}0Lpls%9iP&CZg0uyDg^uDaLtT;Y4~?Ghy*c9jx)_P15aL!>_PF*+=;EA zJ}(Zl&jrll^_qe2Y&HMnB_x0P&KA*r%U0*WUz-3>OVjCewj@0C1TbJ8`lnwDNO-bJ zMR61Xb!hY)Q3QxUj)qR&$j?<~o(2g|>Lq<`0HdSM(rI?|v|H^7i+U5J$D%;2+VWWm;YSJB>Z+7_6^fNnRXHK)<0V)j{3&Q$xjSSn~2BFQ9s%Qk} z9-Ys2F;%8w>>%?1iKrJ<8PmsZ{m>o{rSPX8x`%&Q`-H-GfA`TBHwqxs8-F?-Q3A3B z)RAQiUV(xH0P+ApmSMUAm?&OQFJ*SmJ7N)M8;jlB5Sqc+MyXFW@|!~n#7r=+?rkyB zB2;NXF*)w^?B-ui=6zF+ZTo0~^|hJL?JM-9P9Cf+Qs^SMvq%`C=0+SN+=ynAkT@lL zC=jx{gtQyF>jh>HOSg?Kx-EA%RgGY&+;$H&JM@`-Zl;FC1=qQ$E~TNI35`3iL@Ab5 z_rQfkYj-kRr$%sC9L3w!t{z5VWQsMr*M8CO?$<`BcWQ-w&*xY>;|=WFwc9Rnx6@YY z=5~IVRfEQHC*+fHz9=7ERM`>Ewj_#SVA_1$(@zIkDr5cl-9awz)6;eRlwRc=DA+~6 zjn0~k0O2Pckyx#>b+#eCzE?e?34o`%zAeth!>N^B=o}7&Fa5Q;qg0l6`(;c;St0&b zU+kP1G+r7`qhZA~+f6szy@#J1M*fFt(b~_Cx&3=~SY^i*p#ar>F`Z63E$R9Csc2-t zJS710K0zo_FyIPB0bpzbBs{M)e!a9qL)XYgW_p?FEwE5!bGcN5&dV?3UQC0O6>!Rg zTeU$oEQlxZtPc-^sqWtF+Q&;QugDoQ2i;(itu7}<5C~-{M?hzEf1C1@iIkwZKCo=x z6M+Xj7Bt8QKQCCD?O3; zinR_O!X%!_3r>!|t%vepr*{fq=gn*~d-_3@OaTZ6nGynk%L<;m_+*ot#nUW6dZX{{ z-Xa@oP$o5MLPv}M#wU*?S;mhMs!)cOH04PiEEGGMP-Qoou`4KjlrpgqdV(|Qn}lH^ zEPE8>*wW>4miyrL@pWfpT|ls$06E-;(=|?|eY2Wj_2JyN>(xm$*Qv!QM+S%feRShf zcH)jA?}-{u@|y>;PAkzlDV%1^Ogdr~08y-1%%Mqa&5u8$7jLirJ=P;YtY4lwMVbRe zhiMNO0milPRP7)J5|-~8Ip2v3lHFR=6AIh2Pbh5mt7@NAP;4Xw;t`nynabwY zM>N9wAyqG#LvwR}5F!~bEjgu8BxQ&y=vFPbz1sAe=edzLQO~T%ij9 zaxThrI(H!=hzATu3^E2v0A(Oe0Wbz>(!)L*#U!vF>N(SEy@mjICMX*n)m2uodwWBr z1^`uByr7|(OSwpamjQIKh=(ScPlM@rjp~&>bw|QvhF2HaH6^ZQ7a86?2wDcft35(n zQGs|%*i(QU?Wy!dX6B1yupScB-Ge-iifBZ^VSS^kmTOU#a$&xzcEZ)&mG(NB4_YwaHCm_+GX(sg)8ve|!4~RW zz^KM@LEUWJ7EE_^wX#@82BC01AVXOM@r+h_y7 z?PN&-LjP$`Zg$^F(k@Lew3v2Mw4&>>^K1;zCoip%<@d`DD&v6 z^MU8JkgsBOFY^!0xe3;_w0Dvq?-y-C^r$mD*~a3W)^gYRb<<`$WMAU;_~~`PjLeCPjTWi^WTaS7^d)~)%x%hC(}Kvvwp(NWxy#FL57GZ1Oy9#9pTDTWgwt> ziQ0qR19kTH$Q#utrj(}|4FI0T7X%V$a$gj(hd_%^EEbj$ix5`|i7i%%>Se*Ag|aMI z*vK2IKSh#}F?*X+&z4H$O5EQQWNihvv|TID$Vm(^mNCkhdQtg8B*;x4izQTN=OL!& zE{Nu*?&!qTHp>?+Z!hB+8{({aLKq_O@o#@ zt256p9It63dwyCE%wVS|*__m()Vbrs^#~@%W^zzlldq=-shTmOHFdqNBA1d0f(MOm zQ@&ivjN4rIY&g>F)&%PQw0P9qX?9(zm2Ta{bWrChE|Sul#${y+memw;qp474?0jL1 zapZcju30mxgxNl+5~PbY&pMpu-uf_XEx$co_t@@p!owuc#35M3lidwZyTdu&V}|3s z%DY!O;>Cq7E)Id?EEO}c1vgO)KHmv?mFt8|KOGK~PS6Cti#Kk8QyT!;)7fRZu@D7b z-92pt3dxfqA`CL22w;_=2*8@Qz`LD6%Eb1yvI(?;)yf8VnHhau+}Nb9abQpF!l;)< z6!EgoLs>5?xv}A?Z-ysPR#*3+o>GpR%ZK%Rt-5Aw>;^7K&`lMq1cNuo9$oYZib%a0 zQIkBb-nNV)EZd~D&dA)hQhLu%eb&#KN?k}{eqLxj^rvxXiha1g=$lhmhSkPci|%;5 z?gLhyb%Md7F?a-1XV$=Gl2$m6$cs$)kzF<1GMn9rQcbf1v!bn=*k{ zlj}?b5CEF_qcWBYQ!O{xbor`Jbm6uPIqg%8xHH)felnzqThEpNqFc@p4X4w7KR+IU zpO&*I>cP*m9y|D4XB(plNw$%>I@0A4L>d%vMuM`gqPK&S+A_3kqhy1=x6ety8=Ykt zDQj85gwEr>4NRXu7Pc1NqZAd&V39(9I5r5bb1fMDs>j~!Y#SHnT0K7sbg3;}lVlsjNw-{W1=Ws(PGRq8yr`3IHK(21)7Ct$ zihX1rTj!R#^_kBfYN}+zRkE{riL|le8PRFY^ycBE8t?HWu}^wqc~{-C2){#|xlfPE zgDf6oKIW4oxH0B1sb$WIb$*>GDiiUsgGEJAoJgq3ZSCfcHfW{1cLTRo0=DrdMA(QE z2U!_eGETMjuyivD$zEoH-{84C3*^bl%=tU%xZ>!ggQex0I3bx)1zl~%@h2g83A>xp zpF(H+_J;*XKTpBVPWxMtEqU$JWyH??DX~|ZMRT3pMZOY`Sd(fwi&&7cFHXYXLM(_c(m9|?=V<`ECF{H_7SZdZ zVuTTZVQ7|8RK3kj2k>>NdHMrVqw{KE#)_xPUha+Tofq<(Vw3C9K1GYfP1FSCs>kno zF4sf|67>Px$Z*Xm9x}ZOO>Ph(I+e<$v~%>xI@BAr+Sv6CBIb76C=Mj4T^e<($Z8s6 zH?gzT)+qf&893%n*RzHmAzk66?vlH66|Yyp-BM?#mfGM99;~#ju~q0Q6<_o{m~b#Kv!AE_>3}QXb*ma{#e9M-qhBr36-Scb(wwXU;1RVp59pwyK|&XvNZ zTdCU23uTxtwDYLSHaS`r7Nvq$^gwahi=3o>5U|^|?XtswY=`z%4zER z?GCNgtY~gbYppO}R7!roMzo;oqo|Fncvvs@vqr|wF}#^%$dF2tZMh0&;c&|&wG!9e zHB53#3Qe++Zdz8{m3e>QsO|fCp~s2H_(sBW?R|SbJCoFs7Gk_bMU)Cy*zD!e*;|Ri zEc1uo+^%`-`njUW_*e>8cRG$&>g%TCm(eNn4jOZ>)jn7~F1LB(#p>Fei07-j=vc?9 z?QAR{QVWv=p^%)Fb+|P*^|Cr1bQ+3`D#VAR%fv@6xSuE=#2W4PdtA`zXA`~!5X4I) zfOMIHU$G3?PuvSh0UByn_C}XEo7t;DfHGg_SzfFZN7r*5Wo3ivx?%u$mWPp-fhPek z``}e)KOgEdvDO~(wu7g~H(S8QP;oH1USrx`#i}fB`x@>Cw_4`XT#3udw}gIcFK=6) zDUF4<&a>8KV>3iyU5=@}XoQ4%oT53&TA+&z8Fx2L+*y(p0yfk$yVwWmHaM+fq0xGR*n5`vUI7?BSHx}p4q+E6??04t}9)d)84jY0<-3JjkRX4uBCaxar~0> zScY+mHhm1-{=BIWMEB+9JZyE7)$q~_r&d6hk_L0Fz2aGGFSiKE0*-(`JH4D)K`=51 zw04;&Mq98&*;a-wB;&Z7Z?`*kC!gdUJ&mV1(56bEGAWMLgTiAo7m6gw)P?V;ZPGf5 zQ#!VS5TBdr=dmwLrH~}wS!(=m^&Up!1X*!P@w4pM69V`GCGVpSF7DDLd z{S@G`I~^C9U=MrNvY3LEYBAEH04)ouH)6z95TFXHXQi&I$R2cEwBE>4jCfXWcr&nA zS&Uw%K_3SrRn{v!#1*_q0aZ*OM*M^ma{Sn$hZM?L=OY*Q!O6^J%PWlKtseYL~R?*LN@NFMj*o2_T3oBo^F6Zojwt%*@m_^BGOUMkMNbD zP2wpEi4d}s;;^`C*EYP~Epr@q1w2)JoH^t6-3Z;ld*^YSV~x zENH66h^p2W6`AtY)SF$0pgQfgq$owv)H?>}ReKcG3x9A#KW>0Cpa?+oyCV+>fwNS*SeXv3h$zpUCkHCIesMjb}l5;XC${OG=#KHk%CFr;!h4B zZPp*fZ(dwx_$;0Rz);Q+z-$_6%bnKa9U)KZm*iShA|M*7bbD57bzARkW^ZrAEa3a2 z0%B$;b9xf5&jlTuIve+_tK>_Z=C+QNc@Qj^?I@=5$|@?YP|nUZSSFL+T`msXkUsi7 zTU5I)68rgwIVZQv_!9l(VJM%>B9hOb(Buz9yLCKmpYN3KpOU?(kOCYf5k!cG9Swl$ z707@YN&({@UMFS|ts;)D=4 z&VEYg_ZatBHtE8c*7Dps`;xtj$mj`5W$w-sSsJ#e&g3y#?}%X>9}3EBn|y}TkUlW? z_4l=KuX(iZ0lfQiI-Rt#46+<27FgdmJ4LbQUg-m z1iYAWnJ?v7y;Q_JXRmlV=%`Wz-Co=4x~}!NgU557NgdmmH{%h<6PT2)SMg*Rj;72- zOH0HWu@bLZ`U*jy{feZ7Vlx!gHb+V-ML2i?N3k+@k_rOy0RcnrI(X zMo0D{?#_(q^4xWQI;E~k8F92`vXwUmoY_I6qjD(+phgP_u_Lmg0@ zFj3q*+WEb#Z|q88D(AKiM=WMCk#513eOS$u!>fQ}5{_#dvx|izvkL56`xZiu)k*!Cw+_? z0LZ4B%XB)U5J5&|0>V=n0*ZAd0zkY0GWKM^FZ8p|n?a%M?RCa5fzDe-y-ck!I@?&u zQW@(S$QtcoBgebjKrMHs#bd6w-eN22mq}{ZT#+r?#Z2DKhYQQtm;nNs|Hy~URz?$T z8&1Qy;Pyn3EQ&=VA=Nd_Zh=UQebI`?qB|(9#_oya7g)gQO%?7pJuwZ4Og=GWw5>iE z!QJn-%r{L>WjPDh0Mg|=n@(3M^cC_PpXj5mtEM*jSMXsd`c+#$F-^!&El+QndVuGifGVz+9)k8_bgVX{--&hL~i! zyDA6o+S~_&+&aL$SaRV}3m0)T=G5NTt+}e*zOeL-vb4OZJeb^>6@^{6J07CJ1|&Yt z=TpEtUFWP$o?+CXbekVWiu#9lRT}Ho7~&)weV*i?!g**h;s&?34koqyz)uZ3c@XU= zkGh&I$VI?jHG8zD-6GtI*;Qr)but@DYG|=&-#@K5LQMVq-HJ1YW=JXkXuT()UncNe zuYzI4bB$l%)Tq6*N@n_;Sa!nU6ZCWpTCe%eM*QiMjpX@XYMN6+mQRzdE;rNZbVKA8 z);s|86_s5c$>d2}kqtx=RY+z_yJU3~V+hF8m?K$x|5!ghqkOJkQecSTnlyNcEpXr5Zvv^(A6)5&qn({x$X zXjWieZd~&mXdX8pm?0}5ShidZk(35h9|49Bc&g%PuFA^Gsi3n2h%pZa*6s1(p1 zM9tV{Kma`Hm7PEciJp2Gzy8GNsfkW0>4Qi%ma?xYiAWiS(Rr~F3!2muLN@A$CIXa^ zgwi_OSb;^EH`{tcCmC6lqAo3*k|Z@SRLHKgtde;Zsl1bUM9_#J8CkN(JI@ghX6)`P zImlR*Bc=zI7t0%0Q7#XXz&1u-G>u!^~i#(h=qG_29H>F(x zf~d5ZfDFgS!-(t#JZBcPhP}}b_I!D`X|r&ncALh2tWOzJ?Pjfq=iNqVI1moLl@#ao zTr;@3-Po=vayP+U6xwE|-qeOU?z2KjOYV%B28qjV-gAB7EpmQq$4sS~v4+ihSE6RJ zfuob^X_~j1bg82GM1*-wqepkSKlp%(TmE|4Jq7!8c12ZKJc3RW)s#epxJ!Rmn?u$x zefw}Kf3|C!K~M^_(>$4zW!a#shXUG=9QRgstC6FuR}(7;?JCvLkbhD{tg~^_Mc*+` zy~i`{Q&+vQ?D@sA<+8B3x}wl;qB3cCJ3f(iX8*wRM7>nM{Qm#*|NbBUI^WM_*Z%9j z{;z+Xf-d;ifBnmywaG~o%D1)!n`~RYm2U7|`razwE%j1MVc3~WuZ_&ObLgO1Rll>U zULWU-&T9+ZI*vOlx@tUGYOj=bM?&fZIFyrqn9 zZttnP%k8-ST7qr4<6*9u%I0sp_n_lBqd$%h{Sl3NN`JG)^-9i${8TutoL0jYE4qKs zAD(-9jr&&ePe|Mc>2;C)u?`Q^{n#7h$M;{a5t-<%HGz@)7`@8DAB&Il>vgo{;(O@Y zcmLqmp`_nEuFFq&Q-<6t{q}yGDe0Cw)h97#y;s{H=BE_#Z|RN?|J71!b9>Io<2Iv@ z;kHfY$>qSF{v@>#Zl9_T_n#jR@R39J6FS&%(NlM!R)He|K;)I87e~;em@w+v($S1<;UO&=9t*6=gViGAKU2n zQ}6ZFZhT(K`0^R5=hGwlmNf}-YEdR2pkcddVRxDjlD@ltJpAp>?@`(oE>jx-B%u51 zDeyrYpB5o+SyZ9;oX}K(tLK@`>DY0r`=?p&!E{o68|$K8G}69O1tI`Sp4u^%5exED zI)Yfl!0;Z~Vg60R{p+Xe>3A7bJHXfrz0;brtBMJVF3G?G4ig;OSEj^Q^?5N0ZEM z{zf51Udu7eJ&q6jao*SbaMP}dk*Husq^tZ&+Pk?KWRscG7B|D>oz3L3zYGV*2#T`4 zHg_RWajHd1kI$J|dz^J>HhaTjsstXTJU!GbP$;6sf@0Hlu@HwuLmKK5)XaNZj%$TZ zEqb|Psz-7}FCWAvcXMlQe~*s%8NYto7s>P}P3Hivr~+vCjaZ*1;*%WrD?obt2*jvR z>V|AA3ZQGTuId1&vsk1CMO4`945UUh5R7Yk;~tsNi2Q;*+?vWRRhr$;4{f|-qLPKU z@I2rs#2PRSb>fhX3=bq_85^ylT0*oDH|t<`|Nc>dj37U|BIJ>uBg zWz*6TGKaLJ!$QK(W+`O>3a$WFb=p~>Z`H}7hQO#!*NC>0$u#%23s)*3CfVg=lV!d; zipYy5#nRr%i~Y`-c|qRhnp+os41T*<4Qu62vx#k^SV`o?z$gW?6LVwkHo4c9g@LTa zZF^(wSk>&>zN$vDmWoc^Boi6CF|EC3#}3{()`yL6RgJlAA1HFXCSRyOauZ&=@RR?# zyf1?u1dg2mMTmy8X}=FHC%8`UvsF!D!p`+EnJ?VPGfh&MzE;-t9vAY!GtuaaMM~Bc zZzu~L1ZynVk8muhYl{@;s!n(czEr;u{E`Mp1F+sMF-)VbMayV>tG7(Q(Nc>(?NG67 zF6{aC2&-HxY6_l6<6xG{JS|-IwUogkUy*j7zk}#fkauvtJAyQGSK{U$YP^E&h&5u9~6JcwCt!u>W_efLc z7AY_hhJ*T;@5{K%gRx?Qq23s<_I=jL{PD>lJ37&Y4$#ZZ&_ z;uOQ9k~U=4KbUcLE}03s&-|VQ%X-2@?T}T;*tay{E*S6kcO=egBtDQfvB{BLLBcRs ze7RxU*Jt9zG13z>%xJ2yeh}oa>J2SB;H^{ihhTQkQuT;)Rno1kam-G!zhT8Ix0xP{ z@}wZuuRYcrSJXt>5S)rox?BtCygJMHu;krsIgiSW-w9W8=KTg~Mb3;J($A$+mBV3{ z5jnlD(C)lUuA;klt()8%{duX-gRkK&n;#PYf}KTwXP}I2R%iHYBu4^QmD_52ZhZeX z1eJ~@qNqL{LpfFylk2^i(yn;H;_<}hNug%dR%0NWm}NMGgj9!N9u<_cNSirwP7$*X zAIz{Fbapft$nLXKble@+qJLP_+~&kZq(#a(43j>Tb~#wbQVnppUMf=AU%P?psM7il zV<=Gi53u0z@EOG*h-g#+bjH^5R4Ejx``z@W#r`#8ki1N|6<#-WC3BFc)~w$Eb|dA+ z`j!sy)_nA`B z*DARk=1#(YOjXz)ofY}X@YA=angZ|z$kXbT3Pu4{5%A=x@Iy1DdgErQw)bgNCwR7V zwoKIx7YQg5gDdn^(NTc8yWWYy6$MuT$~6b|*6Lv(qyB}qN0Eazm9-u>;=NwJhq^pP**-pGo<3nonsHjfIK zgNLa_Hc`$J+k3Hq_$`xkl_JWidmNOF(XG6Vik8r z>8=nzgzGR$_Uel{{Q1UT*1m2*^eyAomi_^2Z9*$QDLs;y`W^_AB=~0f<%P(d4r(Y# z`~7>&&dozwjh4LdVQJkAS_QMzQb*8J$jf)e9krk5iB4PU{`ow&gvjP+n=&^lf=E)J zu$?FZ^i7syw@+EJ_Vtdpq`d1WumY!dLR(QvtD{^aql`&O{~bskvDjq)qn+XW6Epqy z)X=$r{|q0;@6RIz$nR@=eH~1>m87rj#|3;u!2HDT3jzXEqy&frkjU=6m&uYql4R|d z`{;dQsd<|KWYwWBmFaHu^quNYE!oKmk%gt9@l9+PhPl#l$9Ouz1&ES*H6eKL(dyQ_ z(Oc=I+ESu1Q67$X*>y*);G@uz<)UI(i|>CjuyT5zGw#n&Keu`Jk%m0C+}590_+?%K z2q@&=Unt0*kh8Mmv`tzWFTE%~)$nvQjn&eI(CX8)aFnSGH3jblb?Ud+6wI@hIn;W7Z|Uuil@=rrj~` z#kflU>bUDHyVIXO4V(OSHvz=Q?W5!T({;TUKl~BD`1MXQL23wV`D;=nS@-grhQ1xG zPm-)iQr}DxnZT03Vp%d`B0aGEPLr-KE|&j3_s(h2d||$~ zT%A5Vc~fi}nJzj|hCN6A-4uFs(mGe#bE6E;(V4|9>oeZiDtC3;g$AQ>2Mm=v2z$=e z{gB&i%eN-+ZXgC?SF}!hyxST(bhm9>-Jv^LK6^AS$hKcrBx7CQotkIO1xPW=g-lB| zF&lBkxcN!9rA+@3S*Kqs^EcF^AABFlZ`3d3KcSAug>3+!Iq`y%lKieV zIO02u{_LTS{5Jla()P{2ttkblkXXR?q?6$vaScGfMos})lH%Uk_~4s)Va6VDX>sqg zipDbDxBD&`FVX8TVq$pF&cGf)##tr%UIO9WX-EoK7FXu1DZXw>UP155Rk=Oh$u*5r z@5(9{v*pRIAG?^G#Na;cp7C4^w)<@BOSn4$$au2_K);&WT8R-1n_I8@@)Kw_H<~_kv({uv78sK< zjMi&?A``dk`XrW;#H40PlFyygcUvR2`mBwX&nKclW3yv zHZy!%&!sI>>{#5@?_kWXx|?rLo}?qK8f14Il|j%4y{&f7%yv=IiI0uS%8`z^L|K?2 zTEe1F_~{*DRaTH;niuAJu6)_%ATP5Is;5nd%1=V9s znt%nWhjqoccb4iQhM8qiCgx}JMueGAt#=&&N+M`P%WQI*3+Mn1?yc%E`VZySJCFT! z>>29O2pQaKK3TV6Lc1Muu~ZMnd!BY~?J`EGCG2)_;~a}b8P4uw(dB!9_UyBND9eCq zN3C!O!Ky$kBeEM!D$B{OTctH=mYqjzCJj0=7dme%H^;jx$Q;7qUh!;U=M+8_3ZY3rq-$R&f1-6%gtek&DJrwlNj}6 zQZe6rXgD`URbIt2m@3RIZgnKKrp*6)X_<%i=>N&V8F#ZkPXO9xldo4~A^?&9Az1|` z0Co%jegsh6b|9)@rM62W_NAwmegY8Nh8%eNrXd)MJYfJt1G%@(W6iu}WTzEdt&dil z{=P{9B`-$XTkG=UH1~dDr4EI4B`RAVw%QABb5R#cQboCX)q?sKGm4JlDXM}q zE$F^woO?+O_W2r7RA-WS$Bo5NuE_)@(Il2lf|8E=*6_h#AGTTz%)88sLzi zkcKHYnu|om=ztaHZ5C0Oiq2(amPDV%C$rlQBDW1wH=M$J+X} zef?P%F|vG-%iSVc!$L2li=hB2zcq&tdrqxk@by5}8+MGjnjV=X*VS7vDKMWK&-NGSecBB~rQNpcgoMg50X4Zl56M+H>+Dn)k6f)d`KV@B zeZ`;J?s#X50FmNk#rkOoRG0Y3r}zGOAW~zAL6}(0yEYQW8K_`*11HC5{Tj`weL4^2 zgtrZ0W14Zw#2=UGq?bGgPdDs`s}((xp?(z)EjY$;88E7vjq%7tT*M}Q zn8Y{Rah@ukgxJ5VhjLm$Yf#4_C5QJ`dILV4^c?Z(QdhE4+(p#vo-C1xBbm~Z+4?!f zn~SK1#4Q|>FOGf6D|sDC$MlAMb9&Umi7m|Srse0N6kivquur?x96ZCRdNqoT{YzKN zrRfyIwY#f1F&xQ)g-Cv*+7$sxalH zC`=2pY4mia!i*Y{@Y~S49QWvK1n*keUs+aA9NcN%G5P7jkVi9G6@R5LU<33=Z~pXO zhhg>KEKbrN3IH~~{nxwop{zC)I(OniPMbCOtqPK)7jpn10HRc6pdcUmUp|F!GV{q& zOQe>bo{I&{s@^h*1#0ulbR}{9rYQ*|-$`(g{~7m@iP;m-g@`7WCq+`^NhW|(M94DH zY)S^Hm^Lm-k4YS%DOs_#eXvYNpbQ+a%0e`?#wpBWKqhH?D5&JfL=12kVn=kMs)tc| zGYaDzn?OdBMH1_Yynbq$nj_5PaSYx;Q@_sYeghBwu<-kw6v#h@J6Ek4-c$95Mc9eU zpz??j=#8Bp>Q3)^_P`$ZzI@R-HrhQw+=ZJJU)Yw~st2m_qy9?Rhj@ciIq2O^e@Omg zDpScNMe4d_bn}GIVCgNQhkda^4Hyn1ISVmNZOQ+i*ZHe zBp!C>Nls=xocx|*;&|h<$iuTMJk(RFY|S>G^I67AN9#LW1lL{LQiFW6O59>FhE{83 z6KOYR)rm40i0&xFtY+~p#SW))XR_DC@Ko`Z%4@2vyR+&?e#KXNKG*WozAkuHe3#f? z+saf%)&+@7VlJHDlStE~bGFVwB3xG=4tn_1hDZTpcc3c2T3nh#0|5 zg-hEnkKiudrRek|^9j~>bhpm?*+pDlc#*4^4puw&LJC5Fn|!IjUZUm`(#<@MYO$#R zyM`aq1*1cD%TW6Gz)b1;-(|Q0V z`N<`gdc{O}*G5wE*&|CtV-+z;5|P-%Ep3XX8%;-T6O&b?MC;?vg%N9=swQ4`_pWGh zStIc^&rD10+@6kS%@Ch2Xx2yLDfcBhj~(l^cMIOP)7|jjh%pA?-ra*j8CtYHFBnKd~Q_%>_hQ_!CT@d{4v`psmD)rZB1^IS&+;cnLfa1n8 zu681{Ii0-ut{ZYs1EU+QX<&FX&dgs`pR32~;G1(-xe2$P ztK(H2WL|e~I2$r~vEjAd)|DD~xOoDIX#I4lmb_5TUG#?rXJgX~q#XO>!Bwgg8}7{Y zzAP@=Q+_s1O@L*$EMrFqcW_)5g&e8KKh8<&f)jZqVN36N6x}m7V3^(o| zAb~8&5Ll7ybrANey5nGd5FoEoJEi4Hz zDp4u!QGywcc3mg4R^fS;v+C7*pRr!O?q_D7zgDN$<#xHC6|vg4)}QDh--u==WhT8= zkmjtM$-I0JtH3oZEF8S(n?RE~YouB^mK-xonj!OBWY!&t9Pj zB)@sL54SLrYLYL_J9djp7ndY;t7Bd?gcmco~(z?A5_Fm}t^8V6y^C8ToZ7MeIt_;FZ_BBB}r&f}$%t+1N z2fA}+dALW{Bp>&%Kb`v766fk<_PN=~k#%4j%E^^!n)%ZUhl{g?t#qB6Q+v&ycPe*6 zw`Yp{jeon9{tD3FO$gcV#18ycnc;t@pe=T>KMG`{t=H?rc%@8X=3*=^fH8my1_FoV z2qYpy+ls9yVFVZ`;2sx_fWB5(8TFi1pS1~mXMmY|Em@)gviD&FvQwE9V4BstyQ-j6 z!^|-Ex{pO=I18e!`_HuWZs-y`aE;&??$PWTX<4xDsSUcc$|5LmuE-jbT|lnzlSfGf z>sPnp;8;t&u$KoddcPEXL3j&&5CP4&jieuxXda~Iv<`+kB`lNQ?~EI$Vaqp|P_eS! zDjx*MQvIn{V_N>CQB;A~9{iuS#rx)J1AsB^*6S5mBXfercxHe=Y!RU8n;oZEGFt#l z(Rv4BiaklM5gq5!BH6d?WZr8drgp^h9(fc+sl>4!nqJbr&8eSUgNp$H zv#n4r!B}5yo;la1wBod&){Kf5ZTHo|2#j=W6X)oI;iG!OJAC0KBD!20y^a?t!K}!I zp%)2zJH0)(skLgSb+Dg8iPmZOeptaaPnk`&?#^lXe1BB|kM9@9JfCvPB)cUQ zeIZFyMoRX=#qj2PMH@_%wxs*l9qqY^-)i^5*;I0c=O?3_30l<93pwCNS-DeBH*;Og zP`r_HmYl70#XcR9%!4bn+vDP(lYh$WV`>(Um9Qa>-HeSiC953AbMBmb2|YY+sp%~%pCmzC7iJw}-s_rKwDS$U`XU(FMdOLqPAIYw7st7%|(Fa_0|ihss_B(+a0Io8{QXNgv? zsQHIJ8!P*KS;Qtu+8o(rojQgewrLkh>@aZQk5$z_|NieB@9z;F`?o85*!=fVFi0)e zO#tEMwO*|mB32Xug-8+!fv-h2`P%-03ST=kAV@!WQu#^aCZ97ai5eWAvSh&2>=q{NFwN$opHk?#S-!q~PMi`_9IIRp zizv5R2umGty%&y^(}1HoYVn|QuCBxP+F;c>b?eUbl3O?J0;zzd=DT~UCGu%IS#@{m zPK;S^b9Yr-99y|CPp7S7l_u}oc)w+WuF5Vc(O4`;lUG6(70~a~7v}D=(Zc<0Jf8Jftd|*?UJ`2UM8>c_I>SKSAf#@P zM3=Xxu0}U%s5B}f7;*!#X%xwEdA(gTI;zX=@v`ZL<@Rbe`2L)ATVWDvGfL%ckq5!n zOON!3cE@nrvF0$*+Hfpi^$G_$t|O==N^@x)vyL0Z=hV#8ajx~dRoPG35BWl>t@LlVrBmCEO+vY>PqTR3KORnS+zS^EU49HoJMkg@ z4-^02MvTgD1H1Ttf!C=X#!on!&4FF7w+x7M-vYQ$6cGKYRZ0j5MPOn8i2|-jWm#&1 zzsWa1omtDQ&3XjjBgqbGo3*_Ee(I(tC3&Rj#?uXPD>}3`4b`?Ap?own;U@`WO+4WG z?ml;B5%sMYr#_zQa3;n%I=@o>vf4bVi(9KPD!t&rqj;t0ar~a%TZVC|QyJN;(FaLP zHTZo+Xf7AN|Eg~vJ2|5OX#NOQ3Ljh%geN$WuMg|z5F-+!KmpPx44>Eb<@EqcR5U6f zrF$3rgD#Y;?K5Dzmb%=Fyu|U>m#+l|WHPym$67|_SpQ6Gcu8~FNJHtUyk5-8wM!o1 zYiYWEKZ2>X%QmsD`zf8NoOw%PM5bDqkt#c4QR=wAyws&1jAGlx^)4W6kjc)JM> z% 0 { + n := len(buf) + if n > len(c.buf) { + n = len(c.buf) + } + for i, b := range buf[:n] { + c.buf[i] = internal.ReverseLUT[b] + } + cval = crc32.Update(cval, crc32.IEEETable, c.buf[:n]) + buf = buf[n:] + } + c.val = internal.ReverseUint32(cval) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go b/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go new file mode 100644 index 00000000..ddd32f50 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go @@ -0,0 +1,13 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build !gofuzz + +// This file exists to suppress fuzzing details from release builds. + +package bzip2 + +type fuzzReader struct{} + +func (*fuzzReader) updateChecksum(int64, uint32) {} diff --git a/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go b/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go new file mode 100644 index 00000000..54122351 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go @@ -0,0 +1,77 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build gofuzz + +// This file exists to export internal implementation details for fuzz testing. + +package bzip2 + +func ForwardBWT(buf []byte) (ptr int) { + var bwt burrowsWheelerTransform + return bwt.Encode(buf) +} + +func ReverseBWT(buf []byte, ptr int) { + var bwt burrowsWheelerTransform + bwt.Decode(buf, ptr) +} + +type fuzzReader struct { + Checksums Checksums +} + +// updateChecksum updates Checksums. +// +// If a valid pos is provided, it appends the (pos, val) pair to the slice. +// Otherwise, it will update the last record with the new value. +func (fr *fuzzReader) updateChecksum(pos int64, val uint32) { + if pos >= 0 { + fr.Checksums = append(fr.Checksums, Checksum{pos, val}) + } else { + fr.Checksums[len(fr.Checksums)-1].Value = val + } +} + +type Checksum struct { + Offset int64 // Bit offset of the checksum + Value uint32 // Checksum value +} + +type Checksums []Checksum + +// Apply overwrites all checksum fields in d with the ones in cs. +func (cs Checksums) Apply(d []byte) []byte { + d = append([]byte(nil), d...) + for _, c := range cs { + setU32(d, c.Offset, c.Value) + } + return d +} + +func setU32(d []byte, pos int64, val uint32) { + for i := uint(0); i < 32; i++ { + bpos := uint64(pos) + uint64(i) + d[bpos/8] &= ^byte(1 << (7 - bpos%8)) + d[bpos/8] |= byte(val>>(31-i)) << (7 - bpos%8) + } +} + +// Verify checks that all checksum fields in d matches those in cs. +func (cs Checksums) Verify(d []byte) bool { + for _, c := range cs { + if getU32(d, c.Offset) != c.Value { + return false + } + } + return true +} + +func getU32(d []byte, pos int64) (val uint32) { + for i := uint(0); i < 32; i++ { + bpos := uint64(pos) + uint64(i) + val |= (uint32(d[bpos/8] >> (7 - bpos%8))) << (31 - i) + } + return val +} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go new file mode 100644 index 00000000..cd4eee82 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go @@ -0,0 +1,28 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package sais implements a linear time suffix array algorithm. +package sais + +//go:generate go run sais_gen.go byte sais_byte.go +//go:generate go run sais_gen.go int sais_int.go + +// This package ports the C sais implementation by Yuta Mori. The ports are +// located in sais_byte.go and sais_int.go, which are identical to each other +// except for the types. Since Go does not support generics, we use generators to +// create the two files. +// +// References: +// https://sites.google.com/site/yuta256/sais +// https://www.researchgate.net/publication/221313676_Linear_Time_Suffix_Array_Construction_Using_D-Critical_Substrings +// https://www.researchgate.net/publication/224176324_Two_Efficient_Algorithms_for_Linear_Time_Suffix_Array_Construction + +// ComputeSA computes the suffix array of t and places the result in sa. +// Both t and sa must be the same length. +func ComputeSA(t []byte, sa []int) { + if len(sa) != len(t) { + panic("mismatching sizes") + } + computeSA_byte(t, sa, 0, len(t), 256) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go new file mode 100644 index 00000000..01b8529b --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go @@ -0,0 +1,661 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Code generated by sais_gen.go. DO NOT EDIT. + +// ==================================================== +// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// ==================================================== + +package sais + +func getCounts_byte(T []byte, C []int, n, k int) { + var i int + for i = 0; i < k; i++ { + C[i] = 0 + } + for i = 0; i < n; i++ { + C[T[i]]++ + } +} + +func getBuckets_byte(C, B []int, k int, end bool) { + var i, sum int + if end { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum + } + } else { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum - C[i] + } + } +} + +func sortLMS1_byte(T []byte, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + b-- + if int(T[j]) > c1 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS1_byte(T []byte, SA []int, n, m int) int { + var i, j, p, q, plen, qlen, name int + var c0, c1 int + var diff bool + + // Compact all the sorted substrings into the first m items of SA. + // 2*m must be not larger than n (provable). + for i = 0; SA[i] < 0; i++ { + SA[i] = ^SA[i] + } + if i < m { + for j, i = i, i+1; ; i++ { + if p = SA[i]; p < 0 { + SA[j] = ^p + j++ + SA[i] = 0 + if j == m { + break + } + } + } + } + + // Store the length of all substrings. + i = n - 1 + j = n - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + SA[m+((i+1)>>1)] = j - i + j = i + 1 + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + // Find the lexicographic names of all substrings. + name = 0 + qlen = 0 + for i, q = 0, n; i < m; i++ { + p = SA[i] + plen = SA[m+(p>>1)] + diff = true + if (plen == qlen) && ((q + plen) < n) { + for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { + } + if j == plen { + diff = false + } + } + if diff { + name++ + q = p + qlen = plen + } + SA[m+(p>>1)] = name + } + return name +} + +func sortLMS2_byte(T []byte, SA, C, B, D []int, n, k int) { + var b, i, j, t, d int + var c0, c1 int + + // Compute SAl. + getBuckets_byte(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + t = 1 + } else { + t = 0 + } + j += n + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i, d = 0, 0; i < n; i++ { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) < c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + for i = n - 1; 0 <= i; i-- { + if SA[i] > 0 { + if SA[i] < n { + SA[i] += n + for j = i - 1; SA[j] < n; j-- { + } + SA[j] -= n + i = j + } + } + } + + // Compute SAs. + getBuckets_byte(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i, d = n-1, d+1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) > c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + b-- + if t&1 > 0 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS2_byte(SA []int, n, m int) int { + var i, j, d, name int + + // Compact all the sorted LMS substrings into the first m items of SA. + name = 0 + for i = 0; SA[i] < 0; i++ { + j = ^SA[i] + if n <= j { + name += 1 + } + SA[i] = j + } + if i < m { + for d, i = i, i+1; ; i++ { + if j = SA[i]; j < 0 { + j = ^j + if n <= j { + name += 1 + } + SA[d] = j + d++ + SA[i] = 0 + if d == m { + break + } + } + } + } + if name < m { + // Store the lexicographic names. + for i, d = m-1, name+1; 0 <= i; i-- { + if j = SA[i]; n <= j { + j -= n + d-- + } + SA[m+(j>>1)] = d + } + } else { + // Unset flags. + for i = 0; i < m; i++ { + if j = SA[i]; n <= j { + j -= n + SA[i] = j + } + } + } + return name +} + +func induceSA_byte(T []byte, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + j = SA[i] + SA[i] = ^j + if j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + b-- + if (j == 0) || (int(T[j-1]) > c1) { + SA[b] = ^j + } else { + SA[b] = j + } + } else { + SA[i] = ^j + } + } +} + +func computeSA_byte(T []byte, SA []int, fs, n, k int) { + const ( + minBucketSize = 512 + sortLMS2Limit = 0x3fffffff + ) + + var C, B, D, RA []int + var bo int // Offset of B relative to SA + var b, i, j, m, p, q, name, newfs int + var c0, c1 int + var flags uint + + if k <= minBucketSize { + C = make([]int, k) + if k <= fs { + bo = n + fs - k + B = SA[bo:] + flags = 1 + } else { + B = make([]int, k) + flags = 3 + } + } else if k <= fs { + C = SA[n+fs-k:] + if k <= fs-k { + bo = n + fs - 2*k + B = SA[bo:] + flags = 0 + } else if k <= 4*minBucketSize { + B = make([]int, k) + flags = 2 + } else { + B = C + flags = 8 + } + } else { + C = make([]int, k) + B = C + flags = 4 | 8 + } + if n <= sortLMS2Limit && 2 <= (n/k) { + if flags&1 > 0 { + if 2*k <= fs-k { + flags |= 32 + } else { + flags |= 16 + } + } else if flags == 0 && 2*k <= (fs-2*k) { + flags |= 32 + } + } + + // Stage 1: Reduce the problem by at least 1/2. + // Sort all the LMS-substrings. + getCounts_byte(T, C, n, k) + getBuckets_byte(C, B, k, true) // Find ends of buckets + for i = 0; i < n; i++ { + SA[i] = 0 + } + b = -1 + i = n - 1 + j = n + m = 0 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + if b >= 0 { + SA[b] = j + } + B[c1]-- + b = B[c1] + j = i + m++ + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + if m > 1 { + if flags&(16|32) > 0 { + if flags&16 > 0 { + D = make([]int, 2*k) + } else { + D = SA[bo-2*k:] + } + B[T[j+1]]++ + for i, j = 0, 0; i < k; i++ { + j += C[i] + if B[i] != j { + SA[B[i]] += n + } + D[i] = 0 + D[i+k] = 0 + } + sortLMS2_byte(T, SA, C, B, D, n, k) + name = postProcLMS2_byte(SA, n, m) + } else { + sortLMS1_byte(T, SA, C, B, n, k) + name = postProcLMS1_byte(T, SA, n, m) + } + } else if m == 1 { + SA[b] = j + 1 + name = 1 + } else { + name = 0 + } + + // Stage 2: Solve the reduced problem. + // Recurse if names are not yet unique. + if name < m { + newfs = n + fs - 2*m + if flags&(1|4|8) == 0 { + if k+name <= newfs { + newfs -= k + } else { + flags |= 8 + } + } + RA = SA[m+newfs:] + for i, j = m+(n>>1)-1, m-1; m <= i; i-- { + if SA[i] != 0 { + RA[j] = SA[i] - 1 + j-- + } + } + computeSA_int(RA, SA, newfs, m, name) + + i = n - 1 + j = m - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + RA[j] = i + 1 + j-- + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + for i = 0; i < m; i++ { + SA[i] = RA[SA[i]] + } + if flags&4 > 0 { + B = make([]int, k) + C = B + } + if flags&2 > 0 { + B = make([]int, k) + } + } + + // Stage 3: Induce the result for the original problem. + if flags&8 > 0 { + getCounts_byte(T, C, n, k) + } + // Put all left-most S characters into their buckets. + if m > 1 { + getBuckets_byte(C, B, k, true) // Find ends of buckets + i = m - 1 + j = n + p = SA[m-1] + c1 = int(T[p]) + for { + c0 = c1 + q = B[c0] + for q < j { + j-- + SA[j] = 0 + } + for { + j-- + SA[j] = p + if i--; i < 0 { + break + } + p = SA[i] + if c1 = int(T[p]); c1 != c0 { + break + } + } + if i < 0 { + break + } + } + for j > 0 { + j-- + SA[j] = 0 + } + } + induceSA_byte(T, SA, C, B, n, k) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go new file mode 100644 index 00000000..26bf628e --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go @@ -0,0 +1,703 @@ +// Copyright 2017, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build ignore + +package main + +import ( + "bytes" + "go/format" + "io/ioutil" + "log" + "os" + "text/template" +) + +func main() { + if len(os.Args) != 3 { + log.Fatalf("Usage: %s GO_TYPE OUTPUT_FILE", os.Args[0]) + } + typ := os.Args[1] + path := os.Args[2] + + b := new(bytes.Buffer) + t := template.Must(template.New("source").Parse(source)) + if err := t.Execute(b, struct { + Type, GeneratedMessage string + }{typ, "// Code generated by sais_gen.go. DO NOT EDIT."}); err != nil { + log.Fatalf("Template.Execute error: %v", err) + } + out, err := format.Source(bytes.TrimSpace(b.Bytes())) + if err != nil { + log.Fatalf("format.Source error: %v", err) + } + if err := ioutil.WriteFile(path, out, 0644); err != nil { + log.Fatalf("ioutil.WriteFile error: %v", err) + } +} + +const source = ` +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +{{.GeneratedMessage}} + +// ==================================================== +// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// ==================================================== + +package sais + +func getCounts_{{.Type}}(T []{{.Type}}, C []int, n, k int) { + var i int + for i = 0; i < k; i++ { + C[i] = 0 + } + for i = 0; i < n; i++ { + C[T[i]]++ + } +} + +func getBuckets_{{.Type}}(C, B []int, k int, end bool) { + var i, sum int + if end { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum + } + } else { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum - C[i] + } + } +} + +func sortLMS1_{{.Type}}(T []{{.Type}}, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + b-- + if int(T[j]) > c1 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS1_{{.Type}}(T []{{.Type}}, SA []int, n, m int) int { + var i, j, p, q, plen, qlen, name int + var c0, c1 int + var diff bool + + // Compact all the sorted substrings into the first m items of SA. + // 2*m must be not larger than n (provable). + for i = 0; SA[i] < 0; i++ { + SA[i] = ^SA[i] + } + if i < m { + for j, i = i, i+1; ; i++ { + if p = SA[i]; p < 0 { + SA[j] = ^p + j++ + SA[i] = 0 + if j == m { + break + } + } + } + } + + // Store the length of all substrings. + i = n - 1 + j = n - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + SA[m+((i+1)>>1)] = j - i + j = i + 1 + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + // Find the lexicographic names of all substrings. + name = 0 + qlen = 0 + for i, q = 0, n; i < m; i++ { + p = SA[i] + plen = SA[m+(p>>1)] + diff = true + if (plen == qlen) && ((q + plen) < n) { + for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { + } + if j == plen { + diff = false + } + } + if diff { + name++ + q = p + qlen = plen + } + SA[m+(p>>1)] = name + } + return name +} + +func sortLMS2_{{.Type}}(T []{{.Type}}, SA, C, B, D []int, n, k int) { + var b, i, j, t, d int + var c0, c1 int + + // Compute SAl. + getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + t = 1 + } else { + t = 0 + } + j += n + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i, d = 0, 0; i < n; i++ { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) < c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + for i = n - 1; 0 <= i; i-- { + if SA[i] > 0 { + if SA[i] < n { + SA[i] += n + for j = i - 1; SA[j] < n; j-- { + } + SA[j] -= n + i = j + } + } + } + + // Compute SAs. + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i, d = n-1, d+1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) > c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + b-- + if t&1 > 0 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS2_{{.Type}}(SA []int, n, m int) int { + var i, j, d, name int + + // Compact all the sorted LMS substrings into the first m items of SA. + name = 0 + for i = 0; SA[i] < 0; i++ { + j = ^SA[i] + if n <= j { + name += 1 + } + SA[i] = j + } + if i < m { + for d, i = i, i+1; ; i++ { + if j = SA[i]; j < 0 { + j = ^j + if n <= j { + name += 1 + } + SA[d] = j + d++ + SA[i] = 0 + if d == m { + break + } + } + } + } + if name < m { + // Store the lexicographic names. + for i, d = m-1, name+1; 0 <= i; i-- { + if j = SA[i]; n <= j { + j -= n + d-- + } + SA[m+(j>>1)] = d + } + } else { + // Unset flags. + for i = 0; i < m; i++ { + if j = SA[i]; n <= j { + j -= n + SA[i] = j + } + } + } + return name +} + +func induceSA_{{.Type}}(T []{{.Type}}, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + j = SA[i] + SA[i] = ^j + if j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + b-- + if (j == 0) || (int(T[j-1]) > c1) { + SA[b] = ^j + } else { + SA[b] = j + } + } else { + SA[i] = ^j + } + } +} + +func computeSA_{{.Type}}(T []{{.Type}}, SA []int, fs, n, k int) { + const ( + minBucketSize = 512 + sortLMS2Limit = 0x3fffffff + ) + + var C, B, D, RA []int + var bo int // Offset of B relative to SA + var b, i, j, m, p, q, name, newfs int + var c0, c1 int + var flags uint + + if k <= minBucketSize { + C = make([]int, k) + if k <= fs { + bo = n + fs - k + B = SA[bo:] + flags = 1 + } else { + B = make([]int, k) + flags = 3 + } + } else if k <= fs { + C = SA[n+fs-k:] + if k <= fs-k { + bo = n + fs - 2*k + B = SA[bo:] + flags = 0 + } else if k <= 4*minBucketSize { + B = make([]int, k) + flags = 2 + } else { + B = C + flags = 8 + } + } else { + C = make([]int, k) + B = C + flags = 4 | 8 + } + if n <= sortLMS2Limit && 2 <= (n/k) { + if flags&1 > 0 { + if 2*k <= fs-k { + flags |= 32 + } else { + flags |= 16 + } + } else if flags == 0 && 2*k <= (fs-2*k) { + flags |= 32 + } + } + + // Stage 1: Reduce the problem by at least 1/2. + // Sort all the LMS-substrings. + getCounts_{{.Type}}(T, C, n, k) + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + for i = 0; i < n; i++ { + SA[i] = 0 + } + b = -1 + i = n - 1 + j = n + m = 0 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + if b >= 0 { + SA[b] = j + } + B[c1]-- + b = B[c1] + j = i + m++ + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + if m > 1 { + if flags&(16|32) > 0 { + if flags&16 > 0 { + D = make([]int, 2*k) + } else { + D = SA[bo-2*k:] + } + B[T[j+1]]++ + for i, j = 0, 0; i < k; i++ { + j += C[i] + if B[i] != j { + SA[B[i]] += n + } + D[i] = 0 + D[i+k] = 0 + } + sortLMS2_{{.Type}}(T, SA, C, B, D, n, k) + name = postProcLMS2_{{.Type}}(SA, n, m) + } else { + sortLMS1_{{.Type}}(T, SA, C, B, n, k) + name = postProcLMS1_{{.Type}}(T, SA, n, m) + } + } else if m == 1 { + SA[b] = j + 1 + name = 1 + } else { + name = 0 + } + + // Stage 2: Solve the reduced problem. + // Recurse if names are not yet unique. + if name < m { + newfs = n + fs - 2*m + if flags&(1|4|8) == 0 { + if k+name <= newfs { + newfs -= k + } else { + flags |= 8 + } + } + RA = SA[m+newfs:] + for i, j = m+(n>>1)-1, m-1; m <= i; i-- { + if SA[i] != 0 { + RA[j] = SA[i] - 1 + j-- + } + } + computeSA_int(RA, SA, newfs, m, name) + + i = n - 1 + j = m - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + RA[j] = i + 1 + j-- + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + for i = 0; i < m; i++ { + SA[i] = RA[SA[i]] + } + if flags&4 > 0 { + B = make([]int, k) + C = B + } + if flags&2 > 0 { + B = make([]int, k) + } + } + + // Stage 3: Induce the result for the original problem. + if flags&8 > 0 { + getCounts_{{.Type}}(T, C, n, k) + } + // Put all left-most S characters into their buckets. + if m > 1 { + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + i = m - 1 + j = n + p = SA[m-1] + c1 = int(T[p]) + for { + c0 = c1 + q = B[c0] + for q < j { + j-- + SA[j] = 0 + } + for { + j-- + SA[j] = p + if i--; i < 0 { + break + } + p = SA[i] + if c1 = int(T[p]); c1 != c0 { + break + } + } + if i < 0 { + break + } + } + for j > 0 { + j-- + SA[j] = 0 + } + } + induceSA_{{.Type}}(T, SA, C, B, n, k) +} +` diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go new file mode 100644 index 00000000..280682f0 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go @@ -0,0 +1,661 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Code generated by sais_gen.go. DO NOT EDIT. + +// ==================================================== +// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// ==================================================== + +package sais + +func getCounts_int(T []int, C []int, n, k int) { + var i int + for i = 0; i < k; i++ { + C[i] = 0 + } + for i = 0; i < n; i++ { + C[T[i]]++ + } +} + +func getBuckets_int(C, B []int, k int, end bool) { + var i, sum int + if end { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum + } + } else { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum - C[i] + } + } +} + +func sortLMS1_int(T []int, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + b-- + if int(T[j]) > c1 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS1_int(T []int, SA []int, n, m int) int { + var i, j, p, q, plen, qlen, name int + var c0, c1 int + var diff bool + + // Compact all the sorted substrings into the first m items of SA. + // 2*m must be not larger than n (provable). + for i = 0; SA[i] < 0; i++ { + SA[i] = ^SA[i] + } + if i < m { + for j, i = i, i+1; ; i++ { + if p = SA[i]; p < 0 { + SA[j] = ^p + j++ + SA[i] = 0 + if j == m { + break + } + } + } + } + + // Store the length of all substrings. + i = n - 1 + j = n - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + SA[m+((i+1)>>1)] = j - i + j = i + 1 + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + // Find the lexicographic names of all substrings. + name = 0 + qlen = 0 + for i, q = 0, n; i < m; i++ { + p = SA[i] + plen = SA[m+(p>>1)] + diff = true + if (plen == qlen) && ((q + plen) < n) { + for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { + } + if j == plen { + diff = false + } + } + if diff { + name++ + q = p + qlen = plen + } + SA[m+(p>>1)] = name + } + return name +} + +func sortLMS2_int(T []int, SA, C, B, D []int, n, k int) { + var b, i, j, t, d int + var c0, c1 int + + // Compute SAl. + getBuckets_int(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + t = 1 + } else { + t = 0 + } + j += n + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i, d = 0, 0; i < n; i++ { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) < c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + for i = n - 1; 0 <= i; i-- { + if SA[i] > 0 { + if SA[i] < n { + SA[i] += n + for j = i - 1; SA[j] < n; j-- { + } + SA[j] -= n + i = j + } + } + } + + // Compute SAs. + getBuckets_int(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i, d = n-1, d+1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) > c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + b-- + if t&1 > 0 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS2_int(SA []int, n, m int) int { + var i, j, d, name int + + // Compact all the sorted LMS substrings into the first m items of SA. + name = 0 + for i = 0; SA[i] < 0; i++ { + j = ^SA[i] + if n <= j { + name += 1 + } + SA[i] = j + } + if i < m { + for d, i = i, i+1; ; i++ { + if j = SA[i]; j < 0 { + j = ^j + if n <= j { + name += 1 + } + SA[d] = j + d++ + SA[i] = 0 + if d == m { + break + } + } + } + } + if name < m { + // Store the lexicographic names. + for i, d = m-1, name+1; 0 <= i; i-- { + if j = SA[i]; n <= j { + j -= n + d-- + } + SA[m+(j>>1)] = d + } + } else { + // Unset flags. + for i = 0; i < m; i++ { + if j = SA[i]; n <= j { + j -= n + SA[i] = j + } + } + } + return name +} + +func induceSA_int(T []int, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + j = SA[i] + SA[i] = ^j + if j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + b-- + if (j == 0) || (int(T[j-1]) > c1) { + SA[b] = ^j + } else { + SA[b] = j + } + } else { + SA[i] = ^j + } + } +} + +func computeSA_int(T []int, SA []int, fs, n, k int) { + const ( + minBucketSize = 512 + sortLMS2Limit = 0x3fffffff + ) + + var C, B, D, RA []int + var bo int // Offset of B relative to SA + var b, i, j, m, p, q, name, newfs int + var c0, c1 int + var flags uint + + if k <= minBucketSize { + C = make([]int, k) + if k <= fs { + bo = n + fs - k + B = SA[bo:] + flags = 1 + } else { + B = make([]int, k) + flags = 3 + } + } else if k <= fs { + C = SA[n+fs-k:] + if k <= fs-k { + bo = n + fs - 2*k + B = SA[bo:] + flags = 0 + } else if k <= 4*minBucketSize { + B = make([]int, k) + flags = 2 + } else { + B = C + flags = 8 + } + } else { + C = make([]int, k) + B = C + flags = 4 | 8 + } + if n <= sortLMS2Limit && 2 <= (n/k) { + if flags&1 > 0 { + if 2*k <= fs-k { + flags |= 32 + } else { + flags |= 16 + } + } else if flags == 0 && 2*k <= (fs-2*k) { + flags |= 32 + } + } + + // Stage 1: Reduce the problem by at least 1/2. + // Sort all the LMS-substrings. + getCounts_int(T, C, n, k) + getBuckets_int(C, B, k, true) // Find ends of buckets + for i = 0; i < n; i++ { + SA[i] = 0 + } + b = -1 + i = n - 1 + j = n + m = 0 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + if b >= 0 { + SA[b] = j + } + B[c1]-- + b = B[c1] + j = i + m++ + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + if m > 1 { + if flags&(16|32) > 0 { + if flags&16 > 0 { + D = make([]int, 2*k) + } else { + D = SA[bo-2*k:] + } + B[T[j+1]]++ + for i, j = 0, 0; i < k; i++ { + j += C[i] + if B[i] != j { + SA[B[i]] += n + } + D[i] = 0 + D[i+k] = 0 + } + sortLMS2_int(T, SA, C, B, D, n, k) + name = postProcLMS2_int(SA, n, m) + } else { + sortLMS1_int(T, SA, C, B, n, k) + name = postProcLMS1_int(T, SA, n, m) + } + } else if m == 1 { + SA[b] = j + 1 + name = 1 + } else { + name = 0 + } + + // Stage 2: Solve the reduced problem. + // Recurse if names are not yet unique. + if name < m { + newfs = n + fs - 2*m + if flags&(1|4|8) == 0 { + if k+name <= newfs { + newfs -= k + } else { + flags |= 8 + } + } + RA = SA[m+newfs:] + for i, j = m+(n>>1)-1, m-1; m <= i; i-- { + if SA[i] != 0 { + RA[j] = SA[i] - 1 + j-- + } + } + computeSA_int(RA, SA, newfs, m, name) + + i = n - 1 + j = m - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + RA[j] = i + 1 + j-- + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + for i = 0; i < m; i++ { + SA[i] = RA[SA[i]] + } + if flags&4 > 0 { + B = make([]int, k) + C = B + } + if flags&2 > 0 { + B = make([]int, k) + } + } + + // Stage 3: Induce the result for the original problem. + if flags&8 > 0 { + getCounts_int(T, C, n, k) + } + // Put all left-most S characters into their buckets. + if m > 1 { + getBuckets_int(C, B, k, true) // Find ends of buckets + i = m - 1 + j = n + p = SA[m-1] + c1 = int(T[p]) + for { + c0 = c1 + q = B[c0] + for q < j { + j-- + SA[j] = 0 + } + for { + j-- + SA[j] = p + if i--; i < 0 { + break + } + p = SA[i] + if c1 = int(T[p]); c1 != c0 { + break + } + } + if i < 0 { + break + } + } + for j > 0 { + j-- + SA[j] = 0 + } + } + induceSA_int(T, SA, C, B, n, k) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go b/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go new file mode 100644 index 00000000..5c71b343 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go @@ -0,0 +1,131 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import "github.com/dsnet/compress/internal/errors" + +// moveToFront implements both the MTF and RLE stages of bzip2 at the same time. +// Any runs of zeros in the encoded output will be replaced by a sequence of +// RUNA and RUNB symbols are encode the length of the run. +// +// The RLE encoding used can actually be encoded to and decoded from using +// normal two's complement arithmetic. The methodology for doing so is below. +// +// Assuming the following: +// num: The value being encoded by RLE encoding. +// run: A sequence of RUNA and RUNB symbols represented as a binary integer, +// where RUNA is the 0 bit, RUNB is the 1 bit, and least-significant RUN +// symbols are at the least-significant bit positions. +// cnt: The number of RUNA and RUNB symbols. +// +// Then the RLE encoding used by bzip2 has this mathematical property: +// num+1 == (1< len(mtf.dictBuf) { + panicf(errors.Internal, "alphabet too large") + } + copy(mtf.dictBuf[:], dict) + mtf.dictLen = len(dict) + mtf.blkSize = blkSize +} + +func (mtf *moveToFront) Encode(vals []byte) (syms []uint16) { + dict := mtf.dictBuf[:mtf.dictLen] + syms = mtf.syms[:0] + + if len(vals) > mtf.blkSize { + panicf(errors.Internal, "exceeded block size") + } + + var lastNum uint32 + for _, val := range vals { + // Normal move-to-front transform. + var idx uint8 // Reverse lookup idx in dict + for di, dv := range dict { + if dv == val { + idx = uint8(di) + break + } + } + copy(dict[1:], dict[:idx]) + dict[0] = val + + // Run-length encoding augmentation. + if idx == 0 { + lastNum++ + continue + } + if lastNum > 0 { + for rc := lastNum + 1; rc != 1; rc >>= 1 { + syms = append(syms, uint16(rc&1)) + } + lastNum = 0 + } + syms = append(syms, uint16(idx)+1) + } + if lastNum > 0 { + for rc := lastNum + 1; rc != 1; rc >>= 1 { + syms = append(syms, uint16(rc&1)) + } + } + mtf.syms = syms + return syms +} + +func (mtf *moveToFront) Decode(syms []uint16) (vals []byte) { + dict := mtf.dictBuf[:mtf.dictLen] + vals = mtf.vals[:0] + + var lastCnt uint + var lastRun uint32 + for _, sym := range syms { + // Run-length encoding augmentation. + if sym < 2 { + lastRun |= uint32(sym) << lastCnt + lastCnt++ + continue + } + if lastCnt > 0 { + cnt := int((1< mtf.blkSize || lastCnt > 24 { + panicf(errors.Corrupted, "run-length decoding exceeded block size") + } + for i := cnt; i > 0; i-- { + vals = append(vals, dict[0]) + } + lastCnt, lastRun = 0, 0 + } + + // Normal move-to-front transform. + val := dict[sym-1] // Forward lookup val in dict + copy(dict[1:], dict[:sym-1]) + dict[0] = val + + if len(vals) >= mtf.blkSize { + panicf(errors.Corrupted, "run-length decoding exceeded block size") + } + vals = append(vals, val) + } + if lastCnt > 0 { + cnt := int((1< mtf.blkSize || lastCnt > 24 { + panicf(errors.Corrupted, "run-length decoding exceeded block size") + } + for i := cnt; i > 0; i-- { + vals = append(vals, dict[0]) + } + } + mtf.vals = vals + return vals +} diff --git a/vendor/github.com/dsnet/compress/bzip2/prefix.go b/vendor/github.com/dsnet/compress/bzip2/prefix.go new file mode 100644 index 00000000..4847d809 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/prefix.go @@ -0,0 +1,374 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import ( + "io" + + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" + "github.com/dsnet/compress/internal/prefix" +) + +const ( + minNumTrees = 2 + maxNumTrees = 6 + + maxPrefixBits = 20 // Maximum bit-width of a prefix code + maxNumSyms = 256 + 2 // Maximum number of symbols in the alphabet + numBlockSyms = 50 // Number of bytes in a block +) + +// encSel and decSel are used to handle the prefix encoding for tree selectors. +// The prefix encoding is as follows: +// +// Code TreeIdx +// 0 <=> 0 +// 10 <=> 1 +// 110 <=> 2 +// 1110 <=> 3 +// 11110 <=> 4 +// 111110 <=> 5 +// 111111 <=> 6 Invalid tree index, so should fail +// +var encSel, decSel = func() (e prefix.Encoder, d prefix.Decoder) { + var selCodes [maxNumTrees + 1]prefix.PrefixCode + for i := range selCodes { + selCodes[i] = prefix.PrefixCode{Sym: uint32(i), Len: uint32(i + 1)} + } + selCodes[maxNumTrees] = prefix.PrefixCode{Sym: maxNumTrees, Len: maxNumTrees} + prefix.GeneratePrefixes(selCodes[:]) + e.Init(selCodes[:]) + d.Init(selCodes[:]) + return +}() + +type prefixReader struct{ prefix.Reader } + +func (pr *prefixReader) Init(r io.Reader) { + pr.Reader.Init(r, true) +} + +func (pr *prefixReader) ReadBitsBE64(nb uint) uint64 { + if nb <= 32 { + v := uint32(pr.ReadBits(nb)) + return uint64(internal.ReverseUint32N(v, nb)) + } + v0 := internal.ReverseUint32(uint32(pr.ReadBits(32))) + v1 := internal.ReverseUint32(uint32(pr.ReadBits(nb - 32))) + v := uint64(v0)<<32 | uint64(v1) + return v >> (64 - nb) +} + +func (pr *prefixReader) ReadPrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Decoder) { + for i, pc := range codes { + clen := int(pr.ReadBitsBE64(5)) + sum := 1 << maxPrefixBits + for sym := range pc { + for { + if clen < 1 || clen > maxPrefixBits { + panicf(errors.Corrupted, "invalid prefix bit-length: %d", clen) + } + + b, ok := pr.TryReadBits(1) + if !ok { + b = pr.ReadBits(1) + } + if b == 0 { + break + } + + b, ok = pr.TryReadBits(1) + if !ok { + b = pr.ReadBits(1) + } + clen -= int(b*2) - 1 // +1 or -1 + } + pc[sym] = prefix.PrefixCode{Sym: uint32(sym), Len: uint32(clen)} + sum -= (1 << maxPrefixBits) >> uint(clen) + } + + if sum == 0 { + // Fast path, but only handles complete trees. + if err := prefix.GeneratePrefixes(pc); err != nil { + errors.Panic(err) // Using complete trees; should never fail + } + } else { + // Slow path, but handles anything. + pc = handleDegenerateCodes(pc) // Never fails, but may fail later + codes[i] = pc + } + trees[i].Init(pc) + } +} + +type prefixWriter struct{ prefix.Writer } + +func (pw *prefixWriter) Init(w io.Writer) { + pw.Writer.Init(w, true) +} + +func (pw *prefixWriter) WriteBitsBE64(v uint64, nb uint) { + if nb <= 32 { + v := internal.ReverseUint32N(uint32(v), nb) + pw.WriteBits(uint(v), nb) + return + } + v <<= (64 - nb) + v0 := internal.ReverseUint32(uint32(v >> 32)) + v1 := internal.ReverseUint32(uint32(v)) + pw.WriteBits(uint(v0), 32) + pw.WriteBits(uint(v1), nb-32) + return +} + +func (pw *prefixWriter) WritePrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Encoder) { + for i, pc := range codes { + if err := prefix.GeneratePrefixes(pc); err != nil { + errors.Panic(err) // Using complete trees; should never fail + } + trees[i].Init(pc) + + clen := int(pc[0].Len) + pw.WriteBitsBE64(uint64(clen), 5) + for _, c := range pc { + for int(c.Len) < clen { + pw.WriteBits(3, 2) // 11 + clen-- + } + for int(c.Len) > clen { + pw.WriteBits(1, 2) // 10 + clen++ + } + pw.WriteBits(0, 1) + } + } +} + +// handleDegenerateCodes converts a degenerate tree into a canonical tree. +// +// For example, when the input is an under-subscribed tree: +// input: []PrefixCode{ +// {Sym: 0, Len: 3}, +// {Sym: 1, Len: 4}, +// {Sym: 2, Len: 3}, +// } +// output: []PrefixCode{ +// {Sym: 0, Len: 3, Val: 0}, // 000 +// {Sym: 1, Len: 4, Val: 2}, // 0010 +// {Sym: 2, Len: 3, Val: 4}, // 100 +// {Sym: 258, Len: 4, Val: 10}, // 1010 +// {Sym: 259, Len: 3, Val: 6}, // 110 +// {Sym: 260, Len: 1, Val: 1}, // 1 +// } +// +// For example, when the input is an over-subscribed tree: +// input: []PrefixCode{ +// {Sym: 0, Len: 1}, +// {Sym: 1, Len: 3}, +// {Sym: 2, Len: 4}, +// {Sym: 3, Len: 3}, +// {Sym: 4, Len: 2}, +// } +// output: []PrefixCode{ +// {Sym: 0, Len: 1, Val: 0}, // 0 +// {Sym: 1, Len: 3, Val: 3}, // 011 +// {Sym: 3, Len: 3, Val: 7}, // 111 +// {Sym: 4, Len: 2, Val: 1}, // 01 +// } +func handleDegenerateCodes(codes prefix.PrefixCodes) prefix.PrefixCodes { + // Since there is no formal definition for the BZip2 format, there is no + // specification that says that the code lengths must form a complete + // prefix tree (IE: it is neither over-subscribed nor under-subscribed). + // Thus, the original C implementation becomes the reference for how prefix + // decoding is done in these edge cases. Unfortunately, the C version does + // not error when an invalid tree is used, but rather allows decoding to + // continue and only errors if some bit pattern happens to cause an error. + // Thus, it is possible for an invalid tree to end up decoding an input + // "properly" so long as invalid bit patterns are not present. In order to + // replicate this non-specified behavior, we use a ported version of the + // C code to generate the codes as a valid canonical tree by substituting + // invalid nodes with invalid symbols. + // + // ==================================================== + // This program, "bzip2", the associated library "libbzip2", and all + // documentation, are copyright (C) 1996-2010 Julian R Seward. All + // rights reserved. + // + // Redistribution and use in source and binary forms, with or without + // modification, are permitted provided that the following conditions + // are met: + // + // 1. Redistributions of source code must retain the above copyright + // notice, this list of conditions and the following disclaimer. + // + // 2. The origin of this software must not be misrepresented; you must + // not claim that you wrote the original software. If you use this + // software in a product, an acknowledgment in the product + // documentation would be appreciated but is not required. + // + // 3. Altered source versions must be plainly marked as such, and must + // not be misrepresented as being the original software. + // + // 4. The name of the author may not be used to endorse or promote + // products derived from this software without specific prior written + // permission. + // + // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS + // OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + // + // Julian Seward, jseward@bzip.org + // bzip2/libbzip2 version 1.0.6 of 6 September 2010 + // ==================================================== + var ( + limits [maxPrefixBits + 2]int32 + bases [maxPrefixBits + 2]int32 + perms [maxNumSyms]int32 + + minLen = uint32(maxPrefixBits) + maxLen = uint32(0) + ) + + const ( + statusOkay = iota + statusInvalid + statusNeedBits + statusMaxBits + ) + + // createTables is the BZ2_hbCreateDecodeTables function from the C code. + createTables := func(codes []prefix.PrefixCode) { + for _, c := range codes { + if c.Len > maxLen { + maxLen = c.Len + } + if c.Len < minLen { + minLen = c.Len + } + } + + var pp int + for i := minLen; i <= maxLen; i++ { + for j, c := range codes { + if c.Len == i { + perms[pp] = int32(j) + pp++ + } + } + } + + var vec int32 + for _, c := range codes { + bases[c.Len+1]++ + } + for i := 1; i < len(bases); i++ { + bases[i] += bases[i-1] + } + for i := minLen; i <= maxLen; i++ { + vec += bases[i+1] - bases[i] + limits[i] = vec - 1 + vec <<= 1 + } + for i := minLen + 1; i <= maxLen; i++ { + bases[i] = ((limits[i-1] + 1) << 1) - bases[i] + } + } + + // getSymbol is the GET_MTF_VAL macro from the C code. + getSymbol := func(c prefix.PrefixCode) (uint32, int) { + v := internal.ReverseUint32(c.Val) + n := c.Len + + zn := minLen + if zn > n { + return 0, statusNeedBits + } + zvec := int32(v >> (32 - zn)) + v <<= zn + for { + if zn > maxLen { + return 0, statusMaxBits + } + if zvec <= limits[zn] { + break + } + zn++ + if zn > n { + return 0, statusNeedBits + } + zvec = (zvec << 1) | int32(v>>31) + v <<= 1 + } + if zvec-bases[zn] < 0 || zvec-bases[zn] >= maxNumSyms { + return 0, statusInvalid + } + return uint32(perms[zvec-bases[zn]]), statusOkay + } + + // Step 1: Create the prefix trees using the C algorithm. + createTables(codes) + + // Step 2: Starting with the shortest bit pattern, explore the whole tree. + // If tree is under-subscribed, the worst-case runtime is O(1< 0 { + codes = append(codes, c) + } + } + return codes +} diff --git a/vendor/github.com/dsnet/compress/bzip2/reader.go b/vendor/github.com/dsnet/compress/bzip2/reader.go new file mode 100644 index 00000000..86d3f718 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/reader.go @@ -0,0 +1,274 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import ( + "io" + + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" + "github.com/dsnet/compress/internal/prefix" +) + +type Reader struct { + InputOffset int64 // Total number of bytes read from underlying io.Reader + OutputOffset int64 // Total number of bytes emitted from Read + + rd prefixReader + err error + level int // The current compression level + rdHdrFtr int // Number of times we read the stream header and footer + blkCRC uint32 // CRC-32 IEEE of each block (as stored) + endCRC uint32 // Checksum of all blocks using bzip2's custom method + + crc crc + mtf moveToFront + bwt burrowsWheelerTransform + rle runLengthEncoding + + // These fields are allocated with Reader and re-used later. + treeSels []uint8 + codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode + codes1D [maxNumTrees]prefix.PrefixCodes + trees1D [maxNumTrees]prefix.Decoder + syms []uint16 + + fuzzReader // Exported functionality when fuzz testing +} + +type ReaderConfig struct { + _ struct{} // Blank field to prevent unkeyed struct literals +} + +func NewReader(r io.Reader, conf *ReaderConfig) (*Reader, error) { + zr := new(Reader) + zr.Reset(r) + return zr, nil +} + +func (zr *Reader) Reset(r io.Reader) error { + *zr = Reader{ + rd: zr.rd, + + mtf: zr.mtf, + bwt: zr.bwt, + rle: zr.rle, + + treeSels: zr.treeSels, + trees1D: zr.trees1D, + syms: zr.syms, + } + zr.rd.Init(r) + return nil +} + +func (zr *Reader) Read(buf []byte) (int, error) { + for { + cnt, err := zr.rle.Read(buf) + if err != rleDone && zr.err == nil { + zr.err = err + } + if cnt > 0 { + zr.crc.update(buf[:cnt]) + zr.OutputOffset += int64(cnt) + return cnt, nil + } + if zr.err != nil || len(buf) == 0 { + return 0, zr.err + } + + // Read the next chunk. + zr.rd.Offset = zr.InputOffset + func() { + defer errors.Recover(&zr.err) + if zr.rdHdrFtr%2 == 0 { + // Check if we are already at EOF. + if err := zr.rd.PullBits(1); err != nil { + if err == io.ErrUnexpectedEOF && zr.rdHdrFtr > 0 { + err = io.EOF // EOF is okay if we read at least one stream + } + errors.Panic(err) + } + + // Read stream header. + if zr.rd.ReadBitsBE64(16) != hdrMagic { + panicf(errors.Corrupted, "invalid stream magic") + } + if ver := zr.rd.ReadBitsBE64(8); ver != 'h' { + if ver == '0' { + panicf(errors.Deprecated, "bzip1 format is not supported") + } + panicf(errors.Corrupted, "invalid version: %q", ver) + } + lvl := int(zr.rd.ReadBitsBE64(8)) - '0' + if lvl < BestSpeed || lvl > BestCompression { + panicf(errors.Corrupted, "invalid block size: %d", lvl*blockSize) + } + zr.level = lvl + zr.rdHdrFtr++ + } else { + // Check and update the CRC. + if internal.GoFuzz { + zr.updateChecksum(-1, zr.crc.val) // Update with value + zr.blkCRC = zr.crc.val // Suppress CRC failures + } + if zr.blkCRC != zr.crc.val { + panicf(errors.Corrupted, "mismatching block checksum") + } + zr.endCRC = (zr.endCRC<<1 | zr.endCRC>>31) ^ zr.blkCRC + } + buf := zr.decodeBlock() + zr.rle.Init(buf) + }() + if zr.InputOffset, err = zr.rd.Flush(); zr.err == nil { + zr.err = err + } + if zr.err != nil { + zr.err = errWrap(zr.err, errors.Corrupted) + return 0, zr.err + } + } +} + +func (zr *Reader) Close() error { + if zr.err == io.EOF || zr.err == errClosed { + zr.rle.Init(nil) // Make sure future reads fail + zr.err = errClosed + return nil + } + return zr.err // Return the persistent error +} + +func (zr *Reader) decodeBlock() []byte { + if magic := zr.rd.ReadBitsBE64(48); magic != blkMagic { + if magic == endMagic { + endCRC := uint32(zr.rd.ReadBitsBE64(32)) + if internal.GoFuzz { + zr.updateChecksum(zr.rd.BitsRead()-32, zr.endCRC) + endCRC = zr.endCRC // Suppress CRC failures + } + if zr.endCRC != endCRC { + panicf(errors.Corrupted, "mismatching stream checksum") + } + zr.endCRC = 0 + zr.rd.ReadPads() + zr.rdHdrFtr++ + return nil + } + panicf(errors.Corrupted, "invalid block or footer magic") + } + + zr.crc.val = 0 + zr.blkCRC = uint32(zr.rd.ReadBitsBE64(32)) + if internal.GoFuzz { + zr.updateChecksum(zr.rd.BitsRead()-32, 0) // Record offset only + } + if zr.rd.ReadBitsBE64(1) != 0 { + panicf(errors.Deprecated, "block randomization is not supported") + } + + // Read BWT related fields. + ptr := int(zr.rd.ReadBitsBE64(24)) // BWT origin pointer + + // Read MTF related fields. + var dictArr [256]uint8 + dict := dictArr[:0] + bmapHi := uint16(zr.rd.ReadBits(16)) + for i := 0; i < 256; i, bmapHi = i+16, bmapHi>>1 { + if bmapHi&1 > 0 { + bmapLo := uint16(zr.rd.ReadBits(16)) + for j := 0; j < 16; j, bmapLo = j+1, bmapLo>>1 { + if bmapLo&1 > 0 { + dict = append(dict, uint8(i+j)) + } + } + } + } + + // Step 1: Prefix encoding. + syms := zr.decodePrefix(len(dict)) + + // Step 2: Move-to-front transform and run-length encoding. + zr.mtf.Init(dict, zr.level*blockSize) + buf := zr.mtf.Decode(syms) + + // Step 3: Burrows-Wheeler transformation. + if ptr >= len(buf) { + panicf(errors.Corrupted, "origin pointer (0x%06x) exceeds block size: %d", ptr, len(buf)) + } + zr.bwt.Decode(buf, ptr) + + return buf +} + +func (zr *Reader) decodePrefix(numSyms int) (syms []uint16) { + numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOF symbols + if numSyms < 3 { + panicf(errors.Corrupted, "not enough prefix symbols: %d", numSyms) + } + + // Read information about the trees and tree selectors. + var mtf internal.MoveToFront + numTrees := int(zr.rd.ReadBitsBE64(3)) + if numTrees < minNumTrees || numTrees > maxNumTrees { + panicf(errors.Corrupted, "invalid number of prefix trees: %d", numTrees) + } + numSels := int(zr.rd.ReadBitsBE64(15)) + if cap(zr.treeSels) < numSels { + zr.treeSels = make([]uint8, numSels) + } + treeSels := zr.treeSels[:numSels] + for i := range treeSels { + sym, ok := zr.rd.TryReadSymbol(&decSel) + if !ok { + sym = zr.rd.ReadSymbol(&decSel) + } + if int(sym) >= numTrees { + panicf(errors.Corrupted, "invalid prefix tree selector: %d", sym) + } + treeSels[i] = uint8(sym) + } + mtf.Decode(treeSels) + zr.treeSels = treeSels + + // Initialize prefix codes. + for i := range zr.codes2D[:numTrees] { + zr.codes1D[i] = zr.codes2D[i][:numSyms] + } + zr.rd.ReadPrefixCodes(zr.codes1D[:numTrees], zr.trees1D[:numTrees]) + + // Read prefix encoded symbols of compressed data. + var tree *prefix.Decoder + var blkLen, selIdx int + syms = zr.syms[:0] + for { + if blkLen == 0 { + blkLen = numBlockSyms + if selIdx >= len(treeSels) { + panicf(errors.Corrupted, "not enough prefix tree selectors") + } + tree = &zr.trees1D[treeSels[selIdx]] + selIdx++ + } + blkLen-- + sym, ok := zr.rd.TryReadSymbol(tree) + if !ok { + sym = zr.rd.ReadSymbol(tree) + } + + if int(sym) == numSyms-1 { + break // EOF marker + } + if int(sym) >= numSyms { + panicf(errors.Corrupted, "invalid prefix symbol: %d", sym) + } + if len(syms) >= zr.level*blockSize { + panicf(errors.Corrupted, "number of prefix symbols exceeds block size") + } + syms = append(syms, uint16(sym)) + } + zr.syms = syms + return syms +} diff --git a/vendor/github.com/dsnet/compress/bzip2/rle1.go b/vendor/github.com/dsnet/compress/bzip2/rle1.go new file mode 100644 index 00000000..1d789f65 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/rle1.go @@ -0,0 +1,101 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import "github.com/dsnet/compress/internal/errors" + +// rleDone is a special "error" to indicate that the RLE stage is done. +var rleDone = errorf(errors.Unknown, "RLE1 stage is completed") + +// runLengthEncoding implements the first RLE stage of bzip2. Every sequence +// of 4..255 duplicated bytes is replaced by only the first 4 bytes, and a +// single byte representing the repeat length. Similar to the C bzip2 +// implementation, the encoder will always terminate repeat sequences with a +// count (even if it is the end of the buffer), and it will also never produce +// run lengths of 256..259. The decoder can handle the latter case. +// +// For example, if the input was: +// input: "AAAAAAABBBBCCCD" +// +// Then the output will be: +// output: "AAAA\x03BBBB\x00CCCD" +type runLengthEncoding struct { + buf []byte + idx int + lastVal byte + lastCnt int +} + +func (rle *runLengthEncoding) Init(buf []byte) { + *rle = runLengthEncoding{buf: buf} +} + +func (rle *runLengthEncoding) Write(buf []byte) (int, error) { + for i, b := range buf { + if rle.lastVal != b { + rle.lastCnt = 0 + } + rle.lastCnt++ + switch { + case rle.lastCnt < 4: + if rle.idx >= len(rle.buf) { + return i, rleDone + } + rle.buf[rle.idx] = b + rle.idx++ + case rle.lastCnt == 4: + if rle.idx+1 >= len(rle.buf) { + return i, rleDone + } + rle.buf[rle.idx] = b + rle.idx++ + rle.buf[rle.idx] = 0 + rle.idx++ + case rle.lastCnt < 256: + rle.buf[rle.idx-1]++ + default: + if rle.idx >= len(rle.buf) { + return i, rleDone + } + rle.lastCnt = 1 + rle.buf[rle.idx] = b + rle.idx++ + } + rle.lastVal = b + } + return len(buf), nil +} + +func (rle *runLengthEncoding) Read(buf []byte) (int, error) { + for i := range buf { + switch { + case rle.lastCnt == -4: + if rle.idx >= len(rle.buf) { + return i, errorf(errors.Corrupted, "missing terminating run-length repeater") + } + rle.lastCnt = int(rle.buf[rle.idx]) + rle.idx++ + if rle.lastCnt > 0 { + break // Break the switch + } + fallthrough // Count was zero, continue the work + case rle.lastCnt <= 0: + if rle.idx >= len(rle.buf) { + return i, rleDone + } + b := rle.buf[rle.idx] + rle.idx++ + if b != rle.lastVal { + rle.lastCnt = 0 + rle.lastVal = b + } + } + buf[i] = rle.lastVal + rle.lastCnt-- + } + return len(buf), nil +} + +func (rle *runLengthEncoding) Bytes() []byte { return rle.buf[:rle.idx] } diff --git a/vendor/github.com/dsnet/compress/bzip2/writer.go b/vendor/github.com/dsnet/compress/bzip2/writer.go new file mode 100644 index 00000000..5c1a4c66 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/writer.go @@ -0,0 +1,307 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import ( + "io" + + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" + "github.com/dsnet/compress/internal/prefix" +) + +type Writer struct { + InputOffset int64 // Total number of bytes issued to Write + OutputOffset int64 // Total number of bytes written to underlying io.Writer + + wr prefixWriter + err error + level int // The current compression level + wrHdr bool // Have we written the stream header? + blkCRC uint32 // CRC-32 IEEE of each block + endCRC uint32 // Checksum of all blocks using bzip2's custom method + + crc crc + rle runLengthEncoding + bwt burrowsWheelerTransform + mtf moveToFront + + // These fields are allocated with Writer and re-used later. + buf []byte + treeSels []uint8 + treeSelsMTF []uint8 + codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode + codes1D [maxNumTrees]prefix.PrefixCodes + trees1D [maxNumTrees]prefix.Encoder +} + +type WriterConfig struct { + Level int + + _ struct{} // Blank field to prevent unkeyed struct literals +} + +func NewWriter(w io.Writer, conf *WriterConfig) (*Writer, error) { + var lvl int + if conf != nil { + lvl = conf.Level + } + if lvl == 0 { + lvl = DefaultCompression + } + if lvl < BestSpeed || lvl > BestCompression { + return nil, errorf(errors.Invalid, "compression level: %d", lvl) + } + zw := new(Writer) + zw.level = lvl + zw.Reset(w) + return zw, nil +} + +func (zw *Writer) Reset(w io.Writer) error { + *zw = Writer{ + wr: zw.wr, + level: zw.level, + + rle: zw.rle, + bwt: zw.bwt, + mtf: zw.mtf, + + buf: zw.buf, + treeSels: zw.treeSels, + treeSelsMTF: zw.treeSelsMTF, + trees1D: zw.trees1D, + } + zw.wr.Init(w) + if len(zw.buf) != zw.level*blockSize { + zw.buf = make([]byte, zw.level*blockSize) + } + zw.rle.Init(zw.buf) + return nil +} + +func (zw *Writer) Write(buf []byte) (int, error) { + if zw.err != nil { + return 0, zw.err + } + + cnt := len(buf) + for { + wrCnt, err := zw.rle.Write(buf) + if err != rleDone && zw.err == nil { + zw.err = err + } + zw.crc.update(buf[:wrCnt]) + buf = buf[wrCnt:] + if len(buf) == 0 { + zw.InputOffset += int64(cnt) + return cnt, nil + } + if zw.err = zw.flush(); zw.err != nil { + return 0, zw.err + } + } +} + +func (zw *Writer) flush() error { + vals := zw.rle.Bytes() + if len(vals) == 0 { + return nil + } + zw.wr.Offset = zw.OutputOffset + func() { + defer errors.Recover(&zw.err) + if !zw.wrHdr { + // Write stream header. + zw.wr.WriteBitsBE64(hdrMagic, 16) + zw.wr.WriteBitsBE64('h', 8) + zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) + zw.wrHdr = true + } + zw.encodeBlock(vals) + }() + var err error + if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { + zw.err = err + } + if zw.err != nil { + zw.err = errWrap(zw.err, errors.Internal) + return zw.err + } + zw.endCRC = (zw.endCRC<<1 | zw.endCRC>>31) ^ zw.blkCRC + zw.blkCRC = 0 + zw.rle.Init(zw.buf) + return nil +} + +func (zw *Writer) Close() error { + if zw.err == errClosed { + return nil + } + + // Flush RLE buffer if there is left-over data. + if zw.err = zw.flush(); zw.err != nil { + return zw.err + } + + // Write stream footer. + zw.wr.Offset = zw.OutputOffset + func() { + defer errors.Recover(&zw.err) + if !zw.wrHdr { + // Write stream header. + zw.wr.WriteBitsBE64(hdrMagic, 16) + zw.wr.WriteBitsBE64('h', 8) + zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) + zw.wrHdr = true + } + zw.wr.WriteBitsBE64(endMagic, 48) + zw.wr.WriteBitsBE64(uint64(zw.endCRC), 32) + zw.wr.WritePads(0) + }() + var err error + if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { + zw.err = err + } + if zw.err != nil { + zw.err = errWrap(zw.err, errors.Internal) + return zw.err + } + + zw.err = errClosed + return nil +} + +func (zw *Writer) encodeBlock(buf []byte) { + zw.blkCRC = zw.crc.val + zw.wr.WriteBitsBE64(blkMagic, 48) + zw.wr.WriteBitsBE64(uint64(zw.blkCRC), 32) + zw.wr.WriteBitsBE64(0, 1) + zw.crc.val = 0 + + // Step 1: Burrows-Wheeler transformation. + ptr := zw.bwt.Encode(buf) + zw.wr.WriteBitsBE64(uint64(ptr), 24) + + // Step 2: Move-to-front transform and run-length encoding. + var dictMap [256]bool + for _, c := range buf { + dictMap[c] = true + } + + var dictArr [256]uint8 + var bmapLo [16]uint16 + dict := dictArr[:0] + bmapHi := uint16(0) + for i, b := range dictMap { + if b { + c := uint8(i) + dict = append(dict, c) + bmapHi |= 1 << (c >> 4) + bmapLo[c>>4] |= 1 << (c & 0xf) + } + } + + zw.wr.WriteBits(uint(bmapHi), 16) + for _, m := range bmapLo { + if m > 0 { + zw.wr.WriteBits(uint(m), 16) + } + } + + zw.mtf.Init(dict, len(buf)) + syms := zw.mtf.Encode(buf) + + // Step 3: Prefix encoding. + zw.encodePrefix(syms, len(dict)) +} + +func (zw *Writer) encodePrefix(syms []uint16, numSyms int) { + numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOB symbols + if numSyms < 3 { + panicf(errors.Internal, "unable to encode EOB marker") + } + syms = append(syms, uint16(numSyms-1)) // EOB marker + + // Compute number of prefix trees needed. + numTrees := maxNumTrees + for i, lim := range []int{200, 600, 1200, 2400} { + if len(syms) < lim { + numTrees = minNumTrees + i + break + } + } + + // Compute number of block selectors. + numSels := (len(syms) + numBlockSyms - 1) / numBlockSyms + if cap(zw.treeSels) < numSels { + zw.treeSels = make([]uint8, numSels) + } + treeSels := zw.treeSels[:numSels] + for i := range treeSels { + treeSels[i] = uint8(i % numTrees) + } + + // Initialize prefix codes. + for i := range zw.codes2D[:numTrees] { + pc := zw.codes2D[i][:numSyms] + for j := range pc { + pc[j] = prefix.PrefixCode{Sym: uint32(j)} + } + zw.codes1D[i] = pc + } + + // First cut at assigning prefix trees to each group. + var codes prefix.PrefixCodes + var blkLen, selIdx int + for _, sym := range syms { + if blkLen == 0 { + blkLen = numBlockSyms + codes = zw.codes2D[treeSels[selIdx]][:numSyms] + selIdx++ + } + blkLen-- + codes[sym].Cnt++ + } + + // TODO(dsnet): Use K-means to cluster groups to each prefix tree. + + // Generate lengths and prefixes based on symbol frequencies. + for i := range zw.trees1D[:numTrees] { + pc := prefix.PrefixCodes(zw.codes2D[i][:numSyms]) + pc.SortByCount() + if err := prefix.GenerateLengths(pc, maxPrefixBits); err != nil { + errors.Panic(err) + } + pc.SortBySymbol() + } + + // Write out information about the trees and tree selectors. + var mtf internal.MoveToFront + zw.wr.WriteBitsBE64(uint64(numTrees), 3) + zw.wr.WriteBitsBE64(uint64(numSels), 15) + zw.treeSelsMTF = append(zw.treeSelsMTF[:0], treeSels...) + mtf.Encode(zw.treeSelsMTF) + for _, sym := range zw.treeSelsMTF { + zw.wr.WriteSymbol(uint(sym), &encSel) + } + zw.wr.WritePrefixCodes(zw.codes1D[:numTrees], zw.trees1D[:numTrees]) + + // Write out prefix encoded symbols of compressed data. + var tree *prefix.Encoder + blkLen, selIdx = 0, 0 + for _, sym := range syms { + if blkLen == 0 { + blkLen = numBlockSyms + tree = &zw.trees1D[treeSels[selIdx]] + selIdx++ + } + blkLen-- + ok := zw.wr.TryWriteSymbol(uint(sym), tree) + if !ok { + zw.wr.WriteSymbol(uint(sym), tree) + } + } +} diff --git a/vendor/github.com/dsnet/compress/go.mod b/vendor/github.com/dsnet/compress/go.mod new file mode 100644 index 00000000..7a0bc001 --- /dev/null +++ b/vendor/github.com/dsnet/compress/go.mod @@ -0,0 +1,10 @@ +module github.com/dsnet/compress + +go 1.9 + +require ( + github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 + github.com/klauspost/compress v1.4.1 + github.com/klauspost/cpuid v1.2.0 // indirect + github.com/ulikunitz/xz v0.5.6 +) diff --git a/vendor/github.com/dsnet/compress/go.sum b/vendor/github.com/dsnet/compress/go.sum new file mode 100644 index 00000000..b6fd40c7 --- /dev/null +++ b/vendor/github.com/dsnet/compress/go.sum @@ -0,0 +1,8 @@ +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= diff --git a/vendor/github.com/dsnet/compress/internal/common.go b/vendor/github.com/dsnet/compress/internal/common.go new file mode 100644 index 00000000..da4e7034 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/common.go @@ -0,0 +1,107 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package internal is a collection of common compression algorithms. +// +// For performance reasons, these packages lack strong error checking and +// require that the caller to ensure that strict invariants are kept. +package internal + +var ( + // IdentityLUT returns the input key itself. + IdentityLUT = func() (lut [256]byte) { + for i := range lut { + lut[i] = uint8(i) + } + return lut + }() + + // ReverseLUT returns the input key with its bits reversed. + ReverseLUT = func() (lut [256]byte) { + for i := range lut { + b := uint8(i) + b = (b&0xaa)>>1 | (b&0x55)<<1 + b = (b&0xcc)>>2 | (b&0x33)<<2 + b = (b&0xf0)>>4 | (b&0x0f)<<4 + lut[i] = b + } + return lut + }() +) + +// ReverseUint32 reverses all bits of v. +func ReverseUint32(v uint32) (x uint32) { + x |= uint32(ReverseLUT[byte(v>>0)]) << 24 + x |= uint32(ReverseLUT[byte(v>>8)]) << 16 + x |= uint32(ReverseLUT[byte(v>>16)]) << 8 + x |= uint32(ReverseLUT[byte(v>>24)]) << 0 + return x +} + +// ReverseUint32N reverses the lower n bits of v. +func ReverseUint32N(v uint32, n uint) (x uint32) { + return ReverseUint32(v << (32 - n)) +} + +// ReverseUint64 reverses all bits of v. +func ReverseUint64(v uint64) (x uint64) { + x |= uint64(ReverseLUT[byte(v>>0)]) << 56 + x |= uint64(ReverseLUT[byte(v>>8)]) << 48 + x |= uint64(ReverseLUT[byte(v>>16)]) << 40 + x |= uint64(ReverseLUT[byte(v>>24)]) << 32 + x |= uint64(ReverseLUT[byte(v>>32)]) << 24 + x |= uint64(ReverseLUT[byte(v>>40)]) << 16 + x |= uint64(ReverseLUT[byte(v>>48)]) << 8 + x |= uint64(ReverseLUT[byte(v>>56)]) << 0 + return x +} + +// ReverseUint64N reverses the lower n bits of v. +func ReverseUint64N(v uint64, n uint) (x uint64) { + return ReverseUint64(v << (64 - n)) +} + +// MoveToFront is a data structure that allows for more efficient move-to-front +// transformations. This specific implementation assumes that the alphabet is +// densely packed within 0..255. +type MoveToFront struct { + dict [256]uint8 // Mapping from indexes to values + tail int // Number of tail bytes that are already ordered +} + +func (m *MoveToFront) Encode(vals []uint8) { + copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity + + var max int + for i, val := range vals { + var idx uint8 // Reverse lookup idx in dict + for di, dv := range m.dict { + if dv == val { + idx = uint8(di) + break + } + } + vals[i] = idx + + max |= int(idx) + copy(m.dict[1:], m.dict[:idx]) + m.dict[0] = val + } + m.tail = 256 - max - 1 +} + +func (m *MoveToFront) Decode(idxs []uint8) { + copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity + + var max int + for i, idx := range idxs { + val := m.dict[idx] // Forward lookup val in dict + idxs[i] = val + + max |= int(idx) + copy(m.dict[1:], m.dict[:idx]) + m.dict[0] = val + } + m.tail = 256 - max - 1 +} diff --git a/vendor/github.com/dsnet/compress/internal/debug.go b/vendor/github.com/dsnet/compress/internal/debug.go new file mode 100644 index 00000000..01df1f89 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/debug.go @@ -0,0 +1,12 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build debug,!gofuzz + +package internal + +const ( + Debug = true + GoFuzz = false +) diff --git a/vendor/github.com/dsnet/compress/internal/errors/errors.go b/vendor/github.com/dsnet/compress/internal/errors/errors.go new file mode 100644 index 00000000..c631afbd --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/errors/errors.go @@ -0,0 +1,120 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package errors implements functions to manipulate compression errors. +// +// In idiomatic Go, it is an anti-pattern to use panics as a form of error +// reporting in the API. Instead, the expected way to transmit errors is by +// returning an error value. Unfortunately, the checking of "err != nil" in +// tight loops commonly found in compression causes non-negligible performance +// degradation. While this may not be idiomatic, the internal packages of this +// repository rely on panics as a normal means to convey errors. In order to +// ensure that these panics do not leak across the public API, the public +// packages must recover from these panics and present an error value. +// +// The Panic and Recover functions in this package provide a safe way to +// recover from errors only generated from within this repository. +// +// Example usage: +// func Foo() (err error) { +// defer errors.Recover(&err) +// +// if rand.Intn(2) == 0 { +// // Unexpected panics will not be caught by Recover. +// io.Closer(nil).Close() +// } else { +// // Errors thrown by Panic will be caught by Recover. +// errors.Panic(errors.New("whoopsie")) +// } +// } +// +package errors + +import "strings" + +const ( + // Unknown indicates that there is no classification for this error. + Unknown = iota + + // Internal indicates that this error is due to an internal bug. + // Users should file a issue report if this type of error is encountered. + Internal + + // Invalid indicates that this error is due to the user misusing the API + // and is indicative of a bug on the user's part. + Invalid + + // Deprecated indicates the use of a deprecated and unsupported feature. + Deprecated + + // Corrupted indicates that the input stream is corrupted. + Corrupted + + // Closed indicates that the handlers are closed. + Closed +) + +var codeMap = map[int]string{ + Unknown: "unknown error", + Internal: "internal error", + Invalid: "invalid argument", + Deprecated: "deprecated format", + Corrupted: "corrupted input", + Closed: "closed handler", +} + +type Error struct { + Code int // The error type + Pkg string // Name of the package where the error originated + Msg string // Descriptive message about the error (optional) +} + +func (e Error) Error() string { + var ss []string + for _, s := range []string{e.Pkg, codeMap[e.Code], e.Msg} { + if s != "" { + ss = append(ss, s) + } + } + return strings.Join(ss, ": ") +} + +func (e Error) CompressError() {} +func (e Error) IsInternal() bool { return e.Code == Internal } +func (e Error) IsInvalid() bool { return e.Code == Invalid } +func (e Error) IsDeprecated() bool { return e.Code == Deprecated } +func (e Error) IsCorrupted() bool { return e.Code == Corrupted } +func (e Error) IsClosed() bool { return e.Code == Closed } + +func IsInternal(err error) bool { return isCode(err, Internal) } +func IsInvalid(err error) bool { return isCode(err, Invalid) } +func IsDeprecated(err error) bool { return isCode(err, Deprecated) } +func IsCorrupted(err error) bool { return isCode(err, Corrupted) } +func IsClosed(err error) bool { return isCode(err, Closed) } + +func isCode(err error, code int) bool { + if cerr, ok := err.(Error); ok && cerr.Code == code { + return true + } + return false +} + +// errWrap is used by Panic and Recover to ensure that only errors raised by +// Panic are recovered by Recover. +type errWrap struct{ e *error } + +func Recover(err *error) { + switch ex := recover().(type) { + case nil: + // Do nothing. + case errWrap: + *err = *ex.e + default: + panic(ex) + } +} + +func Panic(err error) { + panic(errWrap{&err}) +} diff --git a/vendor/github.com/dsnet/compress/internal/gofuzz.go b/vendor/github.com/dsnet/compress/internal/gofuzz.go new file mode 100644 index 00000000..5035c9d6 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/gofuzz.go @@ -0,0 +1,12 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build gofuzz + +package internal + +const ( + Debug = true + GoFuzz = true +) diff --git a/vendor/github.com/dsnet/compress/internal/prefix/debug.go b/vendor/github.com/dsnet/compress/internal/prefix/debug.go new file mode 100644 index 00000000..04fce70b --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/debug.go @@ -0,0 +1,159 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build debug + +package prefix + +import ( + "fmt" + "math" + "strings" +) + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func lenBase2(n uint) int { + return int(math.Ceil(math.Log2(float64(n + 1)))) +} +func padBase2(v, n uint, m int) string { + s := fmt.Sprintf("%b", 1< 0 { + return strings.Repeat(" ", pad) + s + } + return s +} + +func lenBase10(n int) int { + return int(math.Ceil(math.Log10(float64(n + 1)))) +} +func padBase10(n, m int) string { + s := fmt.Sprintf("%d", n) + if pad := m - len(s); pad > 0 { + return strings.Repeat(" ", pad) + s + } + return s +} + +func (rc RangeCodes) String() string { + var maxLen, maxBase int + for _, c := range rc { + maxLen = max(maxLen, int(c.Len)) + maxBase = max(maxBase, int(c.Base)) + } + + var ss []string + ss = append(ss, "{") + for i, c := range rc { + base := padBase10(int(c.Base), lenBase10(maxBase)) + if c.Len > 0 { + base += fmt.Sprintf("-%d", c.End()-1) + } + ss = append(ss, fmt.Sprintf("\t%s: {len: %s, range: %s},", + padBase10(int(i), lenBase10(len(rc)-1)), + padBase10(int(c.Len), lenBase10(maxLen)), + base, + )) + } + ss = append(ss, "}") + return strings.Join(ss, "\n") +} + +func (pc PrefixCodes) String() string { + var maxSym, maxLen, maxCnt int + for _, c := range pc { + maxSym = max(maxSym, int(c.Sym)) + maxLen = max(maxLen, int(c.Len)) + maxCnt = max(maxCnt, int(c.Cnt)) + } + + var ss []string + ss = append(ss, "{") + for _, c := range pc { + var cntStr string + if maxCnt > 0 { + cnt := int(32*float32(c.Cnt)/float32(maxCnt) + 0.5) + cntStr = fmt.Sprintf("%s |%s", + padBase10(int(c.Cnt), lenBase10(maxCnt)), + strings.Repeat("#", cnt), + ) + } + ss = append(ss, fmt.Sprintf("\t%s: %s, %s", + padBase10(int(c.Sym), lenBase10(maxSym)), + padBase2(uint(c.Val), uint(c.Len), maxLen), + cntStr, + )) + } + ss = append(ss, "}") + return strings.Join(ss, "\n") +} + +func (pd Decoder) String() string { + var ss []string + ss = append(ss, "{") + if len(pd.chunks) > 0 { + ss = append(ss, "\tchunks: {") + for i, c := range pd.chunks { + label := "sym" + if uint(c&countMask) > uint(pd.chunkBits) { + label = "idx" + } + ss = append(ss, fmt.Sprintf("\t\t%s: {%s: %s, len: %s}", + padBase2(uint(i), uint(pd.chunkBits), int(pd.chunkBits)), + label, padBase10(int(c>>countBits), 3), + padBase10(int(c&countMask), 2), + )) + } + ss = append(ss, "\t},") + + for j, links := range pd.links { + ss = append(ss, fmt.Sprintf("\tlinks[%d]: {", j)) + linkBits := lenBase2(uint(pd.linkMask)) + for i, c := range links { + ss = append(ss, fmt.Sprintf("\t\t%s: {sym: %s, len: %s},", + padBase2(uint(i), uint(linkBits), int(linkBits)), + padBase10(int(c>>countBits), 3), + padBase10(int(c&countMask), 2), + )) + } + ss = append(ss, "\t},") + } + } + ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pd.chunkMask)) + ss = append(ss, fmt.Sprintf("\tlinkMask: %b,", pd.linkMask)) + ss = append(ss, fmt.Sprintf("\tchunkBits: %d,", pd.chunkBits)) + ss = append(ss, fmt.Sprintf("\tMinBits: %d,", pd.MinBits)) + ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pd.NumSyms)) + ss = append(ss, "}") + return strings.Join(ss, "\n") +} + +func (pe Encoder) String() string { + var maxLen int + for _, c := range pe.chunks { + maxLen = max(maxLen, int(c&countMask)) + } + + var ss []string + ss = append(ss, "{") + if len(pe.chunks) > 0 { + ss = append(ss, "\tchunks: {") + for i, c := range pe.chunks { + ss = append(ss, fmt.Sprintf("\t\t%s: %s,", + padBase10(i, 3), + padBase2(uint(c>>countBits), uint(c&countMask), maxLen), + )) + } + ss = append(ss, "\t},") + } + ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pe.chunkMask)) + ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pe.NumSyms)) + ss = append(ss, "}") + return strings.Join(ss, "\n") +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/decoder.go b/vendor/github.com/dsnet/compress/internal/prefix/decoder.go new file mode 100644 index 00000000..a9bc2dcb --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/decoder.go @@ -0,0 +1,136 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "sort" + + "github.com/dsnet/compress/internal" +) + +// The algorithm used to decode variable length codes is based on the lookup +// method in zlib. If the code is less-than-or-equal to maxChunkBits, +// then the symbol can be decoded using a single lookup into the chunks table. +// Otherwise, the links table will be used for a second level lookup. +// +// The chunks slice is keyed by the contents of the bit buffer ANDed with +// the chunkMask to avoid a out-of-bounds lookup. The value of chunks is a tuple +// that is decoded as follow: +// +// var length = chunks[bitBuffer&chunkMask] & countMask +// var symbol = chunks[bitBuffer&chunkMask] >> countBits +// +// If the decoded length is larger than chunkBits, then an overflow link table +// must be used for further decoding. In this case, the symbol is actually the +// index into the links tables. The second-level links table returned is +// processed in the same way as the chunks table. +// +// if length > chunkBits { +// var index = symbol // Previous symbol is index into links tables +// length = links[index][bitBuffer>>chunkBits & linkMask] & countMask +// symbol = links[index][bitBuffer>>chunkBits & linkMask] >> countBits +// } +// +// See the following: +// http://www.gzip.org/algorithm.txt + +type Decoder struct { + chunks []uint32 // First-level lookup map + links [][]uint32 // Second-level lookup map + chunkMask uint32 // Mask the length of the chunks table + linkMask uint32 // Mask the length of the link table + chunkBits uint32 // Bit-length of the chunks table + + MinBits uint32 // The minimum number of bits to safely make progress + NumSyms uint32 // Number of symbols +} + +// Init initializes Decoder according to the codes provided. +func (pd *Decoder) Init(codes PrefixCodes) { + // Handle special case trees. + if len(codes) <= 1 { + switch { + case len(codes) == 0: // Empty tree (should error if used later) + *pd = Decoder{chunks: pd.chunks[:0], links: pd.links[:0], NumSyms: 0} + case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero) + pd.chunks = append(pd.chunks[:0], codes[0].Sym< c.Len { + minBits = c.Len + } + if maxBits < c.Len { + maxBits = c.Len + } + } + + // Allocate chunks table as needed. + const maxChunkBits = 9 // This can be tuned for better performance + pd.NumSyms = uint32(len(codes)) + pd.MinBits = minBits + pd.chunkBits = maxBits + if pd.chunkBits > maxChunkBits { + pd.chunkBits = maxChunkBits + } + numChunks := 1 << pd.chunkBits + pd.chunks = allocUint32s(pd.chunks, numChunks) + pd.chunkMask = uint32(numChunks - 1) + + // Allocate links tables as needed. + pd.links = pd.links[:0] + pd.linkMask = 0 + if pd.chunkBits < maxBits { + numLinks := 1 << (maxBits - pd.chunkBits) + pd.linkMask = uint32(numLinks - 1) + + var linkIdx uint32 + for i := range pd.chunks { + pd.chunks[i] = 0 // Logic below relies on zero value as uninitialized + } + for _, c := range codes { + if c.Len > pd.chunkBits && pd.chunks[c.Val&pd.chunkMask] == 0 { + pd.chunks[c.Val&pd.chunkMask] = (linkIdx << countBits) | (pd.chunkBits + 1) + linkIdx++ + } + } + + pd.links = extendSliceUint32s(pd.links, int(linkIdx)) + linksFlat := allocUint32s(pd.links[0], numLinks*int(linkIdx)) + for i, j := 0, 0; i < len(pd.links); i, j = i+1, j+numLinks { + pd.links[i] = linksFlat[j : j+numLinks] + } + } + + // Fill out chunks and links tables with values. + for _, c := range codes { + chunk := c.Sym<> countBits + links := pd.links[linkIdx] + skip := 1 << uint(c.Len-pd.chunkBits) + for j := int(c.Val >> pd.chunkBits); j < len(links); j += skip { + links[j] = chunk + } + } + } +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/encoder.go b/vendor/github.com/dsnet/compress/internal/prefix/encoder.go new file mode 100644 index 00000000..4424a011 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/encoder.go @@ -0,0 +1,66 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "sort" + + "github.com/dsnet/compress/internal" +) + +type Encoder struct { + chunks []uint32 // First-level lookup map + chunkMask uint32 // Mask the length of the chunks table + + NumSyms uint32 // Number of symbols +} + +// Init initializes Encoder according to the codes provided. +func (pe *Encoder) Init(codes PrefixCodes) { + // Handle special case trees. + if len(codes) <= 1 { + switch { + case len(codes) == 0: // Empty tree (should error if used later) + *pe = Encoder{chunks: pe.chunks[:0], NumSyms: 0} + case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero) + pe.chunks = append(pe.chunks[:0], codes[0].Val< 0; n >>= 1 { + numChunks <<= 1 + } + pe.NumSyms = uint32(len(codes)) + +retry: + // Allocate and reset chunks. + pe.chunks = allocUint32s(pe.chunks, numChunks) + pe.chunkMask = uint32(numChunks - 1) + for i := range pe.chunks { + pe.chunks[i] = 0 // Logic below relies on zero value as uninitialized + } + + // Insert each symbol, checking that there are no conflicts. + for _, c := range codes { + if pe.chunks[c.Sym&pe.chunkMask] > 0 { + // Collision found our "hash" table, so grow and try again. + numChunks <<= 1 + goto retry + } + pe.chunks[c.Sym&pe.chunkMask] = c.Val<> uint(c.Len) + } + return sum == 0 || len(pc) == 0 +} + +// checkPrefixes reports whether all codes have non-overlapping prefixes. +func (pc PrefixCodes) checkPrefixes() bool { + for i, c1 := range pc { + for j, c2 := range pc { + mask := uint32(1)< 0 { + c.Val = internal.ReverseUint32N(c.Val, uint(c.Len)) + if vals[c.Len].Cnt > 0 && vals[c.Len].Val+1 != c.Val { + return false + } + vals[c.Len].Val = c.Val + vals[c.Len].Cnt++ + } + } + + // Rule 2. + var last PrefixCode + for _, v := range vals { + if v.Cnt > 0 { + curVal := v.Val - v.Cnt + 1 + if last.Cnt != 0 && last.Val >= curVal { + return false + } + last = v + } + } + return true +} + +// GenerateLengths assigns non-zero bit-lengths to all codes. Codes with high +// frequency counts will be assigned shorter codes to reduce bit entropy. +// This function is used primarily by compressors. +// +// The input codes must have the Cnt field populated, be sorted by count. +// Even if a code has a count of 0, a non-zero bit-length will be assigned. +// +// The result will have the Len field populated. The algorithm used guarantees +// that Len <= maxBits and that it is a complete prefix tree. The resulting +// codes will remain sorted by count. +func GenerateLengths(codes PrefixCodes, maxBits uint) error { + if len(codes) <= 1 { + if len(codes) == 1 { + codes[0].Len = 0 + } + return nil + } + + // Verify that the codes are in ascending order by count. + cntLast := codes[0].Cnt + for _, c := range codes[1:] { + if c.Cnt < cntLast { + return errorf(errors.Invalid, "non-monotonically increasing symbol counts") + } + cntLast = c.Cnt + } + + // Construct a Huffman tree used to generate the bit-lengths. + // + // The Huffman tree is a binary tree where each symbol lies as a leaf node + // on this tree. The length of the prefix code to assign is the depth of + // that leaf from the root. The Huffman algorithm, which runs in O(n), + // is used to generate the tree. It assumes that codes are sorted in + // increasing order of frequency. + // + // The algorithm is as follows: + // 1. Start with two queues, F and Q, where F contains all of the starting + // symbols sorted such that symbols with lowest counts come first. + // 2. While len(F)+len(Q) > 1: + // 2a. Dequeue the node from F or Q that has the lowest weight as N0. + // 2b. Dequeue the node from F or Q that has the lowest weight as N1. + // 2c. Create a new node N that has N0 and N1 as its children. + // 2d. Enqueue N into the back of Q. + // 3. The tree's root node is Q[0]. + type node struct { + cnt uint32 + + // n0 or c0 represent the left child of this node. + // Since Go does not have unions, only one of these will be set. + // Similarly, n1 or c1 represent the right child of this node. + // + // If n0 or n1 is set, then it represents a "pointer" to another + // node in the Huffman tree. Since Go's pointer analysis cannot reason + // that these node pointers do not escape (golang.org/issue/13493), + // we use an index to a node in the nodes slice as a pseudo-pointer. + // + // If c0 or c1 is set, then it represents a leaf "node" in the + // Huffman tree. The leaves are the PrefixCode values themselves. + n0, n1 int // Index to child nodes + c0, c1 *PrefixCode + } + var nodeIdx int + var nodeArr [1024]node // Large enough to handle most cases on the stack + nodes := nodeArr[:] + if len(nodes) < len(codes) { + nodes = make([]node, len(codes)) // Number of internal nodes < number of leaves + } + freqs, queue := codes, nodes[:0] + for len(freqs)+len(queue) > 1 { + // These are the two smallest nodes at the front of freqs and queue. + var n node + if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { + n.c0, freqs = &freqs[0], freqs[1:] + n.cnt += n.c0.Cnt + } else { + n.cnt += queue[0].cnt + n.n0 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0] + nodeIdx++ + queue = queue[1:] + } + if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { + n.c1, freqs = &freqs[0], freqs[1:] + n.cnt += n.c1.Cnt + } else { + n.cnt += queue[0].cnt + n.n1 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0] + nodeIdx++ + queue = queue[1:] + } + queue = append(queue, n) + } + rootIdx := nodeIdx + + // Search the whole binary tree, noting when we hit each leaf node. + // We do not care about the exact Huffman tree structure, but rather we only + // care about depth of each of the leaf nodes. That is, the depth determines + // how long each symbol is in bits. + // + // Since the number of leaves is n, there is at most n internal nodes. + // Thus, this algorithm runs in O(n). + var fixBits bool + var explore func(int, uint) + explore = func(rootIdx int, level uint) { + root := &nodes[rootIdx] + + // Explore left branch. + if root.c0 == nil { + explore(root.n0, level+1) + } else { + fixBits = fixBits || (level > maxBits) + root.c0.Len = uint32(level) + } + + // Explore right branch. + if root.c1 == nil { + explore(root.n1, level+1) + } else { + fixBits = fixBits || (level > maxBits) + root.c1.Len = uint32(level) + } + } + explore(rootIdx, 1) + + // Fix the bit-lengths if we violate the maxBits requirement. + if fixBits { + // Create histogram for number of symbols with each bit-length. + var symBitsArr [valueBits + 1]uint32 + symBits := symBitsArr[:] // symBits[nb] indicates number of symbols using nb bits + for _, c := range codes { + for int(c.Len) >= len(symBits) { + symBits = append(symBits, 0) + } + symBits[c.Len]++ + } + + // Fudge the tree such that the largest bit-length is <= maxBits. + // This is accomplish by effectively doing a tree rotation. That is, we + // increase the bit-length of some higher frequency code, so that the + // bit-lengths of lower frequency codes can be decreased. + // + // Visually, this looks like the following transform: + // + // Level Before After + // __ ___ + // / \ / \ + // n-1 X / \ /\ /\ + // n X /\ X X X X + // n+1 X X + // + var treeRotate func(uint) + treeRotate = func(nb uint) { + if symBits[nb-1] == 0 { + treeRotate(nb - 1) + } + symBits[nb-1] -= 1 // Push this node to the level below + symBits[nb] += 3 // This level gets one node from above, two from below + symBits[nb+1] -= 2 // Push two nodes to the level above + } + for i := uint(len(symBits)) - 1; i > maxBits; i-- { + for symBits[i] > 0 { + treeRotate(i - 1) + } + } + + // Assign bit-lengths to each code. Since codes is sorted in increasing + // order of frequency, that means that the most frequently used symbols + // should have the shortest bit-lengths. Thus, we copy symbols to codes + // from the back of codes first. + cs := codes + for nb, cnt := range symBits { + if cnt > 0 { + pos := len(cs) - int(cnt) + cs2 := cs[pos:] + for i := range cs2 { + cs2[i].Len = uint32(nb) + } + cs = cs[:pos] + } + } + if len(cs) != 0 { + panic("not all codes were used up") + } + } + + if internal.Debug && !codes.checkLengths() { + panic("incomplete prefix tree detected") + } + return nil +} + +// GeneratePrefixes assigns a prefix value to all codes according to the +// bit-lengths. This function is used by both compressors and decompressors. +// +// The input codes must have the Sym and Len fields populated and be +// sorted by symbol. The bit-lengths of each code must be properly allocated, +// such that it forms a complete tree. +// +// The result will have the Val field populated and will produce a canonical +// prefix tree. The resulting codes will remain sorted by symbol. +func GeneratePrefixes(codes PrefixCodes) error { + if len(codes) <= 1 { + if len(codes) == 1 { + if codes[0].Len != 0 { + return errorf(errors.Invalid, "degenerate prefix tree with one node") + } + codes[0].Val = 0 + } + return nil + } + + // Compute basic statistics on the symbols. + var bitCnts [valueBits + 1]uint + c0 := codes[0] + bitCnts[c0.Len]++ + minBits, maxBits, symLast := c0.Len, c0.Len, c0.Sym + for _, c := range codes[1:] { + if c.Sym <= symLast { + return errorf(errors.Invalid, "non-unique or non-monotonically increasing symbols") + } + if minBits > c.Len { + minBits = c.Len + } + if maxBits < c.Len { + maxBits = c.Len + } + bitCnts[c.Len]++ // Histogram of bit counts + symLast = c.Sym // Keep track of last symbol + } + if minBits == 0 { + return errorf(errors.Invalid, "invalid prefix bit-length") + } + + // Compute the next code for a symbol of a given bit length. + var nextCodes [valueBits + 1]uint + var code uint + for i := minBits; i <= maxBits; i++ { + code <<= 1 + nextCodes[i] = code + code += bitCnts[i] + } + if code != 1<= n { + return s[:n] + } + return make([]uint32, n, n*3/2) +} + +func extendSliceUint32s(s [][]uint32, n int) [][]uint32 { + if cap(s) >= n { + return s[:n] + } + ss := make([][]uint32, n, n*3/2) + copy(ss, s[:cap(s)]) + return ss +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/range.go b/vendor/github.com/dsnet/compress/internal/prefix/range.go new file mode 100644 index 00000000..b7eddad5 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/range.go @@ -0,0 +1,93 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +type RangeCode struct { + Base uint32 // Starting base offset of the range + Len uint32 // Bit-length of a subsequent integer to add to base offset +} +type RangeCodes []RangeCode + +type RangeEncoder struct { + rcs RangeCodes + lut [1024]uint32 + minBase uint +} + +// End reports the non-inclusive ending range. +func (rc RangeCode) End() uint32 { return rc.Base + (1 << rc.Len) } + +// MakeRangeCodes creates a RangeCodes, where each region is assumed to be +// contiguously stacked, without any gaps, with bit-lengths taken from bits. +func MakeRangeCodes(minBase uint, bits []uint) (rc RangeCodes) { + for _, nb := range bits { + rc = append(rc, RangeCode{Base: uint32(minBase), Len: uint32(nb)}) + minBase += 1 << nb + } + return rc +} + +// Base reports the inclusive starting range for all ranges. +func (rcs RangeCodes) Base() uint32 { return rcs[0].Base } + +// End reports the non-inclusive ending range for all ranges. +func (rcs RangeCodes) End() uint32 { return rcs[len(rcs)-1].End() } + +// checkValid reports whether the RangeCodes is valid. In order to be valid, +// the following must hold true: +// rcs[i-1].Base <= rcs[i].Base +// rcs[i-1].End <= rcs[i].End +// rcs[i-1].End >= rcs[i].Base +// +// Practically speaking, each range must be increasing and must not have any +// gaps in between. It is okay for ranges to overlap. +func (rcs RangeCodes) checkValid() bool { + if len(rcs) == 0 { + return false + } + pre := rcs[0] + for _, cur := range rcs[1:] { + preBase, preEnd := pre.Base, pre.End() + curBase, curEnd := cur.Base, cur.End() + if preBase > curBase || preEnd > curEnd || preEnd < curBase { + return false + } + pre = cur + } + return true +} + +func (re *RangeEncoder) Init(rcs RangeCodes) { + if !rcs.checkValid() { + panic("invalid range codes") + } + *re = RangeEncoder{rcs: rcs, minBase: uint(rcs.Base())} + for sym, rc := range rcs { + base := int(rc.Base) - int(re.minBase) + end := int(rc.End()) - int(re.minBase) + if base >= len(re.lut) { + break + } + if end > len(re.lut) { + end = len(re.lut) + } + for i := base; i < end; i++ { + re.lut[i] = uint32(sym) + } + } +} + +func (re *RangeEncoder) Encode(offset uint) (sym uint) { + if idx := int(offset - re.minBase); idx < len(re.lut) { + return uint(re.lut[idx]) + } + sym = uint(re.lut[len(re.lut)-1]) +retry: + if int(sym) >= len(re.rcs) || re.rcs[sym].Base > uint32(offset) { + return sym - 1 + } + sym++ + goto retry // Avoid for-loop so that this function can be inlined +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/reader.go b/vendor/github.com/dsnet/compress/internal/prefix/reader.go new file mode 100644 index 00000000..e6252c95 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/reader.go @@ -0,0 +1,335 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "bufio" + "bytes" + "encoding/binary" + "io" + "strings" + + "github.com/dsnet/compress" + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" +) + +// Reader implements a prefix decoder. If the input io.Reader satisfies the +// compress.ByteReader or compress.BufferedReader interface, then it also +// guarantees that it will never read more bytes than is necessary. +// +// For high performance, provide an io.Reader that satisfies the +// compress.BufferedReader interface. If the input does not satisfy either +// compress.ByteReader or compress.BufferedReader, then it will be internally +// wrapped with a bufio.Reader. +type Reader struct { + Offset int64 // Number of bytes read from the underlying io.Reader + + rd io.Reader + byteRd compress.ByteReader // Set if rd is a ByteReader + bufRd compress.BufferedReader // Set if rd is a BufferedReader + + bufBits uint64 // Buffer to hold some bits + numBits uint // Number of valid bits in bufBits + bigEndian bool // Do we treat input bytes as big endian? + + // These fields are only used if rd is a compress.BufferedReader. + bufPeek []byte // Buffer for the Peek data + discardBits int // Number of bits to discard from reader + fedBits uint // Number of bits fed in last call to PullBits + + // These fields are used to reduce allocations. + bb *buffer + br *bytesReader + sr *stringReader + bu *bufio.Reader +} + +// Init initializes the bit Reader to read from r. If bigEndian is true, then +// bits will be read starting from the most-significant bits of a byte +// (as done in bzip2), otherwise it will read starting from the +// least-significant bits of a byte (such as for deflate and brotli). +func (pr *Reader) Init(r io.Reader, bigEndian bool) { + *pr = Reader{ + rd: r, + bigEndian: bigEndian, + + bb: pr.bb, + br: pr.br, + sr: pr.sr, + bu: pr.bu, + } + switch rr := r.(type) { + case *bytes.Buffer: + if pr.bb == nil { + pr.bb = new(buffer) + } + *pr.bb = buffer{Buffer: rr} + pr.bufRd = pr.bb + case *bytes.Reader: + if pr.br == nil { + pr.br = new(bytesReader) + } + *pr.br = bytesReader{Reader: rr} + pr.bufRd = pr.br + case *strings.Reader: + if pr.sr == nil { + pr.sr = new(stringReader) + } + *pr.sr = stringReader{Reader: rr} + pr.bufRd = pr.sr + case compress.BufferedReader: + pr.bufRd = rr + case compress.ByteReader: + pr.byteRd = rr + default: + if pr.bu == nil { + pr.bu = bufio.NewReader(nil) + } + pr.bu.Reset(r) + pr.rd, pr.bufRd = pr.bu, pr.bu + } +} + +// BitsRead reports the total number of bits emitted from any Read method. +func (pr *Reader) BitsRead() int64 { + offset := 8*pr.Offset - int64(pr.numBits) + if pr.bufRd != nil { + discardBits := pr.discardBits + int(pr.fedBits-pr.numBits) + offset = 8*pr.Offset + int64(discardBits) + } + return offset +} + +// IsBufferedReader reports whether the underlying io.Reader is also a +// compress.BufferedReader. +func (pr *Reader) IsBufferedReader() bool { + return pr.bufRd != nil +} + +// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment. +func (pr *Reader) ReadPads() uint { + nb := pr.numBits % 8 + val := uint(pr.bufBits & uint64(1<>= nb + pr.numBits -= nb + return val +} + +// Read reads bytes into buf. +// The bit-ordering mode does not affect this method. +func (pr *Reader) Read(buf []byte) (cnt int, err error) { + if pr.numBits > 0 { + if pr.numBits%8 != 0 { + return 0, errorf(errors.Invalid, "non-aligned bit buffer") + } + for cnt = 0; len(buf) > cnt && pr.numBits > 0; cnt++ { + if pr.bigEndian { + buf[cnt] = internal.ReverseLUT[byte(pr.bufBits)] + } else { + buf[cnt] = byte(pr.bufBits) + } + pr.bufBits >>= 8 + pr.numBits -= 8 + } + return cnt, nil + } + if _, err := pr.Flush(); err != nil { + return 0, err + } + cnt, err = pr.rd.Read(buf) + pr.Offset += int64(cnt) + return cnt, err +} + +// ReadOffset reads an offset value using the provided RangeCodes indexed by +// the symbol read. +func (pr *Reader) ReadOffset(pd *Decoder, rcs RangeCodes) uint { + rc := rcs[pr.ReadSymbol(pd)] + return uint(rc.Base) + pr.ReadBits(uint(rc.Len)) +} + +// TryReadBits attempts to read nb bits using the contents of the bit buffer +// alone. It returns the value and whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pr *Reader) TryReadBits(nb uint) (uint, bool) { + if pr.numBits < nb { + return 0, false + } + val := uint(pr.bufBits & uint64(1<>= nb + pr.numBits -= nb + return val, true +} + +// ReadBits reads nb bits in from the underlying reader. +func (pr *Reader) ReadBits(nb uint) uint { + if err := pr.PullBits(nb); err != nil { + errors.Panic(err) + } + val := uint(pr.bufBits & uint64(1<>= nb + pr.numBits -= nb + return val +} + +// TryReadSymbol attempts to decode the next symbol using the contents of the +// bit buffer alone. It returns the decoded symbol and whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pr *Reader) TryReadSymbol(pd *Decoder) (uint, bool) { + if pr.numBits < uint(pd.MinBits) || len(pd.chunks) == 0 { + return 0, false + } + chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] + nb := uint(chunk & countMask) + if nb > pr.numBits || nb > uint(pd.chunkBits) { + return 0, false + } + pr.bufBits >>= nb + pr.numBits -= nb + return uint(chunk >> countBits), true +} + +// ReadSymbol reads the next symbol using the provided prefix Decoder. +func (pr *Reader) ReadSymbol(pd *Decoder) uint { + if len(pd.chunks) == 0 { + panicf(errors.Invalid, "decode with empty prefix tree") + } + + nb := uint(pd.MinBits) + for { + if err := pr.PullBits(nb); err != nil { + errors.Panic(err) + } + chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] + nb = uint(chunk & countMask) + if nb > uint(pd.chunkBits) { + linkIdx := chunk >> countBits + chunk = pd.links[linkIdx][uint32(pr.bufBits>>pd.chunkBits)&pd.linkMask] + nb = uint(chunk & countMask) + } + if nb <= pr.numBits { + pr.bufBits >>= nb + pr.numBits -= nb + return uint(chunk >> countBits) + } + } +} + +// Flush updates the read offset of the underlying ByteReader. +// If reader is a compress.BufferedReader, then this calls Discard to update +// the read offset. +func (pr *Reader) Flush() (int64, error) { + if pr.bufRd == nil { + return pr.Offset, nil + } + + // Update the number of total bits to discard. + pr.discardBits += int(pr.fedBits - pr.numBits) + pr.fedBits = pr.numBits + + // Discard some bytes to update read offset. + var err error + nd := (pr.discardBits + 7) / 8 // Round up to nearest byte + nd, err = pr.bufRd.Discard(nd) + pr.discardBits -= nd * 8 // -7..0 + pr.Offset += int64(nd) + + // These are invalid after Discard. + pr.bufPeek = nil + return pr.Offset, err +} + +// PullBits ensures that at least nb bits exist in the bit buffer. +// If the underlying reader is a compress.BufferedReader, then this will fill +// the bit buffer with as many bits as possible, relying on Peek and Discard to +// properly advance the read offset. Otherwise, it will use ReadByte to fill the +// buffer with just the right number of bits. +func (pr *Reader) PullBits(nb uint) error { + if pr.bufRd != nil { + pr.discardBits += int(pr.fedBits - pr.numBits) + for { + if len(pr.bufPeek) == 0 { + pr.fedBits = pr.numBits // Don't discard bits just added + if _, err := pr.Flush(); err != nil { + return err + } + + // Peek no more bytes than necessary. + // The computation for cntPeek computes the minimum number of + // bytes to Peek to fill nb bits. + var err error + cntPeek := int(nb+(-nb&7)) / 8 + if cntPeek < pr.bufRd.Buffered() { + cntPeek = pr.bufRd.Buffered() + } + pr.bufPeek, err = pr.bufRd.Peek(cntPeek) + pr.bufPeek = pr.bufPeek[int(pr.numBits/8):] // Skip buffered bits + if len(pr.bufPeek) == 0 { + if pr.numBits >= nb { + break + } + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + } + + n := int(64-pr.numBits) / 8 // Number of bytes to copy to bit buffer + if len(pr.bufPeek) >= 8 { + // Starting with Go 1.7, the compiler should use a wide integer + // load here if the architecture supports it. + u := binary.LittleEndian.Uint64(pr.bufPeek) + if pr.bigEndian { + // Swap all the bits within each byte. + u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 + u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 + u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 + } + + pr.bufBits |= u << pr.numBits + pr.numBits += uint(n * 8) + pr.bufPeek = pr.bufPeek[n:] + break + } else { + if n > len(pr.bufPeek) { + n = len(pr.bufPeek) + } + for _, c := range pr.bufPeek[:n] { + if pr.bigEndian { + c = internal.ReverseLUT[c] + } + pr.bufBits |= uint64(c) << pr.numBits + pr.numBits += 8 + } + pr.bufPeek = pr.bufPeek[n:] + if pr.numBits > 56 { + break + } + } + } + pr.fedBits = pr.numBits + } else { + for pr.numBits < nb { + c, err := pr.byteRd.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + if pr.bigEndian { + c = internal.ReverseLUT[c] + } + pr.bufBits |= uint64(c) << pr.numBits + pr.numBits += 8 + pr.Offset++ + } + } + return nil +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/wrap.go b/vendor/github.com/dsnet/compress/internal/prefix/wrap.go new file mode 100644 index 00000000..49906d4a --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/wrap.go @@ -0,0 +1,146 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "bytes" + "io" + "strings" +) + +// For some of the common Readers, we wrap and extend them to satisfy the +// compress.BufferedReader interface to improve performance. + +type buffer struct { + *bytes.Buffer +} + +type bytesReader struct { + *bytes.Reader + pos int64 + buf []byte + arr [512]byte +} + +type stringReader struct { + *strings.Reader + pos int64 + buf []byte + arr [512]byte +} + +func (r *buffer) Buffered() int { + return r.Len() +} + +func (r *buffer) Peek(n int) ([]byte, error) { + b := r.Bytes() + if len(b) < n { + return b, io.EOF + } + return b[:n], nil +} + +func (r *buffer) Discard(n int) (int, error) { + b := r.Next(n) + if len(b) < n { + return len(b), io.EOF + } + return n, nil +} + +func (r *bytesReader) Buffered() int { + r.update() + if r.Len() > len(r.buf) { + return len(r.buf) + } + return r.Len() +} + +func (r *bytesReader) Peek(n int) ([]byte, error) { + if n > len(r.arr) { + return nil, io.ErrShortBuffer + } + + // Return sub-slice of local buffer if possible. + r.update() + if len(r.buf) >= n { + return r.buf[:n], nil + } + + // Fill entire local buffer, and return appropriate sub-slice. + cnt, err := r.ReadAt(r.arr[:], r.pos) + r.buf = r.arr[:cnt] + if cnt < n { + return r.arr[:cnt], err + } + return r.arr[:n], nil +} + +func (r *bytesReader) Discard(n int) (int, error) { + var err error + if n > r.Len() { + n, err = r.Len(), io.EOF + } + r.Seek(int64(n), io.SeekCurrent) + return n, err +} + +// update reslices the internal buffer to be consistent with the read offset. +func (r *bytesReader) update() { + pos, _ := r.Seek(0, io.SeekCurrent) + if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { + r.buf, r.pos = r.buf[off:], pos + } else { + r.buf, r.pos = nil, pos + } +} + +func (r *stringReader) Buffered() int { + r.update() + if r.Len() > len(r.buf) { + return len(r.buf) + } + return r.Len() +} + +func (r *stringReader) Peek(n int) ([]byte, error) { + if n > len(r.arr) { + return nil, io.ErrShortBuffer + } + + // Return sub-slice of local buffer if possible. + r.update() + if len(r.buf) >= n { + return r.buf[:n], nil + } + + // Fill entire local buffer, and return appropriate sub-slice. + cnt, err := r.ReadAt(r.arr[:], r.pos) + r.buf = r.arr[:cnt] + if cnt < n { + return r.arr[:cnt], err + } + return r.arr[:n], nil +} + +func (r *stringReader) Discard(n int) (int, error) { + var err error + if n > r.Len() { + n, err = r.Len(), io.EOF + } + r.Seek(int64(n), io.SeekCurrent) + return n, err +} + +// update reslices the internal buffer to be consistent with the read offset. +func (r *stringReader) update() { + pos, _ := r.Seek(0, io.SeekCurrent) + if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { + r.buf, r.pos = r.buf[off:], pos + } else { + r.buf, r.pos = nil, pos + } +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/writer.go b/vendor/github.com/dsnet/compress/internal/prefix/writer.go new file mode 100644 index 00000000..c9783905 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/writer.go @@ -0,0 +1,166 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "encoding/binary" + "io" + + "github.com/dsnet/compress/internal/errors" +) + +// Writer implements a prefix encoder. For performance reasons, Writer will not +// write bytes immediately to the underlying stream. +type Writer struct { + Offset int64 // Number of bytes written to the underlying io.Writer + + wr io.Writer + bufBits uint64 // Buffer to hold some bits + numBits uint // Number of valid bits in bufBits + bigEndian bool // Are bits written in big-endian order? + + buf [512]byte + cntBuf int +} + +// Init initializes the bit Writer to write to w. If bigEndian is true, then +// bits will be written starting from the most-significant bits of a byte +// (as done in bzip2), otherwise it will write starting from the +// least-significant bits of a byte (such as for deflate and brotli). +func (pw *Writer) Init(w io.Writer, bigEndian bool) { + *pw = Writer{wr: w, bigEndian: bigEndian} + return +} + +// BitsWritten reports the total number of bits issued to any Write method. +func (pw *Writer) BitsWritten() int64 { + return 8*pw.Offset + 8*int64(pw.cntBuf) + int64(pw.numBits) +} + +// WritePads writes 0-7 bits to the bit buffer to achieve byte-alignment. +func (pw *Writer) WritePads(v uint) { + nb := -pw.numBits & 7 + pw.bufBits |= uint64(v) << pw.numBits + pw.numBits += nb +} + +// Write writes bytes from buf. +// The bit-ordering mode does not affect this method. +func (pw *Writer) Write(buf []byte) (cnt int, err error) { + if pw.numBits > 0 || pw.cntBuf > 0 { + if pw.numBits%8 != 0 { + return 0, errorf(errors.Invalid, "non-aligned bit buffer") + } + if _, err := pw.Flush(); err != nil { + return 0, err + } + } + cnt, err = pw.wr.Write(buf) + pw.Offset += int64(cnt) + return cnt, err +} + +// WriteOffset writes ofs in a (sym, extra) fashion using the provided prefix +// Encoder and RangeEncoder. +func (pw *Writer) WriteOffset(ofs uint, pe *Encoder, re *RangeEncoder) { + sym := re.Encode(ofs) + pw.WriteSymbol(sym, pe) + rc := re.rcs[sym] + pw.WriteBits(ofs-uint(rc.Base), uint(rc.Len)) +} + +// TryWriteBits attempts to write nb bits using the contents of the bit buffer +// alone. It reports whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pw *Writer) TryWriteBits(v, nb uint) bool { + if 64-pw.numBits < nb { + return false + } + pw.bufBits |= uint64(v) << pw.numBits + pw.numBits += nb + return true +} + +// WriteBits writes nb bits of v to the underlying writer. +func (pw *Writer) WriteBits(v, nb uint) { + if _, err := pw.PushBits(); err != nil { + errors.Panic(err) + } + pw.bufBits |= uint64(v) << pw.numBits + pw.numBits += nb +} + +// TryWriteSymbol attempts to encode the next symbol using the contents of the +// bit buffer alone. It reports whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pw *Writer) TryWriteSymbol(sym uint, pe *Encoder) bool { + chunk := pe.chunks[uint32(sym)&pe.chunkMask] + nb := uint(chunk & countMask) + if 64-pw.numBits < nb { + return false + } + pw.bufBits |= uint64(chunk>>countBits) << pw.numBits + pw.numBits += nb + return true +} + +// WriteSymbol writes the symbol using the provided prefix Encoder. +func (pw *Writer) WriteSymbol(sym uint, pe *Encoder) { + if _, err := pw.PushBits(); err != nil { + errors.Panic(err) + } + chunk := pe.chunks[uint32(sym)&pe.chunkMask] + nb := uint(chunk & countMask) + pw.bufBits |= uint64(chunk>>countBits) << pw.numBits + pw.numBits += nb +} + +// Flush flushes all complete bytes from the bit buffer to the byte buffer, and +// then flushes all bytes in the byte buffer to the underlying writer. +// After this call, the bit Writer is will only withhold 7 bits at most. +func (pw *Writer) Flush() (int64, error) { + if pw.numBits < 8 && pw.cntBuf == 0 { + return pw.Offset, nil + } + if _, err := pw.PushBits(); err != nil { + return pw.Offset, err + } + cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) + pw.cntBuf -= cnt + pw.Offset += int64(cnt) + return pw.Offset, err +} + +// PushBits pushes as many bytes as possible from the bit buffer to the byte +// buffer, reporting the number of bits pushed. +func (pw *Writer) PushBits() (uint, error) { + if pw.cntBuf >= len(pw.buf)-8 { + cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) + pw.cntBuf -= cnt + pw.Offset += int64(cnt) + if err != nil { + return 0, err + } + } + + u := pw.bufBits + if pw.bigEndian { + // Swap all the bits within each byte. + u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 + u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 + u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 + } + // Starting with Go 1.7, the compiler should use a wide integer + // store here if the architecture supports it. + binary.LittleEndian.PutUint64(pw.buf[pw.cntBuf:], u) + + nb := pw.numBits / 8 // Number of bytes to copy from bit buffer + pw.cntBuf += int(nb) + pw.bufBits >>= 8 * nb + pw.numBits -= 8 * nb + return 8 * nb, nil +} diff --git a/vendor/github.com/dsnet/compress/internal/release.go b/vendor/github.com/dsnet/compress/internal/release.go new file mode 100644 index 00000000..0990be1c --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/release.go @@ -0,0 +1,21 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build !debug,!gofuzz + +package internal + +// Debug indicates whether the debug build tag was set. +// +// If set, programs may choose to print with more human-readable +// debug information and also perform sanity checks that would otherwise be too +// expensive to run in a release build. +const Debug = false + +// GoFuzz indicates whether the gofuzz build tag was set. +// +// If set, programs may choose to disable certain checks (like checksums) that +// would be nearly impossible for gofuzz to properly get right. +// If GoFuzz is set, it implies that Debug is set as well. +const GoFuzz = false diff --git a/vendor/github.com/dsnet/compress/zbench.sh b/vendor/github.com/dsnet/compress/zbench.sh new file mode 100755 index 00000000..0205920d --- /dev/null +++ b/vendor/github.com/dsnet/compress/zbench.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +# zbench wraps internal/tool/bench and is useful for comparing benchmarks from +# the implementations in this repository relative to other implementations. +# +# See internal/tool/bench/main.go for more details. +cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/bench +go run $(go list -f '{{ join .GoFiles "\n" }}') "$@" diff --git a/vendor/github.com/dsnet/compress/zfuzz.sh b/vendor/github.com/dsnet/compress/zfuzz.sh new file mode 100755 index 00000000..42958ed4 --- /dev/null +++ b/vendor/github.com/dsnet/compress/zfuzz.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +# zfuzz wraps internal/tool/fuzz and is useful for fuzz testing each of +# the implementations in this repository. +cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/fuzz +./fuzz.sh "$@" diff --git a/vendor/github.com/dsnet/compress/zprof.sh b/vendor/github.com/dsnet/compress/zprof.sh new file mode 100755 index 00000000..3cd535be --- /dev/null +++ b/vendor/github.com/dsnet/compress/zprof.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +if [ $# == 0 ]; then + echo "Usage: $0 PKG_PATH TEST_ARGS..." + echo "" + echo "Runs coverage and performance benchmarks for a given package." + echo "The results are stored in the _zprof_ directory." + echo "" + echo "Example:" + echo " $0 flate -test.bench=Decode/Twain/Default" + exit 1 +fi + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG_PATH=$1 +PKG_NAME=$(basename $PKG_PATH) +shift + +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR $PKG_PATH/$PKG_NAME.test" SIGINT SIGTERM EXIT + +( + cd $DIR/$PKG_PATH + + # Print the go version. + go version + + # Perform coverage profiling. + go test github.com/dsnet/compress/$PKG_PATH -coverprofile $TMPDIR/cover.profile + if [ $? != 0 ]; then exit 1; fi + go tool cover -html $TMPDIR/cover.profile -o cover.html + + # Perform performance profiling. + if [ $# != 0 ]; then + go test -c github.com/dsnet/compress/$PKG_PATH + if [ $? != 0 ]; then exit 1; fi + ./$PKG_NAME.test -test.cpuprofile $TMPDIR/cpu.profile -test.memprofile $TMPDIR/mem.profile -test.run - "$@" + PPROF="go tool pprof" + $PPROF -output=cpu.svg -web $PKG_NAME.test $TMPDIR/cpu.profile 2> /dev/null + $PPROF -output=cpu.html -weblist=. $PKG_NAME.test $TMPDIR/cpu.profile 2> /dev/null + $PPROF -output=mem_objects.svg -alloc_objects -web $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + $PPROF -output=mem_objects.html -alloc_objects -weblist=. $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + $PPROF -output=mem_space.svg -alloc_space -web $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + $PPROF -output=mem_space.html -alloc_space -weblist=. $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + fi + + rm -rf $DIR/_zprof_/$PKG_NAME + mkdir -p $DIR/_zprof_/$PKG_NAME + mv *.html *.svg $DIR/_zprof_/$PKG_NAME 2> /dev/null +) diff --git a/vendor/github.com/dsnet/compress/ztest.sh b/vendor/github.com/dsnet/compress/ztest.sh new file mode 100755 index 00000000..15c4c00b --- /dev/null +++ b/vendor/github.com/dsnet/compress/ztest.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +cd $(go list -f '{{ .Dir }}' github.com/dsnet/compress) + +BOLD="\x1b[1mRunning: " +PASS="\x1b[32mPASS" +FAIL="\x1b[31mFAIL" +RESET="\x1b[0m" + +echo -e "${BOLD}fmt${RESET}" +RET_FMT=$(find . -name "*.go" | egrep -v "/(_.*_|\..*|testdata)/" | xargs gofmt -d) +if [[ ! -z "$RET_FMT" ]]; then echo "$RET_FMT"; echo; fi + +echo -e "${BOLD}test${RESET}" +RET_TEST=$(go test -race ./... | egrep -v "^(ok|[?])\s+") +if [[ ! -z "$RET_TEST" ]]; then echo "$RET_TEST"; echo; fi + +echo -e "${BOLD}staticcheck${RESET}" +RET_SCHK=$(staticcheck \ + -ignore " + github.com/dsnet/compress/brotli/*.go:SA4016 + github.com/dsnet/compress/brotli/*.go:S1023 + github.com/dsnet/compress/brotli/*.go:U1000 + github.com/dsnet/compress/bzip2/*.go:S1023 + github.com/dsnet/compress/flate/*.go:U1000 + github.com/dsnet/compress/internal/cgo/lzma/*.go:SA4000 + github.com/dsnet/compress/internal/prefix/*.go:S1004 + github.com/dsnet/compress/internal/prefix/*.go:S1023 + github.com/dsnet/compress/internal/prefix/*.go:SA4016 + github.com/dsnet/compress/internal/tool/bench/*.go:S1007 + github.com/dsnet/compress/xflate/internal/meta/*.go:S1023 + " ./... 2>&1) +if [[ ! -z "$RET_SCHK" ]]; then echo "$RET_SCHK"; echo; fi + +echo -e "${BOLD}lint${RESET}" +RET_LINT=$(golint ./... 2>&1 | + egrep -v "^vendor/" | + egrep -v "should have comment(.*)or be unexported" | + egrep -v "^(.*)type name will be used as(.*)by other packages" | + egrep -v "^brotli/transform.go:(.*)replace i [+]= 1 with i[+]{2}" | + egrep -v "^internal/prefix/prefix.go:(.*)replace symBits(.*) [-]= 1 with symBits(.*)[-]{2}" | + egrep -v "^xflate/common.go:(.*)NoCompression should be of the form" | + egrep -v "^exit status") +if [[ ! -z "$RET_LINT" ]]; then echo "$RET_LINT"; echo; fi + +if [[ ! -z "$RET_FMT" ]] || [ ! -z "$RET_TEST" ] || [[ ! -z "$RET_SCHK" ]] || [[ ! -z "$RET_LINT" ]]; then + echo -e "${FAIL}${RESET}"; exit 1 +else + echo -e "${PASS}${RESET}"; exit 0 +fi diff --git a/vendor/github.com/icedream/go-bsdiff/.gitignore b/vendor/github.com/icedream/go-bsdiff/.gitignore new file mode 100644 index 00000000..b99edc31 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/.gitignore @@ -0,0 +1,2 @@ +go-bsdiff/go-bsdiff +go-bspatch/go-bspatch \ No newline at end of file diff --git a/vendor/github.com/icedream/go-bsdiff/README.md b/vendor/github.com/icedream/go-bsdiff/README.md new file mode 100644 index 00000000..e17c8616 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/README.md @@ -0,0 +1,89 @@ +# bsdiff for Go + +This wrapper implementation for Golang reuses the existing +C version of bsdiff as provided by @mendsley and wraps it +into a Go package, abstracting away all the cgo work that +would need to be done otherwise. + +## Installation + +The library and the helper binaries `go-bsdiff` and `go-bspatch` can be installed like this: + + go get -v github.com/icedream/go-bsdiff/... + +## Usage in application code + +For exact documentation of the library check out [GoDoc](https://godoc.org/github.com/icedream/go-bsdiff). + +Library functionality is provided both as a package `bsdiff` containing both +methods `Diff` and `Patch`, or as subpackages `diff` and `patch` which each +only link the wanted functionality. + +Below example will generate a patch and apply it again in one go. This code +is not safe against errors but it shows how to use the provided routines: + +```go +package main + +import ( + "os" + "github.com/icedream/go-bsdiff" + // Or use the subpackages to only link what you need: + //"github.com/icedream/go-bsdiff/diff" + //"github.com/icedream/go-bsdiff/patch" +) + +const ( + oldFilePath = "your_old_file.dat" + newFilePath = "your_new_file.dat" + patchFilePath = "the_generated.patch" +) + +func generatePatch() error { + oldFile, _ := os.Open(oldFilePath) + defer oldFile.Close() + newFile, _ := os.Open(newFilePath) + defer newFile.Close() + patchFile, _ := os.Create(patchFilePath) + defer patchFile.Close() + + return bsdiff.Diff(oldFile, newFile, patchFile) +} + +func applyPatch() error { + oldFile, _ := os.Open(oldFilePath) + defer oldFile.Close() + newFile, _ := os.Create(newFilePath) + defer newFile.Close() + patchFile, _ := os.Open(patchFilePath) + defer patchFile.Close() + + return bsdiff.Patch(oldFile, newFile, patchFile) +} + +func main() { + generatePatch() + applyPatch() +} +``` + +## Usage of the tools + +The tools `go-bsdiff` and `go-bspatch` both provide a `--help` flag to print +out all information but in their simplest form, they can be used like this: + +```sh +# Creates a patch file $the_generated with differences from +# $your_old_file to $your_new_file. +go-bsdiff "$your_old_file" "$your_new_file" "$the_generated" + +# Applies a patch file $the_generated on $your_old_file +# and saves the new file to $your_new_file. +go-bspatch "$your_old_file" "$your_new_file" "$the_generated" +``` + +## Motivation + +There is [a Go implementation of an older version of bsdiff called binarydist](https://github.com/kr/binarydist). The original bsdiff tool has since been updated so patches generating using the original tool are no longer compatible with the Go implementation. I don't know what the changes between the versions are and unfortunately I don't have the time to search for these changes and port them over as a pull request, otherwise I'd have done that instead. + +Additionally, @mendsley has already done the extra work of rewriting the code to be embeddable in any application code as a library. So why not make use of cgo, which I was going to look into in more detail at some point anyways? diff --git a/vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE b/vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE new file mode 100644 index 00000000..5d406657 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE @@ -0,0 +1,24 @@ + Copyright 2003-2005 Colin Percival + Copyright 2012 Matthew Endsley + All rights reserved + + Redistribution and use in source and binary forms, with or without + modification, are permitted providing that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/icedream/go-bsdiff/diff/diff.go b/vendor/github.com/icedream/go-bsdiff/diff/diff.go new file mode 100644 index 00000000..1ed4cda5 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/diff/diff.go @@ -0,0 +1,34 @@ +package diff + +import ( + "io" + "io/ioutil" + + "github.com/dsnet/compress/bzip2" + "github.com/icedream/go-bsdiff/internal" + "github.com/icedream/go-bsdiff/internal/native" +) + +func Diff(oldReader, newReader io.Reader, patchWriter io.Writer) (err error) { + oldBytes, err := ioutil.ReadAll(oldReader) + if err != nil { + return + } + newBytes, err := ioutil.ReadAll(newReader) + if err != nil { + return + } + + if err = internal.WriteHeader(patchWriter, uint64(len(newBytes))); err != nil { + return + } + + // Compression + bz2Writer, err := bzip2.NewWriter(patchWriter, nil) + if err != nil { + return + } + defer bz2Writer.Close() + + return native.Diff(oldBytes, newBytes, bz2Writer) +} diff --git a/vendor/github.com/icedream/go-bsdiff/go.mod b/vendor/github.com/icedream/go-bsdiff/go.mod new file mode 100644 index 00000000..d072de8e --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/go.mod @@ -0,0 +1,11 @@ +module github.com/icedream/go-bsdiff + +require ( + github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect + github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.2.2 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 +) diff --git a/vendor/github.com/icedream/go-bsdiff/go.sum b/vendor/github.com/icedream/go-bsdiff/go.sum new file mode 100644 index 00000000..81f370ec --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/go.sum @@ -0,0 +1,14 @@ +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 h1:eX+pdPPlD279OWgdx7f6KqIRSONuK7egk+jDx7OM3Ac= +github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76/go.mod h1:KjxHHirfLaw19iGT70HvVjHQsL1vq1SRQB4yOsAfy2s= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/vendor/github.com/icedream/go-bsdiff/internal/magic.go b/vendor/github.com/icedream/go-bsdiff/internal/magic.go new file mode 100644 index 00000000..21766eb9 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/magic.go @@ -0,0 +1,39 @@ +package internal + +import ( + "encoding/binary" + "errors" + "io" +) + +var ( + ErrInvalidMagic = errors.New("Invalid magic") + + sizeEncoding = binary.BigEndian + + magicText = []byte("ENDSLEY/BSDIFF43") +) + +func WriteHeader(w io.Writer, size uint64) (err error) { + if _, err = w.Write(magicText); err != nil { + return + } + err = binary.Write(w, sizeEncoding, size) + return +} + +func ReadHeader(r io.Reader) (size uint64, err error) { + magicBuf := make([]byte, len(magicText)) + n, err := r.Read(magicBuf) + if err != nil { + return + } + if n < len(magicText) { + err = ErrInvalidMagic + return + } + + err = binary.Read(r, sizeEncoding, &size) + + return +} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c new file mode 100644 index 00000000..576101c2 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c @@ -0,0 +1,56 @@ +#include "cgo.h" + +#include "bsdiff.h" +#include "bspatch.h" + +extern int cgo_write_buffer(int bufferIndex, void* buf, int size); + +int cgo_write(struct bsdiff_stream* stream, + const void* buf, int size) { + struct buffer_table_index* bufferEntry; + + bufferEntry = (struct buffer_table_index*)stream->opaque; + + return cgo_write_buffer(bufferEntry->index, (void*)buf, size); +} + +extern int cgo_read_buffer(int bufferIndex, void* buf, int size); + +int cgo_read(const struct bspatch_stream* stream, + void* buf, int size) { + struct buffer_table_index* bufferEntry; + + bufferEntry = (struct buffer_table_index*)stream->opaque; + + return cgo_read_buffer(bufferEntry->index, buf, size) ; +} + +int bsdiff_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex) +{ + struct bsdiff_stream stream; + stream.malloc = malloc; + stream.free = free; + stream.write = cgo_write; + + struct buffer_table_index bufferEntry; + bufferEntry.index = bufferIndex; + stream.opaque = &bufferEntry; + + return bsdiff(oldptr, oldsize, newptr, newsize, &stream); +} + +int bspatch_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex) +{ + struct bspatch_stream stream; + stream.read = cgo_read; + + struct buffer_table_index bufferEntry; + bufferEntry.index = bufferIndex; + stream.opaque = &bufferEntry; + + return bspatch(oldptr, oldsize, newptr, newsize, &stream); +} \ No newline at end of file diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h new file mode 100644 index 00000000..cf692e83 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h @@ -0,0 +1,15 @@ +#include +#include "stdint.h" + +struct buffer_table_index +{ + int index; +}; + +int bsdiff_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex); + +int bspatch_cgo(uint8_t* oldptr, int64_t oldsize, + uint8_t* newptr, int64_t newsize, + int bufferIndex); diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go new file mode 100644 index 00000000..4e4967a9 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go @@ -0,0 +1,43 @@ +package native + +/* +#include "bspatch.h" +*/ +import "C" +import ( + "io" + "log" + "unsafe" +) + +//export cgo_read_buffer +func cgo_read_buffer(bufferIndex C.int, + bufPtr unsafe.Pointer, length C.int) C.int { + goLength := int(length) + + if goLength == 0 { + return 0 + } + + sourceBuffer := readers.Get(int(bufferIndex)) + targetBuffer := cPtrToSlice(bufPtr, goLength) + + errCode := 0 + offset := 0 + for offset < goLength { + n, err := sourceBuffer.Read(targetBuffer) + + if err == io.EOF { + break + } else if err != nil { + log.Println("cgo_read_buffer failed:", err) + errCode = 1 + break + } + + offset += n + targetBuffer = targetBuffer[n:] + } + + return C.int(errCode) +} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go new file mode 100644 index 00000000..1a4e6bd1 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go @@ -0,0 +1,18 @@ +package native + +/* +#include "bsdiff.h" +*/ +import "C" +import "unsafe" + +//export cgo_write_buffer +func cgo_write_buffer(bufferIndex C.int, + dataPtr unsafe.Pointer, size C.int) C.int { + buffer := writers.Get(int(bufferIndex)) + errCode := 0 + if _, err := buffer.Write(cPtrToSlice(dataPtr, int(size))); err != nil { + errCode = 1 + } + return C.int(errCode) +} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/diff.go b/vendor/github.com/icedream/go-bsdiff/internal/native/diff.go new file mode 100644 index 00000000..13606d46 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/diff.go @@ -0,0 +1,29 @@ +package native + +/* +#cgo CFLAGS: -I../../bsdiff + +#include "bsdiff.h" +#include "cgo.h" +*/ +import "C" +import ( + "errors" + "io" +) + +func Diff(oldbytes, newbytes []byte, patch io.Writer) (err error) { + oldptr, oldsize := bytesToUint8PtrAndSize(oldbytes) + newptr, newsize := bytesToUint8PtrAndSize(newbytes) + + bufferIndex := writers.Add(patch) + defer writers.Free(bufferIndex) + + errCode := int(C.bsdiff_cgo(oldptr, oldsize, newptr, newsize, C.int(bufferIndex))) + if errCode != 0 { + err = errors.New("bsdiff failed") + return + } + + return +} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c b/vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c new file mode 100644 index 00000000..a9558c10 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c @@ -0,0 +1,2 @@ +#include "bsdiff.c" +#include "bspatch.c" \ No newline at end of file diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/native.go b/vendor/github.com/icedream/go-bsdiff/internal/native/native.go new file mode 100644 index 00000000..8a9348f9 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/native.go @@ -0,0 +1,31 @@ +package native + +/* +#include +*/ +import "C" +import ( + "reflect" + "unsafe" +) + +var ( + writers = writerTable{} + readers = readerTable{} +) + +func bytesToUint8PtrAndSize(bytes []byte) (ptr *C.uint8_t, size C.int64_t) { + ptr = (*C.uint8_t)(unsafe.Pointer(&bytes[0])) + size = C.int64_t(int64(len(bytes))) + return +} + +func cPtrToSlice(ptr unsafe.Pointer, size int) []byte { + var slice []byte + sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + sliceHeader.Cap = size + sliceHeader.Len = size + sliceHeader.Data = uintptr(ptr) + + return slice +} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/patch.go b/vendor/github.com/icedream/go-bsdiff/internal/native/patch.go new file mode 100644 index 00000000..78f59d11 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/patch.go @@ -0,0 +1,30 @@ +package native + +/* +#cgo CFLAGS: -I../../bsdiff + +#include "bspatch.h" +#include "cgo.h" +*/ +import "C" +import ( + "errors" + "io" + "strconv" +) + +func Patch(oldbytes, newbytes []byte, patch io.Reader) (err error) { + oldptr, oldsize := bytesToUint8PtrAndSize(oldbytes) + newptr, newsize := bytesToUint8PtrAndSize(newbytes) + + bufferIndex := readers.Add(patch) + defer readers.Free(bufferIndex) + + errCode := int(C.bspatch_cgo(oldptr, oldsize, newptr, newsize, C.int(bufferIndex))) + if errCode != 0 { + err = errors.New("bspatch failed with code " + strconv.Itoa(errCode)) + return + } + + return +} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go b/vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go new file mode 100644 index 00000000..a5efa2ce --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go @@ -0,0 +1,44 @@ +package native + +import ( + "io" + "sync" +) + +type readerTable struct { + nextIndex int + table map[int]io.Reader + mutex sync.Mutex +} + +func (bt *readerTable) Add(reader io.Reader) (index int) { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + if bt.table == nil { + bt.table = map[int]io.Reader{} + } + + index = bt.nextIndex + bt.table[index] = reader + + // TODO - Handle int overflow + + bt.nextIndex++ + + return +} + +func (bt *readerTable) Get(index int) io.Reader { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + return bt.table[index] +} + +func (bt *readerTable) Free(index int) { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + delete(bt.table, index) +} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go b/vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go new file mode 100644 index 00000000..ceaffd1c --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go @@ -0,0 +1,44 @@ +package native + +import ( + "io" + "sync" +) + +type writerTable struct { + nextIndex int + table map[int]io.Writer + mutex sync.Mutex +} + +func (bt *writerTable) Add(writer io.Writer) (index int) { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + if bt.table == nil { + bt.table = map[int]io.Writer{} + } + + index = bt.nextIndex + bt.table[index] = writer + + // TODO - Handle int overflow + + bt.nextIndex++ + + return +} + +func (bt *writerTable) Get(index int) io.Writer { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + return bt.table[index] +} + +func (bt *writerTable) Free(index int) { + bt.mutex.Lock() + defer bt.mutex.Unlock() + + delete(bt.table, index) +} diff --git a/vendor/github.com/icedream/go-bsdiff/main.go b/vendor/github.com/icedream/go-bsdiff/main.go new file mode 100644 index 00000000..2d0e7f35 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/main.go @@ -0,0 +1,16 @@ +package bsdiff + +import ( + "io" + + "github.com/icedream/go-bsdiff/diff" + "github.com/icedream/go-bsdiff/patch" +) + +func Diff(oldReader, newReader io.Reader, patchWriter io.Writer) (err error) { + return diff.Diff(oldReader, newReader, patchWriter) +} + +func Patch(oldReader io.Reader, newWriter io.Writer, patchReader io.Reader) (err error) { + return patch.Patch(oldReader, newWriter, patchReader) +} diff --git a/vendor/github.com/icedream/go-bsdiff/patch/patch.go b/vendor/github.com/icedream/go-bsdiff/patch/patch.go new file mode 100644 index 00000000..eb623082 --- /dev/null +++ b/vendor/github.com/icedream/go-bsdiff/patch/patch.go @@ -0,0 +1,31 @@ +package patch + +import ( + "compress/bzip2" + "io" + "io/ioutil" + + "github.com/icedream/go-bsdiff/internal" + "github.com/icedream/go-bsdiff/internal/native" +) + +func Patch(oldReader io.Reader, newWriter io.Writer, patchReader io.Reader) (err error) { + oldBytes, err := ioutil.ReadAll(oldReader) + if err != nil { + return + } + + newLen, err := internal.ReadHeader(patchReader) + if err != nil { + return + } + newBytes := make([]byte, newLen) + + // Decompression + bz2Reader := bzip2.NewReader(patchReader) + + err = native.Patch(oldBytes, newBytes, bz2Reader) + + newWriter.Write(newBytes) + return +} diff --git a/vendor/github.com/kr/binarydist/.gitignore b/vendor/github.com/kr/binarydist/.gitignore new file mode 100644 index 00000000..653f1601 --- /dev/null +++ b/vendor/github.com/kr/binarydist/.gitignore @@ -0,0 +1 @@ +test.* diff --git a/vendor/github.com/kr/binarydist/License b/vendor/github.com/kr/binarydist/License new file mode 100644 index 00000000..183c3898 --- /dev/null +++ b/vendor/github.com/kr/binarydist/License @@ -0,0 +1,22 @@ +Copyright 2012 Keith Rarick + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/kr/binarydist/Readme.md b/vendor/github.com/kr/binarydist/Readme.md new file mode 100644 index 00000000..dadc3683 --- /dev/null +++ b/vendor/github.com/kr/binarydist/Readme.md @@ -0,0 +1,7 @@ +# binarydist + +Package binarydist implements binary diff and patch as described on +. It reads and writes files +compatible with the tools there. + +Documentation at . diff --git a/vendor/github.com/kr/binarydist/bzip2.go b/vendor/github.com/kr/binarydist/bzip2.go new file mode 100644 index 00000000..a2516b81 --- /dev/null +++ b/vendor/github.com/kr/binarydist/bzip2.go @@ -0,0 +1,40 @@ +package binarydist + +import ( + "io" + "os/exec" +) + +type bzip2Writer struct { + c *exec.Cmd + w io.WriteCloser +} + +func (w bzip2Writer) Write(b []byte) (int, error) { + return w.w.Write(b) +} + +func (w bzip2Writer) Close() error { + if err := w.w.Close(); err != nil { + return err + } + return w.c.Wait() +} + +// Package compress/bzip2 implements only decompression, +// so we'll fake it by running bzip2 in another process. +func newBzip2Writer(w io.Writer) (wc io.WriteCloser, err error) { + var bw bzip2Writer + bw.c = exec.Command("bzip2", "-c") + bw.c.Stdout = w + + if bw.w, err = bw.c.StdinPipe(); err != nil { + return nil, err + } + + if err = bw.c.Start(); err != nil { + return nil, err + } + + return bw, nil +} diff --git a/vendor/github.com/kr/binarydist/diff.go b/vendor/github.com/kr/binarydist/diff.go new file mode 100644 index 00000000..1d2d951b --- /dev/null +++ b/vendor/github.com/kr/binarydist/diff.go @@ -0,0 +1,408 @@ +package binarydist + +import ( + "bytes" + "encoding/binary" + "io" + "io/ioutil" +) + +func swap(a []int, i, j int) { a[i], a[j] = a[j], a[i] } + +func split(I, V []int, start, length, h int) { + var i, j, k, x, jj, kk int + + if length < 16 { + for k = start; k < start+length; k += j { + j = 1 + x = V[I[k]+h] + for i = 1; k+i < start+length; i++ { + if V[I[k+i]+h] < x { + x = V[I[k+i]+h] + j = 0 + } + if V[I[k+i]+h] == x { + swap(I, k+i, k+j) + j++ + } + } + for i = 0; i < j; i++ { + V[I[k+i]] = k + j - 1 + } + if j == 1 { + I[k] = -1 + } + } + return + } + + x = V[I[start+length/2]+h] + jj = 0 + kk = 0 + for i = start; i < start+length; i++ { + if V[I[i]+h] < x { + jj++ + } + if V[I[i]+h] == x { + kk++ + } + } + jj += start + kk += jj + + i = start + j = 0 + k = 0 + for i < jj { + if V[I[i]+h] < x { + i++ + } else if V[I[i]+h] == x { + swap(I, i, jj+j) + j++ + } else { + swap(I, i, kk+k) + k++ + } + } + + for jj+j < kk { + if V[I[jj+j]+h] == x { + j++ + } else { + swap(I, jj+j, kk+k) + k++ + } + } + + if jj > start { + split(I, V, start, jj-start, h) + } + + for i = 0; i < kk-jj; i++ { + V[I[jj+i]] = kk - 1 + } + if jj == kk-1 { + I[jj] = -1 + } + + if start+length > kk { + split(I, V, kk, start+length-kk, h) + } +} + +func qsufsort(obuf []byte) []int { + var buckets [256]int + var i, h int + I := make([]int, len(obuf)+1) + V := make([]int, len(obuf)+1) + + for _, c := range obuf { + buckets[c]++ + } + for i = 1; i < 256; i++ { + buckets[i] += buckets[i-1] + } + copy(buckets[1:], buckets[:]) + buckets[0] = 0 + + for i, c := range obuf { + buckets[c]++ + I[buckets[c]] = i + } + + I[0] = len(obuf) + for i, c := range obuf { + V[i] = buckets[c] + } + + V[len(obuf)] = 0 + for i = 1; i < 256; i++ { + if buckets[i] == buckets[i-1]+1 { + I[buckets[i]] = -1 + } + } + I[0] = -1 + + for h = 1; I[0] != -(len(obuf) + 1); h += h { + var n int + for i = 0; i < len(obuf)+1; { + if I[i] < 0 { + n -= I[i] + i -= I[i] + } else { + if n != 0 { + I[i-n] = -n + } + n = V[I[i]] + 1 - i + split(I, V, i, n, h) + i += n + n = 0 + } + } + if n != 0 { + I[i-n] = -n + } + } + + for i = 0; i < len(obuf)+1; i++ { + I[V[i]] = i + } + return I +} + +func matchlen(a, b []byte) (i int) { + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return i +} + +func search(I []int, obuf, nbuf []byte, st, en int) (pos, n int) { + if en-st < 2 { + x := matchlen(obuf[I[st]:], nbuf) + y := matchlen(obuf[I[en]:], nbuf) + + if x > y { + return I[st], x + } else { + return I[en], y + } + } + + x := st + (en-st)/2 + if bytes.Compare(obuf[I[x]:], nbuf) < 0 { + return search(I, obuf, nbuf, x, en) + } else { + return search(I, obuf, nbuf, st, x) + } + panic("unreached") +} + +// Diff computes the difference between old and new, according to the bsdiff +// algorithm, and writes the result to patch. +func Diff(old, new io.Reader, patch io.Writer) error { + obuf, err := ioutil.ReadAll(old) + if err != nil { + return err + } + + nbuf, err := ioutil.ReadAll(new) + if err != nil { + return err + } + + pbuf, err := diffBytes(obuf, nbuf) + if err != nil { + return err + } + + _, err = patch.Write(pbuf) + return err +} + +func diffBytes(obuf, nbuf []byte) ([]byte, error) { + var patch seekBuffer + err := diff(obuf, nbuf, &patch) + if err != nil { + return nil, err + } + return patch.buf, nil +} + +func diff(obuf, nbuf []byte, patch io.WriteSeeker) error { + var lenf int + I := qsufsort(obuf) + db := make([]byte, len(nbuf)) + eb := make([]byte, len(nbuf)) + var dblen, eblen int + + var hdr header + hdr.Magic = magic + hdr.NewSize = int64(len(nbuf)) + err := binary.Write(patch, signMagLittleEndian{}, &hdr) + if err != nil { + return err + } + + // Compute the differences, writing ctrl as we go + pfbz2, err := newBzip2Writer(patch) + if err != nil { + return err + } + var scan, pos, length int + var lastscan, lastpos, lastoffset int + for scan < len(nbuf) { + var oldscore int + scan += length + for scsc := scan; scan < len(nbuf); scan++ { + pos, length = search(I, obuf, nbuf[scan:], 0, len(obuf)) + + for ; scsc < scan+length; scsc++ { + if scsc+lastoffset < len(obuf) && + obuf[scsc+lastoffset] == nbuf[scsc] { + oldscore++ + } + } + + if (length == oldscore && length != 0) || length > oldscore+8 { + break + } + + if scan+lastoffset < len(obuf) && obuf[scan+lastoffset] == nbuf[scan] { + oldscore-- + } + } + + if length != oldscore || scan == len(nbuf) { + var s, Sf int + lenf = 0 + for i := 0; lastscan+i < scan && lastpos+i < len(obuf); { + if obuf[lastpos+i] == nbuf[lastscan+i] { + s++ + } + i++ + if s*2-i > Sf*2-lenf { + Sf = s + lenf = i + } + } + + lenb := 0 + if scan < len(nbuf) { + var s, Sb int + for i := 1; (scan >= lastscan+i) && (pos >= i); i++ { + if obuf[pos-i] == nbuf[scan-i] { + s++ + } + if s*2-i > Sb*2-lenb { + Sb = s + lenb = i + } + } + } + + if lastscan+lenf > scan-lenb { + overlap := (lastscan + lenf) - (scan - lenb) + s := 0 + Ss := 0 + lens := 0 + for i := 0; i < overlap; i++ { + if nbuf[lastscan+lenf-overlap+i] == obuf[lastpos+lenf-overlap+i] { + s++ + } + if nbuf[scan-lenb+i] == obuf[pos-lenb+i] { + s-- + } + if s > Ss { + Ss = s + lens = i + 1 + } + } + + lenf += lens - overlap + lenb -= lens + } + + for i := 0; i < lenf; i++ { + db[dblen+i] = nbuf[lastscan+i] - obuf[lastpos+i] + } + for i := 0; i < (scan-lenb)-(lastscan+lenf); i++ { + eb[eblen+i] = nbuf[lastscan+lenf+i] + } + + dblen += lenf + eblen += (scan - lenb) - (lastscan + lenf) + + err = binary.Write(pfbz2, signMagLittleEndian{}, int64(lenf)) + if err != nil { + pfbz2.Close() + return err + } + + val := (scan - lenb) - (lastscan + lenf) + err = binary.Write(pfbz2, signMagLittleEndian{}, int64(val)) + if err != nil { + pfbz2.Close() + return err + } + + val = (pos - lenb) - (lastpos + lenf) + err = binary.Write(pfbz2, signMagLittleEndian{}, int64(val)) + if err != nil { + pfbz2.Close() + return err + } + + lastscan = scan - lenb + lastpos = pos - lenb + lastoffset = pos - scan + } + } + err = pfbz2.Close() + if err != nil { + return err + } + + // Compute size of compressed ctrl data + l64, err := patch.Seek(0, 1) + if err != nil { + return err + } + hdr.CtrlLen = int64(l64 - 32) + + // Write compressed diff data + pfbz2, err = newBzip2Writer(patch) + if err != nil { + return err + } + n, err := pfbz2.Write(db[:dblen]) + if err != nil { + pfbz2.Close() + return err + } + if n != dblen { + pfbz2.Close() + return io.ErrShortWrite + } + err = pfbz2.Close() + if err != nil { + return err + } + + // Compute size of compressed diff data + n64, err := patch.Seek(0, 1) + if err != nil { + return err + } + hdr.DiffLen = n64 - l64 + + // Write compressed extra data + pfbz2, err = newBzip2Writer(patch) + if err != nil { + return err + } + n, err = pfbz2.Write(eb[:eblen]) + if err != nil { + pfbz2.Close() + return err + } + if n != eblen { + pfbz2.Close() + return io.ErrShortWrite + } + err = pfbz2.Close() + if err != nil { + return err + } + + // Seek to the beginning, write the header, and close the file + _, err = patch.Seek(0, 0) + if err != nil { + return err + } + err = binary.Write(patch, signMagLittleEndian{}, &hdr) + if err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/kr/binarydist/doc.go b/vendor/github.com/kr/binarydist/doc.go new file mode 100644 index 00000000..3c92d875 --- /dev/null +++ b/vendor/github.com/kr/binarydist/doc.go @@ -0,0 +1,24 @@ +// Package binarydist implements binary diff and patch as described on +// http://www.daemonology.net/bsdiff/. It reads and writes files +// compatible with the tools there. +package binarydist + +var magic = [8]byte{'B', 'S', 'D', 'I', 'F', 'F', '4', '0'} + +// File format: +// 0 8 "BSDIFF40" +// 8 8 X +// 16 8 Y +// 24 8 sizeof(newfile) +// 32 X bzip2(control block) +// 32+X Y bzip2(diff block) +// 32+X+Y ??? bzip2(extra block) +// with control block a set of triples (x,y,z) meaning "add x bytes +// from oldfile to x bytes from the diff block; copy y bytes from the +// extra block; seek forwards in oldfile by z bytes". +type header struct { + Magic [8]byte + CtrlLen int64 + DiffLen int64 + NewSize int64 +} diff --git a/vendor/github.com/kr/binarydist/encoding.go b/vendor/github.com/kr/binarydist/encoding.go new file mode 100644 index 00000000..75ba5856 --- /dev/null +++ b/vendor/github.com/kr/binarydist/encoding.go @@ -0,0 +1,53 @@ +package binarydist + +// SignMagLittleEndian is the numeric encoding used by the bsdiff tools. +// It implements binary.ByteOrder using a sign-magnitude format +// and little-endian byte order. Only methods Uint64 and String +// have been written; the rest panic. +type signMagLittleEndian struct{} + +func (signMagLittleEndian) Uint16(b []byte) uint16 { panic("unimplemented") } + +func (signMagLittleEndian) PutUint16(b []byte, v uint16) { panic("unimplemented") } + +func (signMagLittleEndian) Uint32(b []byte) uint32 { panic("unimplemented") } + +func (signMagLittleEndian) PutUint32(b []byte, v uint32) { panic("unimplemented") } + +func (signMagLittleEndian) Uint64(b []byte) uint64 { + y := int64(b[0]) | + int64(b[1])<<8 | + int64(b[2])<<16 | + int64(b[3])<<24 | + int64(b[4])<<32 | + int64(b[5])<<40 | + int64(b[6])<<48 | + int64(b[7]&0x7f)<<56 + + if b[7]&0x80 != 0 { + y = -y + } + return uint64(y) +} + +func (signMagLittleEndian) PutUint64(b []byte, v uint64) { + x := int64(v) + neg := x < 0 + if neg { + x = -x + } + + b[0] = byte(x) + b[1] = byte(x >> 8) + b[2] = byte(x >> 16) + b[3] = byte(x >> 24) + b[4] = byte(x >> 32) + b[5] = byte(x >> 40) + b[6] = byte(x >> 48) + b[7] = byte(x >> 56) + if neg { + b[7] |= 0x80 + } +} + +func (signMagLittleEndian) String() string { return "signMagLittleEndian" } diff --git a/vendor/github.com/kr/binarydist/go.mod b/vendor/github.com/kr/binarydist/go.mod new file mode 100644 index 00000000..ecdfe3ea --- /dev/null +++ b/vendor/github.com/kr/binarydist/go.mod @@ -0,0 +1 @@ +module "github.com/kr/binarydist" diff --git a/vendor/github.com/kr/binarydist/patch.go b/vendor/github.com/kr/binarydist/patch.go new file mode 100644 index 00000000..eb032257 --- /dev/null +++ b/vendor/github.com/kr/binarydist/patch.go @@ -0,0 +1,109 @@ +package binarydist + +import ( + "bytes" + "compress/bzip2" + "encoding/binary" + "errors" + "io" + "io/ioutil" +) + +var ErrCorrupt = errors.New("corrupt patch") + +// Patch applies patch to old, according to the bspatch algorithm, +// and writes the result to new. +func Patch(old io.Reader, new io.Writer, patch io.Reader) error { + var hdr header + err := binary.Read(patch, signMagLittleEndian{}, &hdr) + if err != nil { + return err + } + if hdr.Magic != magic { + return ErrCorrupt + } + if hdr.CtrlLen < 0 || hdr.DiffLen < 0 || hdr.NewSize < 0 { + return ErrCorrupt + } + + ctrlbuf := make([]byte, hdr.CtrlLen) + _, err = io.ReadFull(patch, ctrlbuf) + if err != nil { + return err + } + cpfbz2 := bzip2.NewReader(bytes.NewReader(ctrlbuf)) + + diffbuf := make([]byte, hdr.DiffLen) + _, err = io.ReadFull(patch, diffbuf) + if err != nil { + return err + } + dpfbz2 := bzip2.NewReader(bytes.NewReader(diffbuf)) + + // The entire rest of the file is the extra block. + epfbz2 := bzip2.NewReader(patch) + + obuf, err := ioutil.ReadAll(old) + if err != nil { + return err + } + + nbuf := make([]byte, hdr.NewSize) + + var oldpos, newpos int64 + for newpos < hdr.NewSize { + var ctrl struct{ Add, Copy, Seek int64 } + err = binary.Read(cpfbz2, signMagLittleEndian{}, &ctrl) + if err != nil { + return err + } + + // Sanity-check + if newpos+ctrl.Add > hdr.NewSize { + return ErrCorrupt + } + + // Read diff string + _, err = io.ReadFull(dpfbz2, nbuf[newpos:newpos+ctrl.Add]) + if err != nil { + return ErrCorrupt + } + + // Add old data to diff string + for i := int64(0); i < ctrl.Add; i++ { + if oldpos+i >= 0 && oldpos+i < int64(len(obuf)) { + nbuf[newpos+i] += obuf[oldpos+i] + } + } + + // Adjust pointers + newpos += ctrl.Add + oldpos += ctrl.Add + + // Sanity-check + if newpos+ctrl.Copy > hdr.NewSize { + return ErrCorrupt + } + + // Read extra string + _, err = io.ReadFull(epfbz2, nbuf[newpos:newpos+ctrl.Copy]) + if err != nil { + return ErrCorrupt + } + + // Adjust pointers + newpos += ctrl.Copy + oldpos += ctrl.Seek + } + + // Write the new file + for len(nbuf) > 0 { + n, err := new.Write(nbuf) + if err != nil { + return err + } + nbuf = nbuf[n:] + } + + return nil +} diff --git a/vendor/github.com/kr/binarydist/seek.go b/vendor/github.com/kr/binarydist/seek.go new file mode 100644 index 00000000..96c03461 --- /dev/null +++ b/vendor/github.com/kr/binarydist/seek.go @@ -0,0 +1,43 @@ +package binarydist + +import ( + "errors" +) + +type seekBuffer struct { + buf []byte + pos int +} + +func (b *seekBuffer) Write(p []byte) (n int, err error) { + n = copy(b.buf[b.pos:], p) + if n == len(p) { + b.pos += n + return n, nil + } + b.buf = append(b.buf, p[n:]...) + b.pos += len(p) + return len(p), nil +} + +func (b *seekBuffer) Seek(offset int64, whence int) (ret int64, err error) { + var abs int64 + switch whence { + case 0: + abs = offset + case 1: + abs = int64(b.pos) + offset + case 2: + abs = int64(len(b.buf)) + offset + default: + return 0, errors.New("binarydist: invalid whence") + } + if abs < 0 { + return 0, errors.New("binarydist: negative position") + } + if abs >= 1<<31 { + return 0, errors.New("binarydist: position out of range") + } + b.pos = int(abs) + return abs, nil +} From a411607a57fae7d8a5390c64947d578ffca196ec Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 18:00:48 +0100 Subject: [PATCH 59/74] misc: added loading boolean flag to api.rest state object --- modules/api_rest/api_rest.go | 3 +++ modules/api_rest/api_rest_replay.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 3543a7f9..23b259e0 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -30,6 +30,7 @@ type RestAPI struct { recording bool recTime int + loading bool replaying bool recordFileName string recordWait *sync.WaitGroup @@ -49,6 +50,7 @@ func NewRestAPI(s *session.Session) *RestAPI { }, recording: false, recTime: 0, + loading: false, replaying: false, recordFileName: "", recordWait: &sync.WaitGroup{}, @@ -57,6 +59,7 @@ func NewRestAPI(s *session.Session) *RestAPI { mod.State.Store("recording", &mod.recording) mod.State.Store("replaying", &mod.replaying) + mod.State.Store("loading", &mod.loading) mod.State.Store("rec_time", &mod.recTime) mod.State.Store("rec_filename", &mod.recordFileName) mod.State.Store("rec_frames", 0) diff --git a/modules/api_rest/api_rest_replay.go b/modules/api_rest/api_rest_replay.go index 787bc406..035b1d3c 100644 --- a/modules/api_rest/api_rest_replay.go +++ b/modules/api_rest/api_rest_replay.go @@ -25,6 +25,11 @@ func (mod *RestAPI) startReplay(filename string) (err error) { return err } + mod.loading = true + defer func() { + mod.loading = false + }() + mod.Info("loading %s ...", mod.recordFileName) start := time.Now() From fdc26ca3aac08b9a7ccbfc7dcadf574c03b289fe Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 19:31:20 +0100 Subject: [PATCH 60/74] misc: reporting session replay loading progress as api.rest state object --- modules/api_rest/api_rest.go | 1 + modules/api_rest/api_rest_record.go | 2 +- modules/api_rest/api_rest_replay.go | 3 +- modules/api_rest/record.go | 54 +++++++++++++++++++++++------ session/module.go | 3 +- 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 23b259e0..b20e8030 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -60,6 +60,7 @@ func NewRestAPI(s *session.Session) *RestAPI { mod.State.Store("recording", &mod.recording) mod.State.Store("replaying", &mod.replaying) mod.State.Store("loading", &mod.loading) + mod.State.Store("load_progress", 0) mod.State.Store("rec_time", &mod.recTime) mod.State.Store("rec_filename", &mod.recordFileName) mod.State.Store("rec_frames", 0) diff --git a/modules/api_rest/api_rest_record.go b/modules/api_rest/api_rest_record.go index ab0be552..22f03cd6 100644 --- a/modules/api_rest/api_rest_record.go +++ b/modules/api_rest/api_rest_record.go @@ -43,7 +43,7 @@ func (mod *RestAPI) recorder() { mod.recTime = 0 mod.recording = true mod.replaying = false - mod.record = NewRecord(mod.recordFileName) + mod.record = NewRecord(mod.recordFileName, &mod.SessionModule) mod.Info("started recording to %s ...", mod.recordFileName) diff --git a/modules/api_rest/api_rest_replay.go b/modules/api_rest/api_rest_replay.go index 035b1d3c..b7072d3e 100644 --- a/modules/api_rest/api_rest_replay.go +++ b/modules/api_rest/api_rest_replay.go @@ -25,6 +25,7 @@ func (mod *RestAPI) startReplay(filename string) (err error) { return err } + mod.State.Store("load_progress", 0) mod.loading = true defer func() { mod.loading = false @@ -33,7 +34,7 @@ func (mod *RestAPI) startReplay(filename string) (err error) { mod.Info("loading %s ...", mod.recordFileName) start := time.Now() - if mod.record, err = LoadRecord(mod.recordFileName); err != nil { + if mod.record, err = LoadRecord(mod.recordFileName, &mod.SessionModule); err != nil { return err } loadedIn := time.Since(start) diff --git a/modules/api_rest/record.go b/modules/api_rest/record.go index 1cc734a4..4693ce2a 100644 --- a/modules/api_rest/record.go +++ b/modules/api_rest/record.go @@ -9,6 +9,8 @@ import ( "os" "sync" + "github.com/bettercap/bettercap/session" + "github.com/evilsocket/islazy/fs" "github.com/kr/binarydist" ) @@ -16,6 +18,8 @@ import ( type patch []byte type frame []byte +type progressCallback func(done int) + type RecordEntry struct { sync.Mutex @@ -25,10 +29,11 @@ type RecordEntry struct { NumStates int `json:"-"` CurState int `json:"-"` - frames []frame + frames []frame + progress progressCallback } -func NewRecordEntry() *RecordEntry { +func NewRecordEntry(progress progressCallback) *RecordEntry { return &RecordEntry{ Data: nil, Cur: nil, @@ -36,6 +41,7 @@ func NewRecordEntry() *RecordEntry { NumStates: 0, CurState: 0, frames: nil, + progress: progress, } } @@ -98,8 +104,12 @@ func (e *RecordEntry) Compile() error { e.Cur = newWriter.Bytes() e.frames[i+1] = e.Cur + + e.progress(1) } + e.progress(1) + return nil } @@ -140,20 +150,34 @@ func (e *RecordEntry) Next() []byte { type Record struct { sync.Mutex - fileName string `json:"-"` - Session *RecordEntry `json:"session"` - Events *RecordEntry `json:"events"` + mod *session.SessionModule `json:"-"` + fileName string `json:"-"` + done int `json:"-"` + total int `json:"-"` + progress float64 `json:"-"` + Session *RecordEntry `json:"session"` + Events *RecordEntry `json:"events"` } -func NewRecord(fileName string) *Record { - return &Record{ +func NewRecord(fileName string, mod *session.SessionModule) *Record { + r := &Record{ fileName: fileName, - Session: NewRecordEntry(), - Events: NewRecordEntry(), + mod: mod, } + + r.Session = NewRecordEntry(r.onProgress) + r.Events = NewRecordEntry(r.onProgress) + + return r } -func LoadRecord(fileName string) (*Record, error) { +func (r *Record) onProgress(done int) { + r.done += done + r.progress = float64(r.done) / float64(r.total) * 100.0 + r.mod.State.Store("load_progress", r.progress) +} + +func LoadRecord(fileName string, mod *session.SessionModule) (*Record, error) { if !fs.Exists(fileName) { return nil, fmt.Errorf("%s does not exist", fileName) } @@ -182,6 +206,16 @@ func LoadRecord(fileName string) (*Record, error) { } rec.fileName = fileName + rec.mod = mod + + rec.Session.NumStates = len(rec.Session.States) + rec.Session.progress = rec.onProgress + rec.Events.NumStates = len(rec.Events.States) + rec.Events.progress = rec.onProgress + + rec.done = 0 + rec.total = rec.Session.NumStates + rec.Events.NumStates + 2 + rec.progress = 0.0 // reset state and precompute frames if err = rec.Session.Compile(); err != nil { diff --git a/session/module.go b/session/module.go index 149552d1..f8ba5e75 100644 --- a/session/module.go +++ b/session/module.go @@ -60,7 +60,7 @@ type SessionModule struct { Session *Session Started bool StatusLock *sync.RWMutex - State sync.Map + State *sync.Map handlers []ModuleHandler params map[string]*ModuleParam @@ -77,6 +77,7 @@ func NewSessionModule(name string, s *Session) SessionModule { Session: s, Started: false, StatusLock: &sync.RWMutex{}, + State: &sync.Map{}, handlers: make([]ModuleHandler, 0), params: make(map[string]*ModuleParam), From 4d5876db2fd11df91118c26096cbd9eda26c76cb Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 19:45:14 +0100 Subject: [PATCH 61/74] misc: small fix or general refactoring i did not bother commenting --- Gopkg.lock | 30 - Gopkg.toml | 4 - vendor/github.com/dsnet/compress/.travis.yml | 36 - vendor/github.com/dsnet/compress/LICENSE.md | 24 - vendor/github.com/dsnet/compress/README.md | 75 -- vendor/github.com/dsnet/compress/api.go | 74 -- vendor/github.com/dsnet/compress/bzip2/bwt.go | 110 --- .../github.com/dsnet/compress/bzip2/common.go | 110 --- .../dsnet/compress/bzip2/fuzz_off.go | 13 - .../dsnet/compress/bzip2/fuzz_on.go | 77 -- .../compress/bzip2/internal/sais/common.go | 28 - .../compress/bzip2/internal/sais/sais_byte.go | 661 ---------------- .../compress/bzip2/internal/sais/sais_gen.go | 703 ------------------ .../compress/bzip2/internal/sais/sais_int.go | 661 ---------------- .../dsnet/compress/bzip2/mtf_rle2.go | 131 ---- .../github.com/dsnet/compress/bzip2/prefix.go | 374 ---------- .../github.com/dsnet/compress/bzip2/reader.go | 274 ------- .../github.com/dsnet/compress/bzip2/rle1.go | 101 --- .../github.com/dsnet/compress/bzip2/writer.go | 307 -------- vendor/github.com/dsnet/compress/go.mod | 10 - vendor/github.com/dsnet/compress/go.sum | 8 - .../dsnet/compress/internal/common.go | 107 --- .../dsnet/compress/internal/debug.go | 12 - .../dsnet/compress/internal/errors/errors.go | 120 --- .../dsnet/compress/internal/gofuzz.go | 12 - .../dsnet/compress/internal/prefix/debug.go | 159 ---- .../dsnet/compress/internal/prefix/decoder.go | 136 ---- .../dsnet/compress/internal/prefix/encoder.go | 66 -- .../dsnet/compress/internal/prefix/prefix.go | 400 ---------- .../dsnet/compress/internal/prefix/range.go | 93 --- .../dsnet/compress/internal/prefix/reader.go | 335 --------- .../dsnet/compress/internal/prefix/wrap.go | 146 ---- .../dsnet/compress/internal/prefix/writer.go | 166 ----- .../dsnet/compress/internal/release.go | 21 - vendor/github.com/dsnet/compress/zbench.sh | 12 - vendor/github.com/dsnet/compress/zfuzz.sh | 10 - vendor/github.com/dsnet/compress/zprof.sh | 54 -- vendor/github.com/dsnet/compress/ztest.sh | 54 -- .../github.com/icedream/go-bsdiff/.gitignore | 2 - .../github.com/icedream/go-bsdiff/README.md | 89 --- .../icedream/go-bsdiff/bsdiff/LICENSE | 24 - .../icedream/go-bsdiff/diff/diff.go | 34 - vendor/github.com/icedream/go-bsdiff/go.mod | 11 - vendor/github.com/icedream/go-bsdiff/go.sum | 14 - .../icedream/go-bsdiff/internal/magic.go | 39 - .../icedream/go-bsdiff/internal/native/cgo.c | 56 -- .../icedream/go-bsdiff/internal/native/cgo.h | 15 - .../go-bsdiff/internal/native/cgo_read.go | 43 -- .../go-bsdiff/internal/native/cgo_write.go | 18 - .../go-bsdiff/internal/native/diff.go | 29 - .../go-bsdiff/internal/native/ext_bsdiff.c | 2 - .../go-bsdiff/internal/native/native.go | 31 - .../go-bsdiff/internal/native/patch.go | 30 - .../go-bsdiff/internal/native/table_reader.go | 44 -- .../go-bsdiff/internal/native/table_writer.go | 44 -- vendor/github.com/icedream/go-bsdiff/main.go | 16 - .../icedream/go-bsdiff/patch/patch.go | 31 - 57 files changed, 6286 deletions(-) delete mode 100644 vendor/github.com/dsnet/compress/.travis.yml delete mode 100644 vendor/github.com/dsnet/compress/LICENSE.md delete mode 100644 vendor/github.com/dsnet/compress/README.md delete mode 100644 vendor/github.com/dsnet/compress/api.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/bwt.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/common.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/fuzz_off.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/fuzz_on.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/prefix.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/reader.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/rle1.go delete mode 100644 vendor/github.com/dsnet/compress/bzip2/writer.go delete mode 100644 vendor/github.com/dsnet/compress/go.mod delete mode 100644 vendor/github.com/dsnet/compress/go.sum delete mode 100644 vendor/github.com/dsnet/compress/internal/common.go delete mode 100644 vendor/github.com/dsnet/compress/internal/debug.go delete mode 100644 vendor/github.com/dsnet/compress/internal/errors/errors.go delete mode 100644 vendor/github.com/dsnet/compress/internal/gofuzz.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/debug.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/decoder.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/encoder.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/prefix.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/range.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/reader.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/wrap.go delete mode 100644 vendor/github.com/dsnet/compress/internal/prefix/writer.go delete mode 100644 vendor/github.com/dsnet/compress/internal/release.go delete mode 100755 vendor/github.com/dsnet/compress/zbench.sh delete mode 100755 vendor/github.com/dsnet/compress/zfuzz.sh delete mode 100755 vendor/github.com/dsnet/compress/zprof.sh delete mode 100755 vendor/github.com/dsnet/compress/ztest.sh delete mode 100644 vendor/github.com/icedream/go-bsdiff/.gitignore delete mode 100644 vendor/github.com/icedream/go-bsdiff/README.md delete mode 100644 vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE delete mode 100644 vendor/github.com/icedream/go-bsdiff/diff/diff.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/go.mod delete mode 100644 vendor/github.com/icedream/go-bsdiff/go.sum delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/magic.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/diff.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/native.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/patch.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/main.go delete mode 100644 vendor/github.com/icedream/go-bsdiff/patch/patch.go diff --git a/Gopkg.lock b/Gopkg.lock index 53a365ec..3a04de16 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -66,21 +66,6 @@ pruneopts = "UT" revision = "61ca646babef3bd4dea1deb610bfb0005c0a1298" -[[projects]] - digest = "1:d052bda13fd17bd8cf52ccae57b0a03ffeea80cffc2cffc62741235fa34f92cf" - name = "github.com/dsnet/compress" - packages = [ - ".", - "bzip2", - "bzip2/internal/sais", - "internal", - "internal/errors", - "internal/prefix", - ] - pruneopts = "UT" - revision = "da652975a8eea9fa0735aba8056747a751db0bd3" - version = "v0.0.1" - [[projects]] branch = "master" digest = "1:6f9339c912bbdda81302633ad7e99a28dfa5a639c864061f1929510a9a64aa74" @@ -183,20 +168,6 @@ revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d" version = "v1.4.0" -[[projects]] - digest = "1:e5cbd028e1c6f49057ceaa5012b504e7a311e1f49d411a38f2d4d02b718e423a" - name = "github.com/icedream/go-bsdiff" - packages = [ - ".", - "diff", - "internal", - "internal/native", - "patch", - ] - pruneopts = "UT" - revision = "a1d297ebf5e610377602c9b33a70d0dcee9cd4f6" - version = "v1.0.0" - [[projects]] branch = "master" digest = "1:6480de9b8abc75bfb06947e139aa07429dfed37f95a258e90865c4c84a9e188b" @@ -368,7 +339,6 @@ "github.com/google/gopacket/pcapgo", "github.com/gorilla/mux", "github.com/gorilla/websocket", - "github.com/icedream/go-bsdiff", "github.com/inconshreveable/go-vhost", "github.com/jpillora/go-tld", "github.com/kr/binarydist", diff --git a/Gopkg.toml b/Gopkg.toml index fffc8500..55c4e5b9 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -74,10 +74,6 @@ go-tests = true unused-packages = true -[[constraint]] - name = "github.com/icedream/go-bsdiff" - version = "1.0.0" - [[constraint]] name = "github.com/kr/binarydist" version = "0.1.0" diff --git a/vendor/github.com/dsnet/compress/.travis.yml b/vendor/github.com/dsnet/compress/.travis.yml deleted file mode 100644 index 7e79820e..00000000 --- a/vendor/github.com/dsnet/compress/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -sudo: false -language: go -before_install: - - curl -L https://github.com/google/brotli/archive/v1.0.2.tar.gz | tar -zxv - - (cd brotli-1.0.2 && mkdir out && cd out && ../configure-cmake && make && sudo make install) - - rm -rf brotli-1.0.2 - - curl -L https://github.com/facebook/zstd/archive/v1.3.2.tar.gz | tar -zxv - - (cd zstd-1.3.2 && sudo make install) - - rm -rf zstd-1.3.2 - - sudo ldconfig - - mkdir /tmp/go1.12 - - curl -L -s https://dl.google.com/go/go1.12.linux-amd64.tar.gz | tar -zxf - -C /tmp/go1.12 --strip-components 1 - - unset GOROOT - - (GO111MODULE=on /tmp/go1.12/bin/go mod vendor) - - (cd /tmp && GO111MODULE=on /tmp/go1.12/bin/go get golang.org/x/lint/golint@8f45f776aaf18cebc8d65861cc70c33c60471952) - - (cd /tmp && GO111MODULE=on /tmp/go1.12/bin/go get honnef.co/go/tools/cmd/staticcheck@2019.1) -matrix: - include: - - go: 1.9.x - script: - - go test -v -race ./... - - go: 1.10.x - script: - - go test -v -race ./... - - go: 1.11.x - script: - - go test -v -race ./... - - go: 1.12.x - script: - - ./ztest.sh - - go: master - script: - - go test -v -race ./... - allow_failures: - - go: master - fast_finish: true diff --git a/vendor/github.com/dsnet/compress/LICENSE.md b/vendor/github.com/dsnet/compress/LICENSE.md deleted file mode 100644 index 945b396c..00000000 --- a/vendor/github.com/dsnet/compress/LICENSE.md +++ /dev/null @@ -1,24 +0,0 @@ -Copyright © 2015, Joe Tsai and The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. -* Neither the copyright holder nor the names of its contributors may be used to -endorse or promote products derived from this software without specific prior -written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/dsnet/compress/README.md b/vendor/github.com/dsnet/compress/README.md deleted file mode 100644 index 63afb01c..00000000 --- a/vendor/github.com/dsnet/compress/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Collection of compression libraries for Go # - -[![GoDoc](https://godoc.org/github.com/dsnet/compress/cmp?status.svg)](https://godoc.org/github.com/dsnet/compress) -[![Build Status](https://travis-ci.org/dsnet/compress.svg?branch=master)](https://travis-ci.org/dsnet/compress) -[![Report Card](https://goreportcard.com/badge/github.com/dsnet/compress)](https://goreportcard.com/report/github.com/dsnet/compress) - -## Introduction ## - -**NOTE: This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason.** - -This repository hosts a collection of compression related libraries. The goal of this project is to provide pure Go implementations for popular compression algorithms beyond what the Go standard library provides. The goals for these packages are as follows: -* Maintainable: That the code remains well documented, well tested, readable, easy to maintain, and easy to verify that it conforms to the specification for the format being implemented. -* Performant: To be able to compress and decompress within at least 80% of the rates that the C implementations are able to achieve. -* Flexible: That the code provides low-level and fine granularity control over the compression streams similar to what the C APIs would provide. - -Of these three, the first objective is often at odds with the other two objectives and provides interesting challenges. Higher performance can often be achieved by muddling abstraction layers or using non-intuitive low-level primitives. Also, more features and functionality, while useful in some situations, often complicates the API. Thus, this package will attempt to satisfy all the goals, but will defer to favoring maintainability when the performance or flexibility benefits are not significant enough. - - -## Library Status ## - -For the packages available, only some features are currently implemented: - -| Package | Reader | Writer | -| ------- | :----: | :----: | -| brotli | :white_check_mark: | | -| bzip2 | :white_check_mark: | :white_check_mark: | -| flate | :white_check_mark: | | -| xflate | :white_check_mark: | :white_check_mark: | - -This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason. When the library becomes more mature, it is planned to eventually conform to some strict versioning scheme like [Semantic Versioning](http://semver.org/). - -However, in the meanwhile, this library does provide some basic API guarantees. For the types defined below, the method signatures are guaranteed to not change. Note that the author still reserves the right to change the fields within each ```Reader``` and ```Writer``` structs. -```go -type ReaderConfig struct { ... } -type Reader struct { ... } - func NewReader(io.Reader, *ReaderConfig) (*Reader, error) { ... } - func (*Reader) Read([]byte) (int, error) { ... } - func (*Reader) Close() error { ... } - -type WriterConfig struct { ... } -type Writer struct { ... } - func NewWriter(io.Writer, *WriterConfig) (*Writer, error) { ... } - func (*Writer) Write([]byte) (int, error) { ... } - func (*Writer) Close() error { ... } -``` - -To see what work still remains, see the [Task List](https://github.com/dsnet/compress/wiki/Task-List). - -## Performance ## - -See [Performance Metrics](https://github.com/dsnet/compress/wiki/Performance-Metrics). - - -## Frequently Asked Questions ## - -See [Frequently Asked Questions](https://github.com/dsnet/compress/wiki/Frequently-Asked-Questions). - - -## Installation ## - -Run the command: - -```go get -u github.com/dsnet/compress``` - -This library requires `Go1.9` or higher in order to build. - - -## Packages ## - -| Package | Description | -| :------ | :---------- | -| [brotli](http://godoc.org/github.com/dsnet/compress/brotli) | Package brotli implements the Brotli format, described in RFC 7932. | -| [bzip2](http://godoc.org/github.com/dsnet/compress/bzip2) | Package bzip2 implements the BZip2 compressed data format. | -| [flate](http://godoc.org/github.com/dsnet/compress/flate) | Package flate implements the DEFLATE format, described in RFC 1951. | -| [xflate](http://godoc.org/github.com/dsnet/compress/xflate) | Package xflate implements the XFLATE format, an random-access extension to DEFLATE. | diff --git a/vendor/github.com/dsnet/compress/api.go b/vendor/github.com/dsnet/compress/api.go deleted file mode 100644 index f80a9232..00000000 --- a/vendor/github.com/dsnet/compress/api.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// Package compress is a collection of compression libraries. -package compress - -import ( - "bufio" - "io" - - "github.com/dsnet/compress/internal/errors" -) - -// The Error interface identifies all compression related errors. -type Error interface { - error - CompressError() - - // IsDeprecated reports the use of a deprecated and unsupported feature. - IsDeprecated() bool - - // IsCorrupted reports whether the input stream was corrupted. - IsCorrupted() bool -} - -var _ Error = errors.Error{} - -// ByteReader is an interface accepted by all decompression Readers. -// It guarantees that the decompressor never reads more data than is necessary -// from the underlying io.Reader. -type ByteReader interface { - io.Reader - io.ByteReader -} - -var _ ByteReader = (*bufio.Reader)(nil) - -// BufferedReader is an interface accepted by all decompression Readers. -// It guarantees that the decompressor never reads more data than is necessary -// from the underlying io.Reader. Since BufferedReader allows a decompressor -// to peek at bytes further along in the stream without advancing the read -// pointer, decompression can experience a significant performance gain when -// provided a reader that satisfies this interface. Thus, a decompressor will -// prefer this interface over ByteReader for performance reasons. -// -// The bufio.Reader satisfies this interface. -type BufferedReader interface { - io.Reader - - // Buffered returns the number of bytes currently buffered. - // - // This value becomes invalid following the next Read/Discard operation. - Buffered() int - - // Peek returns the next n bytes without advancing the reader. - // - // If Peek returns fewer than n bytes, it also returns an error explaining - // why the peek is short. Peek must support peeking of at least 8 bytes. - // If 0 <= n <= Buffered(), Peek is guaranteed to succeed without reading - // from the underlying io.Reader. - // - // This result becomes invalid following the next Read/Discard operation. - Peek(n int) ([]byte, error) - - // Discard skips the next n bytes, returning the number of bytes discarded. - // - // If Discard skips fewer than n bytes, it also returns an error. - // If 0 <= n <= Buffered(), Discard is guaranteed to succeed without reading - // from the underlying io.Reader. - Discard(n int) (int, error) -} - -var _ BufferedReader = (*bufio.Reader)(nil) diff --git a/vendor/github.com/dsnet/compress/bzip2/bwt.go b/vendor/github.com/dsnet/compress/bzip2/bwt.go deleted file mode 100644 index 44a2541f..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/bwt.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package bzip2 - -import "github.com/dsnet/compress/bzip2/internal/sais" - -// The Burrows-Wheeler Transform implementation used here is based on the -// Suffix Array by Induced Sorting (SA-IS) methodology by Nong, Zhang, and Chan. -// This implementation uses the sais algorithm originally written by Yuta Mori. -// -// The SA-IS algorithm runs in O(n) and outputs a Suffix Array. There is a -// mathematical relationship between Suffix Arrays and the Burrows-Wheeler -// Transform, such that a SA can be converted to a BWT in O(n) time. -// -// References: -// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf -// https://github.com/cscott/compressjs/blob/master/lib/BWT.js -// https://www.quora.com/How-can-I-optimize-burrows-wheeler-transform-and-inverse-transform-to-work-in-O-n-time-O-n-space -type burrowsWheelerTransform struct { - buf []byte - sa []int - perm []uint32 -} - -func (bwt *burrowsWheelerTransform) Encode(buf []byte) (ptr int) { - if len(buf) == 0 { - return -1 - } - - // TODO(dsnet): Find a way to avoid the duplicate input string method. - // We only need to do this because suffix arrays (by definition) only - // operate non-wrapped suffixes of a string. On the other hand, - // the BWT specifically used in bzip2 operate on a strings that wrap-around - // when being sorted. - - // Step 1: Concatenate the input string to itself so that we can use the - // suffix array algorithm for bzip2's variant of BWT. - n := len(buf) - bwt.buf = append(append(bwt.buf[:0], buf...), buf...) - if cap(bwt.sa) < 2*n { - bwt.sa = make([]int, 2*n) - } - t := bwt.buf[:2*n] - sa := bwt.sa[:2*n] - - // Step 2: Compute the suffix array (SA). The input string, t, will not be - // modified, while the results will be written to the output, sa. - sais.ComputeSA(t, sa) - - // Step 3: Convert the SA to a BWT. Since ComputeSA does not mutate the - // input, we have two copies of the input; in buf and buf2. Thus, we write - // the transformation to buf, while using buf2. - var j int - buf2 := t[n:] - for _, i := range sa { - if i < n { - if i == 0 { - ptr = j - i = n - } - buf[j] = buf2[i-1] - j++ - } - } - return ptr -} - -func (bwt *burrowsWheelerTransform) Decode(buf []byte, ptr int) { - if len(buf) == 0 { - return - } - - // Step 1: Compute cumm, where cumm[ch] reports the total number of - // characters that precede the character ch in the alphabet. - var cumm [256]int - for _, v := range buf { - cumm[v]++ - } - var sum int - for i, v := range cumm { - cumm[i] = sum - sum += v - } - - // Step 2: Compute perm, where perm[ptr] contains a pointer to the next - // byte in buf and the next pointer in perm itself. - if cap(bwt.perm) < len(buf) { - bwt.perm = make([]uint32, len(buf)) - } - perm := bwt.perm[:len(buf)] - for i, b := range buf { - perm[cumm[b]] = uint32(i) - cumm[b]++ - } - - // Step 3: Follow each pointer in perm to the next byte, starting with the - // origin pointer. - if cap(bwt.buf) < len(buf) { - bwt.buf = make([]byte, len(buf)) - } - buf2 := bwt.buf[:len(buf)] - i := perm[ptr] - for j := range buf2 { - buf2[j] = buf[i] - i = perm[i] - } - copy(buf, buf2) -} diff --git a/vendor/github.com/dsnet/compress/bzip2/common.go b/vendor/github.com/dsnet/compress/bzip2/common.go deleted file mode 100644 index c6339815..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/common.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// Package bzip2 implements the BZip2 compressed data format. -// -// Canonical C implementation: -// http://bzip.org -// -// Unofficial format specification: -// https://github.com/dsnet/compress/blob/master/doc/bzip2-format.pdf -package bzip2 - -import ( - "fmt" - "hash/crc32" - - "github.com/dsnet/compress/internal" - "github.com/dsnet/compress/internal/errors" -) - -// There does not exist a formal specification of the BZip2 format. As such, -// much of this work is derived by either reverse engineering the original C -// source code or using secondary sources. -// -// Significant amounts of fuzz testing is done to ensure that outputs from -// this package is properly decoded by the C library. Furthermore, we test that -// both this package and the C library agree about what inputs are invalid. -// -// Compression stack: -// Run-length encoding 1 (RLE1) -// Burrows-Wheeler transform (BWT) -// Move-to-front transform (MTF) -// Run-length encoding 2 (RLE2) -// Prefix encoding (PE) -// -// References: -// http://bzip.org/ -// https://en.wikipedia.org/wiki/Bzip2 -// https://code.google.com/p/jbzip2/ - -const ( - BestSpeed = 1 - BestCompression = 9 - DefaultCompression = 6 -) - -const ( - hdrMagic = 0x425a // Hex of "BZ" - blkMagic = 0x314159265359 // BCD of PI - endMagic = 0x177245385090 // BCD of sqrt(PI) - - blockSize = 100000 -) - -func errorf(c int, f string, a ...interface{}) error { - return errors.Error{Code: c, Pkg: "bzip2", Msg: fmt.Sprintf(f, a...)} -} - -func panicf(c int, f string, a ...interface{}) { - errors.Panic(errorf(c, f, a...)) -} - -// errWrap converts a lower-level errors.Error to be one from this package. -// The replaceCode passed in will be used to replace the code for any errors -// with the errors.Invalid code. -// -// For the Reader, set this to errors.Corrupted. -// For the Writer, set this to errors.Internal. -func errWrap(err error, replaceCode int) error { - if cerr, ok := err.(errors.Error); ok { - if errors.IsInvalid(cerr) { - cerr.Code = replaceCode - } - err = errorf(cerr.Code, "%s", cerr.Msg) - } - return err -} - -var errClosed = errorf(errors.Closed, "") - -// crc computes the CRC-32 used by BZip2. -// -// The CRC-32 computation in bzip2 treats bytes as having bits in big-endian -// order. That is, the MSB is read before the LSB. Thus, we can use the -// standard library version of CRC-32 IEEE with some minor adjustments. -// -// The byte array is used as an intermediate buffer to swap the bits of every -// byte of the input. -type crc struct { - val uint32 - buf [256]byte -} - -// update computes the CRC-32 of appending buf to c. -func (c *crc) update(buf []byte) { - cval := internal.ReverseUint32(c.val) - for len(buf) > 0 { - n := len(buf) - if n > len(c.buf) { - n = len(c.buf) - } - for i, b := range buf[:n] { - c.buf[i] = internal.ReverseLUT[b] - } - cval = crc32.Update(cval, crc32.IEEETable, c.buf[:n]) - buf = buf[n:] - } - c.val = internal.ReverseUint32(cval) -} diff --git a/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go b/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go deleted file mode 100644 index ddd32f50..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// +build !gofuzz - -// This file exists to suppress fuzzing details from release builds. - -package bzip2 - -type fuzzReader struct{} - -func (*fuzzReader) updateChecksum(int64, uint32) {} diff --git a/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go b/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go deleted file mode 100644 index 54122351..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2016, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// +build gofuzz - -// This file exists to export internal implementation details for fuzz testing. - -package bzip2 - -func ForwardBWT(buf []byte) (ptr int) { - var bwt burrowsWheelerTransform - return bwt.Encode(buf) -} - -func ReverseBWT(buf []byte, ptr int) { - var bwt burrowsWheelerTransform - bwt.Decode(buf, ptr) -} - -type fuzzReader struct { - Checksums Checksums -} - -// updateChecksum updates Checksums. -// -// If a valid pos is provided, it appends the (pos, val) pair to the slice. -// Otherwise, it will update the last record with the new value. -func (fr *fuzzReader) updateChecksum(pos int64, val uint32) { - if pos >= 0 { - fr.Checksums = append(fr.Checksums, Checksum{pos, val}) - } else { - fr.Checksums[len(fr.Checksums)-1].Value = val - } -} - -type Checksum struct { - Offset int64 // Bit offset of the checksum - Value uint32 // Checksum value -} - -type Checksums []Checksum - -// Apply overwrites all checksum fields in d with the ones in cs. -func (cs Checksums) Apply(d []byte) []byte { - d = append([]byte(nil), d...) - for _, c := range cs { - setU32(d, c.Offset, c.Value) - } - return d -} - -func setU32(d []byte, pos int64, val uint32) { - for i := uint(0); i < 32; i++ { - bpos := uint64(pos) + uint64(i) - d[bpos/8] &= ^byte(1 << (7 - bpos%8)) - d[bpos/8] |= byte(val>>(31-i)) << (7 - bpos%8) - } -} - -// Verify checks that all checksum fields in d matches those in cs. -func (cs Checksums) Verify(d []byte) bool { - for _, c := range cs { - if getU32(d, c.Offset) != c.Value { - return false - } - } - return true -} - -func getU32(d []byte, pos int64) (val uint32) { - for i := uint(0); i < 32; i++ { - bpos := uint64(pos) + uint64(i) - val |= (uint32(d[bpos/8] >> (7 - bpos%8))) << (31 - i) - } - return val -} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go deleted file mode 100644 index cd4eee82..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// Package sais implements a linear time suffix array algorithm. -package sais - -//go:generate go run sais_gen.go byte sais_byte.go -//go:generate go run sais_gen.go int sais_int.go - -// This package ports the C sais implementation by Yuta Mori. The ports are -// located in sais_byte.go and sais_int.go, which are identical to each other -// except for the types. Since Go does not support generics, we use generators to -// create the two files. -// -// References: -// https://sites.google.com/site/yuta256/sais -// https://www.researchgate.net/publication/221313676_Linear_Time_Suffix_Array_Construction_Using_D-Critical_Substrings -// https://www.researchgate.net/publication/224176324_Two_Efficient_Algorithms_for_Linear_Time_Suffix_Array_Construction - -// ComputeSA computes the suffix array of t and places the result in sa. -// Both t and sa must be the same length. -func ComputeSA(t []byte, sa []int) { - if len(sa) != len(t) { - panic("mismatching sizes") - } - computeSA_byte(t, sa, 0, len(t), 256) -} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go deleted file mode 100644 index 01b8529b..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go +++ /dev/null @@ -1,661 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// Code generated by sais_gen.go. DO NOT EDIT. - -// ==================================================== -// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. -// -// Permission is hereby granted, free of charge, to any person -// obtaining a copy of this software and associated documentation -// files (the "Software"), to deal in the Software without -// restriction, including without limitation the rights to use, -// copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -// OTHER DEALINGS IN THE SOFTWARE. -// ==================================================== - -package sais - -func getCounts_byte(T []byte, C []int, n, k int) { - var i int - for i = 0; i < k; i++ { - C[i] = 0 - } - for i = 0; i < n; i++ { - C[T[i]]++ - } -} - -func getBuckets_byte(C, B []int, k int, end bool) { - var i, sum int - if end { - for i = 0; i < k; i++ { - sum += C[i] - B[i] = sum - } - } else { - for i = 0; i < k; i++ { - sum += C[i] - B[i] = sum - C[i] - } - } -} - -func sortLMS1_byte(T []byte, SA, C, B []int, n, k int) { - var b, i, j int - var c0, c1 int - - // Compute SAl. - if &C[0] == &B[0] { - getCounts_byte(T, C, n, k) - } - getBuckets_byte(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - j-- - if int(T[j]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i = 0; i < n; i++ { - if j = SA[i]; j > 0 { - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - if int(T[j]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - SA[i] = 0 - } else if j < 0 { - SA[i] = ^j - } - } - - // Compute SAs. - if &C[0] == &B[0] { - getCounts_byte(T, C, n, k) - } - getBuckets_byte(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i = n - 1; i >= 0; i-- { - if j = SA[i]; j > 0 { - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - b-- - if int(T[j]) > c1 { - SA[b] = ^(j + 1) - } else { - SA[b] = j - } - SA[i] = 0 - } - } -} - -func postProcLMS1_byte(T []byte, SA []int, n, m int) int { - var i, j, p, q, plen, qlen, name int - var c0, c1 int - var diff bool - - // Compact all the sorted substrings into the first m items of SA. - // 2*m must be not larger than n (provable). - for i = 0; SA[i] < 0; i++ { - SA[i] = ^SA[i] - } - if i < m { - for j, i = i, i+1; ; i++ { - if p = SA[i]; p < 0 { - SA[j] = ^p - j++ - SA[i] = 0 - if j == m { - break - } - } - } - } - - // Store the length of all substrings. - i = n - 1 - j = n - 1 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - SA[m+((i+1)>>1)] = j - i - j = i + 1 - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - - // Find the lexicographic names of all substrings. - name = 0 - qlen = 0 - for i, q = 0, n; i < m; i++ { - p = SA[i] - plen = SA[m+(p>>1)] - diff = true - if (plen == qlen) && ((q + plen) < n) { - for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { - } - if j == plen { - diff = false - } - } - if diff { - name++ - q = p - qlen = plen - } - SA[m+(p>>1)] = name - } - return name -} - -func sortLMS2_byte(T []byte, SA, C, B, D []int, n, k int) { - var b, i, j, t, d int - var c0, c1 int - - // Compute SAl. - getBuckets_byte(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - j-- - if int(T[j]) < c1 { - t = 1 - } else { - t = 0 - } - j += n - if t&1 > 0 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i, d = 0, 0; i < n; i++ { - if j = SA[i]; j > 0 { - if n <= j { - d += 1 - j -= n - } - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - t = int(c0) << 1 - if int(T[j]) < c1 { - t |= 1 - } - if D[t] != d { - j += n - D[t] = d - } - if t&1 > 0 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - SA[i] = 0 - } else if j < 0 { - SA[i] = ^j - } - } - for i = n - 1; 0 <= i; i-- { - if SA[i] > 0 { - if SA[i] < n { - SA[i] += n - for j = i - 1; SA[j] < n; j-- { - } - SA[j] -= n - i = j - } - } - } - - // Compute SAs. - getBuckets_byte(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i, d = n-1, d+1; i >= 0; i-- { - if j = SA[i]; j > 0 { - if n <= j { - d += 1 - j -= n - } - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - t = int(c0) << 1 - if int(T[j]) > c1 { - t |= 1 - } - if D[t] != d { - j += n - D[t] = d - } - b-- - if t&1 > 0 { - SA[b] = ^(j + 1) - } else { - SA[b] = j - } - SA[i] = 0 - } - } -} - -func postProcLMS2_byte(SA []int, n, m int) int { - var i, j, d, name int - - // Compact all the sorted LMS substrings into the first m items of SA. - name = 0 - for i = 0; SA[i] < 0; i++ { - j = ^SA[i] - if n <= j { - name += 1 - } - SA[i] = j - } - if i < m { - for d, i = i, i+1; ; i++ { - if j = SA[i]; j < 0 { - j = ^j - if n <= j { - name += 1 - } - SA[d] = j - d++ - SA[i] = 0 - if d == m { - break - } - } - } - } - if name < m { - // Store the lexicographic names. - for i, d = m-1, name+1; 0 <= i; i-- { - if j = SA[i]; n <= j { - j -= n - d-- - } - SA[m+(j>>1)] = d - } - } else { - // Unset flags. - for i = 0; i < m; i++ { - if j = SA[i]; n <= j { - j -= n - SA[i] = j - } - } - } - return name -} - -func induceSA_byte(T []byte, SA, C, B []int, n, k int) { - var b, i, j int - var c0, c1 int - - // Compute SAl. - if &C[0] == &B[0] { - getCounts_byte(T, C, n, k) - } - getBuckets_byte(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - if j > 0 && int(T[j-1]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i = 0; i < n; i++ { - j = SA[i] - SA[i] = ^j - if j > 0 { - j-- - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - if j > 0 && int(T[j-1]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - } - } - - // Compute SAs. - if &C[0] == &B[0] { - getCounts_byte(T, C, n, k) - } - getBuckets_byte(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i = n - 1; i >= 0; i-- { - if j = SA[i]; j > 0 { - j-- - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - b-- - if (j == 0) || (int(T[j-1]) > c1) { - SA[b] = ^j - } else { - SA[b] = j - } - } else { - SA[i] = ^j - } - } -} - -func computeSA_byte(T []byte, SA []int, fs, n, k int) { - const ( - minBucketSize = 512 - sortLMS2Limit = 0x3fffffff - ) - - var C, B, D, RA []int - var bo int // Offset of B relative to SA - var b, i, j, m, p, q, name, newfs int - var c0, c1 int - var flags uint - - if k <= minBucketSize { - C = make([]int, k) - if k <= fs { - bo = n + fs - k - B = SA[bo:] - flags = 1 - } else { - B = make([]int, k) - flags = 3 - } - } else if k <= fs { - C = SA[n+fs-k:] - if k <= fs-k { - bo = n + fs - 2*k - B = SA[bo:] - flags = 0 - } else if k <= 4*minBucketSize { - B = make([]int, k) - flags = 2 - } else { - B = C - flags = 8 - } - } else { - C = make([]int, k) - B = C - flags = 4 | 8 - } - if n <= sortLMS2Limit && 2 <= (n/k) { - if flags&1 > 0 { - if 2*k <= fs-k { - flags |= 32 - } else { - flags |= 16 - } - } else if flags == 0 && 2*k <= (fs-2*k) { - flags |= 32 - } - } - - // Stage 1: Reduce the problem by at least 1/2. - // Sort all the LMS-substrings. - getCounts_byte(T, C, n, k) - getBuckets_byte(C, B, k, true) // Find ends of buckets - for i = 0; i < n; i++ { - SA[i] = 0 - } - b = -1 - i = n - 1 - j = n - m = 0 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - if b >= 0 { - SA[b] = j - } - B[c1]-- - b = B[c1] - j = i - m++ - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - - if m > 1 { - if flags&(16|32) > 0 { - if flags&16 > 0 { - D = make([]int, 2*k) - } else { - D = SA[bo-2*k:] - } - B[T[j+1]]++ - for i, j = 0, 0; i < k; i++ { - j += C[i] - if B[i] != j { - SA[B[i]] += n - } - D[i] = 0 - D[i+k] = 0 - } - sortLMS2_byte(T, SA, C, B, D, n, k) - name = postProcLMS2_byte(SA, n, m) - } else { - sortLMS1_byte(T, SA, C, B, n, k) - name = postProcLMS1_byte(T, SA, n, m) - } - } else if m == 1 { - SA[b] = j + 1 - name = 1 - } else { - name = 0 - } - - // Stage 2: Solve the reduced problem. - // Recurse if names are not yet unique. - if name < m { - newfs = n + fs - 2*m - if flags&(1|4|8) == 0 { - if k+name <= newfs { - newfs -= k - } else { - flags |= 8 - } - } - RA = SA[m+newfs:] - for i, j = m+(n>>1)-1, m-1; m <= i; i-- { - if SA[i] != 0 { - RA[j] = SA[i] - 1 - j-- - } - } - computeSA_int(RA, SA, newfs, m, name) - - i = n - 1 - j = m - 1 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - RA[j] = i + 1 - j-- - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - for i = 0; i < m; i++ { - SA[i] = RA[SA[i]] - } - if flags&4 > 0 { - B = make([]int, k) - C = B - } - if flags&2 > 0 { - B = make([]int, k) - } - } - - // Stage 3: Induce the result for the original problem. - if flags&8 > 0 { - getCounts_byte(T, C, n, k) - } - // Put all left-most S characters into their buckets. - if m > 1 { - getBuckets_byte(C, B, k, true) // Find ends of buckets - i = m - 1 - j = n - p = SA[m-1] - c1 = int(T[p]) - for { - c0 = c1 - q = B[c0] - for q < j { - j-- - SA[j] = 0 - } - for { - j-- - SA[j] = p - if i--; i < 0 { - break - } - p = SA[i] - if c1 = int(T[p]); c1 != c0 { - break - } - } - if i < 0 { - break - } - } - for j > 0 { - j-- - SA[j] = 0 - } - } - induceSA_byte(T, SA, C, B, n, k) -} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go deleted file mode 100644 index 26bf628e..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go +++ /dev/null @@ -1,703 +0,0 @@ -// Copyright 2017, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// +build ignore - -package main - -import ( - "bytes" - "go/format" - "io/ioutil" - "log" - "os" - "text/template" -) - -func main() { - if len(os.Args) != 3 { - log.Fatalf("Usage: %s GO_TYPE OUTPUT_FILE", os.Args[0]) - } - typ := os.Args[1] - path := os.Args[2] - - b := new(bytes.Buffer) - t := template.Must(template.New("source").Parse(source)) - if err := t.Execute(b, struct { - Type, GeneratedMessage string - }{typ, "// Code generated by sais_gen.go. DO NOT EDIT."}); err != nil { - log.Fatalf("Template.Execute error: %v", err) - } - out, err := format.Source(bytes.TrimSpace(b.Bytes())) - if err != nil { - log.Fatalf("format.Source error: %v", err) - } - if err := ioutil.WriteFile(path, out, 0644); err != nil { - log.Fatalf("ioutil.WriteFile error: %v", err) - } -} - -const source = ` -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -{{.GeneratedMessage}} - -// ==================================================== -// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. -// -// Permission is hereby granted, free of charge, to any person -// obtaining a copy of this software and associated documentation -// files (the "Software"), to deal in the Software without -// restriction, including without limitation the rights to use, -// copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -// OTHER DEALINGS IN THE SOFTWARE. -// ==================================================== - -package sais - -func getCounts_{{.Type}}(T []{{.Type}}, C []int, n, k int) { - var i int - for i = 0; i < k; i++ { - C[i] = 0 - } - for i = 0; i < n; i++ { - C[T[i]]++ - } -} - -func getBuckets_{{.Type}}(C, B []int, k int, end bool) { - var i, sum int - if end { - for i = 0; i < k; i++ { - sum += C[i] - B[i] = sum - } - } else { - for i = 0; i < k; i++ { - sum += C[i] - B[i] = sum - C[i] - } - } -} - -func sortLMS1_{{.Type}}(T []{{.Type}}, SA, C, B []int, n, k int) { - var b, i, j int - var c0, c1 int - - // Compute SAl. - if &C[0] == &B[0] { - getCounts_{{.Type}}(T, C, n, k) - } - getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - j-- - if int(T[j]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i = 0; i < n; i++ { - if j = SA[i]; j > 0 { - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - if int(T[j]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - SA[i] = 0 - } else if j < 0 { - SA[i] = ^j - } - } - - // Compute SAs. - if &C[0] == &B[0] { - getCounts_{{.Type}}(T, C, n, k) - } - getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i = n - 1; i >= 0; i-- { - if j = SA[i]; j > 0 { - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - b-- - if int(T[j]) > c1 { - SA[b] = ^(j + 1) - } else { - SA[b] = j - } - SA[i] = 0 - } - } -} - -func postProcLMS1_{{.Type}}(T []{{.Type}}, SA []int, n, m int) int { - var i, j, p, q, plen, qlen, name int - var c0, c1 int - var diff bool - - // Compact all the sorted substrings into the first m items of SA. - // 2*m must be not larger than n (provable). - for i = 0; SA[i] < 0; i++ { - SA[i] = ^SA[i] - } - if i < m { - for j, i = i, i+1; ; i++ { - if p = SA[i]; p < 0 { - SA[j] = ^p - j++ - SA[i] = 0 - if j == m { - break - } - } - } - } - - // Store the length of all substrings. - i = n - 1 - j = n - 1 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - SA[m+((i+1)>>1)] = j - i - j = i + 1 - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - - // Find the lexicographic names of all substrings. - name = 0 - qlen = 0 - for i, q = 0, n; i < m; i++ { - p = SA[i] - plen = SA[m+(p>>1)] - diff = true - if (plen == qlen) && ((q + plen) < n) { - for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { - } - if j == plen { - diff = false - } - } - if diff { - name++ - q = p - qlen = plen - } - SA[m+(p>>1)] = name - } - return name -} - -func sortLMS2_{{.Type}}(T []{{.Type}}, SA, C, B, D []int, n, k int) { - var b, i, j, t, d int - var c0, c1 int - - // Compute SAl. - getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - j-- - if int(T[j]) < c1 { - t = 1 - } else { - t = 0 - } - j += n - if t&1 > 0 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i, d = 0, 0; i < n; i++ { - if j = SA[i]; j > 0 { - if n <= j { - d += 1 - j -= n - } - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - t = int(c0) << 1 - if int(T[j]) < c1 { - t |= 1 - } - if D[t] != d { - j += n - D[t] = d - } - if t&1 > 0 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - SA[i] = 0 - } else if j < 0 { - SA[i] = ^j - } - } - for i = n - 1; 0 <= i; i-- { - if SA[i] > 0 { - if SA[i] < n { - SA[i] += n - for j = i - 1; SA[j] < n; j-- { - } - SA[j] -= n - i = j - } - } - } - - // Compute SAs. - getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i, d = n-1, d+1; i >= 0; i-- { - if j = SA[i]; j > 0 { - if n <= j { - d += 1 - j -= n - } - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - t = int(c0) << 1 - if int(T[j]) > c1 { - t |= 1 - } - if D[t] != d { - j += n - D[t] = d - } - b-- - if t&1 > 0 { - SA[b] = ^(j + 1) - } else { - SA[b] = j - } - SA[i] = 0 - } - } -} - -func postProcLMS2_{{.Type}}(SA []int, n, m int) int { - var i, j, d, name int - - // Compact all the sorted LMS substrings into the first m items of SA. - name = 0 - for i = 0; SA[i] < 0; i++ { - j = ^SA[i] - if n <= j { - name += 1 - } - SA[i] = j - } - if i < m { - for d, i = i, i+1; ; i++ { - if j = SA[i]; j < 0 { - j = ^j - if n <= j { - name += 1 - } - SA[d] = j - d++ - SA[i] = 0 - if d == m { - break - } - } - } - } - if name < m { - // Store the lexicographic names. - for i, d = m-1, name+1; 0 <= i; i-- { - if j = SA[i]; n <= j { - j -= n - d-- - } - SA[m+(j>>1)] = d - } - } else { - // Unset flags. - for i = 0; i < m; i++ { - if j = SA[i]; n <= j { - j -= n - SA[i] = j - } - } - } - return name -} - -func induceSA_{{.Type}}(T []{{.Type}}, SA, C, B []int, n, k int) { - var b, i, j int - var c0, c1 int - - // Compute SAl. - if &C[0] == &B[0] { - getCounts_{{.Type}}(T, C, n, k) - } - getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - if j > 0 && int(T[j-1]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i = 0; i < n; i++ { - j = SA[i] - SA[i] = ^j - if j > 0 { - j-- - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - if j > 0 && int(T[j-1]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - } - } - - // Compute SAs. - if &C[0] == &B[0] { - getCounts_{{.Type}}(T, C, n, k) - } - getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i = n - 1; i >= 0; i-- { - if j = SA[i]; j > 0 { - j-- - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - b-- - if (j == 0) || (int(T[j-1]) > c1) { - SA[b] = ^j - } else { - SA[b] = j - } - } else { - SA[i] = ^j - } - } -} - -func computeSA_{{.Type}}(T []{{.Type}}, SA []int, fs, n, k int) { - const ( - minBucketSize = 512 - sortLMS2Limit = 0x3fffffff - ) - - var C, B, D, RA []int - var bo int // Offset of B relative to SA - var b, i, j, m, p, q, name, newfs int - var c0, c1 int - var flags uint - - if k <= minBucketSize { - C = make([]int, k) - if k <= fs { - bo = n + fs - k - B = SA[bo:] - flags = 1 - } else { - B = make([]int, k) - flags = 3 - } - } else if k <= fs { - C = SA[n+fs-k:] - if k <= fs-k { - bo = n + fs - 2*k - B = SA[bo:] - flags = 0 - } else if k <= 4*minBucketSize { - B = make([]int, k) - flags = 2 - } else { - B = C - flags = 8 - } - } else { - C = make([]int, k) - B = C - flags = 4 | 8 - } - if n <= sortLMS2Limit && 2 <= (n/k) { - if flags&1 > 0 { - if 2*k <= fs-k { - flags |= 32 - } else { - flags |= 16 - } - } else if flags == 0 && 2*k <= (fs-2*k) { - flags |= 32 - } - } - - // Stage 1: Reduce the problem by at least 1/2. - // Sort all the LMS-substrings. - getCounts_{{.Type}}(T, C, n, k) - getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets - for i = 0; i < n; i++ { - SA[i] = 0 - } - b = -1 - i = n - 1 - j = n - m = 0 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - if b >= 0 { - SA[b] = j - } - B[c1]-- - b = B[c1] - j = i - m++ - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - - if m > 1 { - if flags&(16|32) > 0 { - if flags&16 > 0 { - D = make([]int, 2*k) - } else { - D = SA[bo-2*k:] - } - B[T[j+1]]++ - for i, j = 0, 0; i < k; i++ { - j += C[i] - if B[i] != j { - SA[B[i]] += n - } - D[i] = 0 - D[i+k] = 0 - } - sortLMS2_{{.Type}}(T, SA, C, B, D, n, k) - name = postProcLMS2_{{.Type}}(SA, n, m) - } else { - sortLMS1_{{.Type}}(T, SA, C, B, n, k) - name = postProcLMS1_{{.Type}}(T, SA, n, m) - } - } else if m == 1 { - SA[b] = j + 1 - name = 1 - } else { - name = 0 - } - - // Stage 2: Solve the reduced problem. - // Recurse if names are not yet unique. - if name < m { - newfs = n + fs - 2*m - if flags&(1|4|8) == 0 { - if k+name <= newfs { - newfs -= k - } else { - flags |= 8 - } - } - RA = SA[m+newfs:] - for i, j = m+(n>>1)-1, m-1; m <= i; i-- { - if SA[i] != 0 { - RA[j] = SA[i] - 1 - j-- - } - } - computeSA_int(RA, SA, newfs, m, name) - - i = n - 1 - j = m - 1 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - RA[j] = i + 1 - j-- - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - for i = 0; i < m; i++ { - SA[i] = RA[SA[i]] - } - if flags&4 > 0 { - B = make([]int, k) - C = B - } - if flags&2 > 0 { - B = make([]int, k) - } - } - - // Stage 3: Induce the result for the original problem. - if flags&8 > 0 { - getCounts_{{.Type}}(T, C, n, k) - } - // Put all left-most S characters into their buckets. - if m > 1 { - getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets - i = m - 1 - j = n - p = SA[m-1] - c1 = int(T[p]) - for { - c0 = c1 - q = B[c0] - for q < j { - j-- - SA[j] = 0 - } - for { - j-- - SA[j] = p - if i--; i < 0 { - break - } - p = SA[i] - if c1 = int(T[p]); c1 != c0 { - break - } - } - if i < 0 { - break - } - } - for j > 0 { - j-- - SA[j] = 0 - } - } - induceSA_{{.Type}}(T, SA, C, B, n, k) -} -` diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go deleted file mode 100644 index 280682f0..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go +++ /dev/null @@ -1,661 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// Code generated by sais_gen.go. DO NOT EDIT. - -// ==================================================== -// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. -// -// Permission is hereby granted, free of charge, to any person -// obtaining a copy of this software and associated documentation -// files (the "Software"), to deal in the Software without -// restriction, including without limitation the rights to use, -// copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -// OTHER DEALINGS IN THE SOFTWARE. -// ==================================================== - -package sais - -func getCounts_int(T []int, C []int, n, k int) { - var i int - for i = 0; i < k; i++ { - C[i] = 0 - } - for i = 0; i < n; i++ { - C[T[i]]++ - } -} - -func getBuckets_int(C, B []int, k int, end bool) { - var i, sum int - if end { - for i = 0; i < k; i++ { - sum += C[i] - B[i] = sum - } - } else { - for i = 0; i < k; i++ { - sum += C[i] - B[i] = sum - C[i] - } - } -} - -func sortLMS1_int(T []int, SA, C, B []int, n, k int) { - var b, i, j int - var c0, c1 int - - // Compute SAl. - if &C[0] == &B[0] { - getCounts_int(T, C, n, k) - } - getBuckets_int(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - j-- - if int(T[j]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i = 0; i < n; i++ { - if j = SA[i]; j > 0 { - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - if int(T[j]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - SA[i] = 0 - } else if j < 0 { - SA[i] = ^j - } - } - - // Compute SAs. - if &C[0] == &B[0] { - getCounts_int(T, C, n, k) - } - getBuckets_int(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i = n - 1; i >= 0; i-- { - if j = SA[i]; j > 0 { - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - b-- - if int(T[j]) > c1 { - SA[b] = ^(j + 1) - } else { - SA[b] = j - } - SA[i] = 0 - } - } -} - -func postProcLMS1_int(T []int, SA []int, n, m int) int { - var i, j, p, q, plen, qlen, name int - var c0, c1 int - var diff bool - - // Compact all the sorted substrings into the first m items of SA. - // 2*m must be not larger than n (provable). - for i = 0; SA[i] < 0; i++ { - SA[i] = ^SA[i] - } - if i < m { - for j, i = i, i+1; ; i++ { - if p = SA[i]; p < 0 { - SA[j] = ^p - j++ - SA[i] = 0 - if j == m { - break - } - } - } - } - - // Store the length of all substrings. - i = n - 1 - j = n - 1 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - SA[m+((i+1)>>1)] = j - i - j = i + 1 - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - - // Find the lexicographic names of all substrings. - name = 0 - qlen = 0 - for i, q = 0, n; i < m; i++ { - p = SA[i] - plen = SA[m+(p>>1)] - diff = true - if (plen == qlen) && ((q + plen) < n) { - for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { - } - if j == plen { - diff = false - } - } - if diff { - name++ - q = p - qlen = plen - } - SA[m+(p>>1)] = name - } - return name -} - -func sortLMS2_int(T []int, SA, C, B, D []int, n, k int) { - var b, i, j, t, d int - var c0, c1 int - - // Compute SAl. - getBuckets_int(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - j-- - if int(T[j]) < c1 { - t = 1 - } else { - t = 0 - } - j += n - if t&1 > 0 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i, d = 0, 0; i < n; i++ { - if j = SA[i]; j > 0 { - if n <= j { - d += 1 - j -= n - } - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - t = int(c0) << 1 - if int(T[j]) < c1 { - t |= 1 - } - if D[t] != d { - j += n - D[t] = d - } - if t&1 > 0 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - SA[i] = 0 - } else if j < 0 { - SA[i] = ^j - } - } - for i = n - 1; 0 <= i; i-- { - if SA[i] > 0 { - if SA[i] < n { - SA[i] += n - for j = i - 1; SA[j] < n; j-- { - } - SA[j] -= n - i = j - } - } - } - - // Compute SAs. - getBuckets_int(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i, d = n-1, d+1; i >= 0; i-- { - if j = SA[i]; j > 0 { - if n <= j { - d += 1 - j -= n - } - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - j-- - t = int(c0) << 1 - if int(T[j]) > c1 { - t |= 1 - } - if D[t] != d { - j += n - D[t] = d - } - b-- - if t&1 > 0 { - SA[b] = ^(j + 1) - } else { - SA[b] = j - } - SA[i] = 0 - } - } -} - -func postProcLMS2_int(SA []int, n, m int) int { - var i, j, d, name int - - // Compact all the sorted LMS substrings into the first m items of SA. - name = 0 - for i = 0; SA[i] < 0; i++ { - j = ^SA[i] - if n <= j { - name += 1 - } - SA[i] = j - } - if i < m { - for d, i = i, i+1; ; i++ { - if j = SA[i]; j < 0 { - j = ^j - if n <= j { - name += 1 - } - SA[d] = j - d++ - SA[i] = 0 - if d == m { - break - } - } - } - } - if name < m { - // Store the lexicographic names. - for i, d = m-1, name+1; 0 <= i; i-- { - if j = SA[i]; n <= j { - j -= n - d-- - } - SA[m+(j>>1)] = d - } - } else { - // Unset flags. - for i = 0; i < m; i++ { - if j = SA[i]; n <= j { - j -= n - SA[i] = j - } - } - } - return name -} - -func induceSA_int(T []int, SA, C, B []int, n, k int) { - var b, i, j int - var c0, c1 int - - // Compute SAl. - if &C[0] == &B[0] { - getCounts_int(T, C, n, k) - } - getBuckets_int(C, B, k, false) // Find starts of buckets - j = n - 1 - c1 = int(T[j]) - b = B[c1] - if j > 0 && int(T[j-1]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - for i = 0; i < n; i++ { - j = SA[i] - SA[i] = ^j - if j > 0 { - j-- - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - if j > 0 && int(T[j-1]) < c1 { - SA[b] = ^j - } else { - SA[b] = j - } - b++ - } - } - - // Compute SAs. - if &C[0] == &B[0] { - getCounts_int(T, C, n, k) - } - getBuckets_int(C, B, k, true) // Find ends of buckets - c1 = 0 - b = B[c1] - for i = n - 1; i >= 0; i-- { - if j = SA[i]; j > 0 { - j-- - if c0 = int(T[j]); c0 != c1 { - B[c1] = b - c1 = c0 - b = B[c1] - } - b-- - if (j == 0) || (int(T[j-1]) > c1) { - SA[b] = ^j - } else { - SA[b] = j - } - } else { - SA[i] = ^j - } - } -} - -func computeSA_int(T []int, SA []int, fs, n, k int) { - const ( - minBucketSize = 512 - sortLMS2Limit = 0x3fffffff - ) - - var C, B, D, RA []int - var bo int // Offset of B relative to SA - var b, i, j, m, p, q, name, newfs int - var c0, c1 int - var flags uint - - if k <= minBucketSize { - C = make([]int, k) - if k <= fs { - bo = n + fs - k - B = SA[bo:] - flags = 1 - } else { - B = make([]int, k) - flags = 3 - } - } else if k <= fs { - C = SA[n+fs-k:] - if k <= fs-k { - bo = n + fs - 2*k - B = SA[bo:] - flags = 0 - } else if k <= 4*minBucketSize { - B = make([]int, k) - flags = 2 - } else { - B = C - flags = 8 - } - } else { - C = make([]int, k) - B = C - flags = 4 | 8 - } - if n <= sortLMS2Limit && 2 <= (n/k) { - if flags&1 > 0 { - if 2*k <= fs-k { - flags |= 32 - } else { - flags |= 16 - } - } else if flags == 0 && 2*k <= (fs-2*k) { - flags |= 32 - } - } - - // Stage 1: Reduce the problem by at least 1/2. - // Sort all the LMS-substrings. - getCounts_int(T, C, n, k) - getBuckets_int(C, B, k, true) // Find ends of buckets - for i = 0; i < n; i++ { - SA[i] = 0 - } - b = -1 - i = n - 1 - j = n - m = 0 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - if b >= 0 { - SA[b] = j - } - B[c1]-- - b = B[c1] - j = i - m++ - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - - if m > 1 { - if flags&(16|32) > 0 { - if flags&16 > 0 { - D = make([]int, 2*k) - } else { - D = SA[bo-2*k:] - } - B[T[j+1]]++ - for i, j = 0, 0; i < k; i++ { - j += C[i] - if B[i] != j { - SA[B[i]] += n - } - D[i] = 0 - D[i+k] = 0 - } - sortLMS2_int(T, SA, C, B, D, n, k) - name = postProcLMS2_int(SA, n, m) - } else { - sortLMS1_int(T, SA, C, B, n, k) - name = postProcLMS1_int(T, SA, n, m) - } - } else if m == 1 { - SA[b] = j + 1 - name = 1 - } else { - name = 0 - } - - // Stage 2: Solve the reduced problem. - // Recurse if names are not yet unique. - if name < m { - newfs = n + fs - 2*m - if flags&(1|4|8) == 0 { - if k+name <= newfs { - newfs -= k - } else { - flags |= 8 - } - } - RA = SA[m+newfs:] - for i, j = m+(n>>1)-1, m-1; m <= i; i-- { - if SA[i] != 0 { - RA[j] = SA[i] - 1 - j-- - } - } - computeSA_int(RA, SA, newfs, m, name) - - i = n - 1 - j = m - 1 - c0 = int(T[n-1]) - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - for i >= 0 { - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 > c1 { - break - } - } - if i >= 0 { - RA[j] = i + 1 - j-- - for { - c1 = c0 - if i--; i < 0 { - break - } - if c0 = int(T[i]); c0 < c1 { - break - } - } - } - } - for i = 0; i < m; i++ { - SA[i] = RA[SA[i]] - } - if flags&4 > 0 { - B = make([]int, k) - C = B - } - if flags&2 > 0 { - B = make([]int, k) - } - } - - // Stage 3: Induce the result for the original problem. - if flags&8 > 0 { - getCounts_int(T, C, n, k) - } - // Put all left-most S characters into their buckets. - if m > 1 { - getBuckets_int(C, B, k, true) // Find ends of buckets - i = m - 1 - j = n - p = SA[m-1] - c1 = int(T[p]) - for { - c0 = c1 - q = B[c0] - for q < j { - j-- - SA[j] = 0 - } - for { - j-- - SA[j] = p - if i--; i < 0 { - break - } - p = SA[i] - if c1 = int(T[p]); c1 != c0 { - break - } - } - if i < 0 { - break - } - } - for j > 0 { - j-- - SA[j] = 0 - } - } - induceSA_int(T, SA, C, B, n, k) -} diff --git a/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go b/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go deleted file mode 100644 index 5c71b343..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package bzip2 - -import "github.com/dsnet/compress/internal/errors" - -// moveToFront implements both the MTF and RLE stages of bzip2 at the same time. -// Any runs of zeros in the encoded output will be replaced by a sequence of -// RUNA and RUNB symbols are encode the length of the run. -// -// The RLE encoding used can actually be encoded to and decoded from using -// normal two's complement arithmetic. The methodology for doing so is below. -// -// Assuming the following: -// num: The value being encoded by RLE encoding. -// run: A sequence of RUNA and RUNB symbols represented as a binary integer, -// where RUNA is the 0 bit, RUNB is the 1 bit, and least-significant RUN -// symbols are at the least-significant bit positions. -// cnt: The number of RUNA and RUNB symbols. -// -// Then the RLE encoding used by bzip2 has this mathematical property: -// num+1 == (1< len(mtf.dictBuf) { - panicf(errors.Internal, "alphabet too large") - } - copy(mtf.dictBuf[:], dict) - mtf.dictLen = len(dict) - mtf.blkSize = blkSize -} - -func (mtf *moveToFront) Encode(vals []byte) (syms []uint16) { - dict := mtf.dictBuf[:mtf.dictLen] - syms = mtf.syms[:0] - - if len(vals) > mtf.blkSize { - panicf(errors.Internal, "exceeded block size") - } - - var lastNum uint32 - for _, val := range vals { - // Normal move-to-front transform. - var idx uint8 // Reverse lookup idx in dict - for di, dv := range dict { - if dv == val { - idx = uint8(di) - break - } - } - copy(dict[1:], dict[:idx]) - dict[0] = val - - // Run-length encoding augmentation. - if idx == 0 { - lastNum++ - continue - } - if lastNum > 0 { - for rc := lastNum + 1; rc != 1; rc >>= 1 { - syms = append(syms, uint16(rc&1)) - } - lastNum = 0 - } - syms = append(syms, uint16(idx)+1) - } - if lastNum > 0 { - for rc := lastNum + 1; rc != 1; rc >>= 1 { - syms = append(syms, uint16(rc&1)) - } - } - mtf.syms = syms - return syms -} - -func (mtf *moveToFront) Decode(syms []uint16) (vals []byte) { - dict := mtf.dictBuf[:mtf.dictLen] - vals = mtf.vals[:0] - - var lastCnt uint - var lastRun uint32 - for _, sym := range syms { - // Run-length encoding augmentation. - if sym < 2 { - lastRun |= uint32(sym) << lastCnt - lastCnt++ - continue - } - if lastCnt > 0 { - cnt := int((1< mtf.blkSize || lastCnt > 24 { - panicf(errors.Corrupted, "run-length decoding exceeded block size") - } - for i := cnt; i > 0; i-- { - vals = append(vals, dict[0]) - } - lastCnt, lastRun = 0, 0 - } - - // Normal move-to-front transform. - val := dict[sym-1] // Forward lookup val in dict - copy(dict[1:], dict[:sym-1]) - dict[0] = val - - if len(vals) >= mtf.blkSize { - panicf(errors.Corrupted, "run-length decoding exceeded block size") - } - vals = append(vals, val) - } - if lastCnt > 0 { - cnt := int((1< mtf.blkSize || lastCnt > 24 { - panicf(errors.Corrupted, "run-length decoding exceeded block size") - } - for i := cnt; i > 0; i-- { - vals = append(vals, dict[0]) - } - } - mtf.vals = vals - return vals -} diff --git a/vendor/github.com/dsnet/compress/bzip2/prefix.go b/vendor/github.com/dsnet/compress/bzip2/prefix.go deleted file mode 100644 index 4847d809..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/prefix.go +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package bzip2 - -import ( - "io" - - "github.com/dsnet/compress/internal" - "github.com/dsnet/compress/internal/errors" - "github.com/dsnet/compress/internal/prefix" -) - -const ( - minNumTrees = 2 - maxNumTrees = 6 - - maxPrefixBits = 20 // Maximum bit-width of a prefix code - maxNumSyms = 256 + 2 // Maximum number of symbols in the alphabet - numBlockSyms = 50 // Number of bytes in a block -) - -// encSel and decSel are used to handle the prefix encoding for tree selectors. -// The prefix encoding is as follows: -// -// Code TreeIdx -// 0 <=> 0 -// 10 <=> 1 -// 110 <=> 2 -// 1110 <=> 3 -// 11110 <=> 4 -// 111110 <=> 5 -// 111111 <=> 6 Invalid tree index, so should fail -// -var encSel, decSel = func() (e prefix.Encoder, d prefix.Decoder) { - var selCodes [maxNumTrees + 1]prefix.PrefixCode - for i := range selCodes { - selCodes[i] = prefix.PrefixCode{Sym: uint32(i), Len: uint32(i + 1)} - } - selCodes[maxNumTrees] = prefix.PrefixCode{Sym: maxNumTrees, Len: maxNumTrees} - prefix.GeneratePrefixes(selCodes[:]) - e.Init(selCodes[:]) - d.Init(selCodes[:]) - return -}() - -type prefixReader struct{ prefix.Reader } - -func (pr *prefixReader) Init(r io.Reader) { - pr.Reader.Init(r, true) -} - -func (pr *prefixReader) ReadBitsBE64(nb uint) uint64 { - if nb <= 32 { - v := uint32(pr.ReadBits(nb)) - return uint64(internal.ReverseUint32N(v, nb)) - } - v0 := internal.ReverseUint32(uint32(pr.ReadBits(32))) - v1 := internal.ReverseUint32(uint32(pr.ReadBits(nb - 32))) - v := uint64(v0)<<32 | uint64(v1) - return v >> (64 - nb) -} - -func (pr *prefixReader) ReadPrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Decoder) { - for i, pc := range codes { - clen := int(pr.ReadBitsBE64(5)) - sum := 1 << maxPrefixBits - for sym := range pc { - for { - if clen < 1 || clen > maxPrefixBits { - panicf(errors.Corrupted, "invalid prefix bit-length: %d", clen) - } - - b, ok := pr.TryReadBits(1) - if !ok { - b = pr.ReadBits(1) - } - if b == 0 { - break - } - - b, ok = pr.TryReadBits(1) - if !ok { - b = pr.ReadBits(1) - } - clen -= int(b*2) - 1 // +1 or -1 - } - pc[sym] = prefix.PrefixCode{Sym: uint32(sym), Len: uint32(clen)} - sum -= (1 << maxPrefixBits) >> uint(clen) - } - - if sum == 0 { - // Fast path, but only handles complete trees. - if err := prefix.GeneratePrefixes(pc); err != nil { - errors.Panic(err) // Using complete trees; should never fail - } - } else { - // Slow path, but handles anything. - pc = handleDegenerateCodes(pc) // Never fails, but may fail later - codes[i] = pc - } - trees[i].Init(pc) - } -} - -type prefixWriter struct{ prefix.Writer } - -func (pw *prefixWriter) Init(w io.Writer) { - pw.Writer.Init(w, true) -} - -func (pw *prefixWriter) WriteBitsBE64(v uint64, nb uint) { - if nb <= 32 { - v := internal.ReverseUint32N(uint32(v), nb) - pw.WriteBits(uint(v), nb) - return - } - v <<= (64 - nb) - v0 := internal.ReverseUint32(uint32(v >> 32)) - v1 := internal.ReverseUint32(uint32(v)) - pw.WriteBits(uint(v0), 32) - pw.WriteBits(uint(v1), nb-32) - return -} - -func (pw *prefixWriter) WritePrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Encoder) { - for i, pc := range codes { - if err := prefix.GeneratePrefixes(pc); err != nil { - errors.Panic(err) // Using complete trees; should never fail - } - trees[i].Init(pc) - - clen := int(pc[0].Len) - pw.WriteBitsBE64(uint64(clen), 5) - for _, c := range pc { - for int(c.Len) < clen { - pw.WriteBits(3, 2) // 11 - clen-- - } - for int(c.Len) > clen { - pw.WriteBits(1, 2) // 10 - clen++ - } - pw.WriteBits(0, 1) - } - } -} - -// handleDegenerateCodes converts a degenerate tree into a canonical tree. -// -// For example, when the input is an under-subscribed tree: -// input: []PrefixCode{ -// {Sym: 0, Len: 3}, -// {Sym: 1, Len: 4}, -// {Sym: 2, Len: 3}, -// } -// output: []PrefixCode{ -// {Sym: 0, Len: 3, Val: 0}, // 000 -// {Sym: 1, Len: 4, Val: 2}, // 0010 -// {Sym: 2, Len: 3, Val: 4}, // 100 -// {Sym: 258, Len: 4, Val: 10}, // 1010 -// {Sym: 259, Len: 3, Val: 6}, // 110 -// {Sym: 260, Len: 1, Val: 1}, // 1 -// } -// -// For example, when the input is an over-subscribed tree: -// input: []PrefixCode{ -// {Sym: 0, Len: 1}, -// {Sym: 1, Len: 3}, -// {Sym: 2, Len: 4}, -// {Sym: 3, Len: 3}, -// {Sym: 4, Len: 2}, -// } -// output: []PrefixCode{ -// {Sym: 0, Len: 1, Val: 0}, // 0 -// {Sym: 1, Len: 3, Val: 3}, // 011 -// {Sym: 3, Len: 3, Val: 7}, // 111 -// {Sym: 4, Len: 2, Val: 1}, // 01 -// } -func handleDegenerateCodes(codes prefix.PrefixCodes) prefix.PrefixCodes { - // Since there is no formal definition for the BZip2 format, there is no - // specification that says that the code lengths must form a complete - // prefix tree (IE: it is neither over-subscribed nor under-subscribed). - // Thus, the original C implementation becomes the reference for how prefix - // decoding is done in these edge cases. Unfortunately, the C version does - // not error when an invalid tree is used, but rather allows decoding to - // continue and only errors if some bit pattern happens to cause an error. - // Thus, it is possible for an invalid tree to end up decoding an input - // "properly" so long as invalid bit patterns are not present. In order to - // replicate this non-specified behavior, we use a ported version of the - // C code to generate the codes as a valid canonical tree by substituting - // invalid nodes with invalid symbols. - // - // ==================================================== - // This program, "bzip2", the associated library "libbzip2", and all - // documentation, are copyright (C) 1996-2010 Julian R Seward. All - // rights reserved. - // - // Redistribution and use in source and binary forms, with or without - // modification, are permitted provided that the following conditions - // are met: - // - // 1. Redistributions of source code must retain the above copyright - // notice, this list of conditions and the following disclaimer. - // - // 2. The origin of this software must not be misrepresented; you must - // not claim that you wrote the original software. If you use this - // software in a product, an acknowledgment in the product - // documentation would be appreciated but is not required. - // - // 3. Altered source versions must be plainly marked as such, and must - // not be misrepresented as being the original software. - // - // 4. The name of the author may not be used to endorse or promote - // products derived from this software without specific prior written - // permission. - // - // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS - // OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE - // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - // - // Julian Seward, jseward@bzip.org - // bzip2/libbzip2 version 1.0.6 of 6 September 2010 - // ==================================================== - var ( - limits [maxPrefixBits + 2]int32 - bases [maxPrefixBits + 2]int32 - perms [maxNumSyms]int32 - - minLen = uint32(maxPrefixBits) - maxLen = uint32(0) - ) - - const ( - statusOkay = iota - statusInvalid - statusNeedBits - statusMaxBits - ) - - // createTables is the BZ2_hbCreateDecodeTables function from the C code. - createTables := func(codes []prefix.PrefixCode) { - for _, c := range codes { - if c.Len > maxLen { - maxLen = c.Len - } - if c.Len < minLen { - minLen = c.Len - } - } - - var pp int - for i := minLen; i <= maxLen; i++ { - for j, c := range codes { - if c.Len == i { - perms[pp] = int32(j) - pp++ - } - } - } - - var vec int32 - for _, c := range codes { - bases[c.Len+1]++ - } - for i := 1; i < len(bases); i++ { - bases[i] += bases[i-1] - } - for i := minLen; i <= maxLen; i++ { - vec += bases[i+1] - bases[i] - limits[i] = vec - 1 - vec <<= 1 - } - for i := minLen + 1; i <= maxLen; i++ { - bases[i] = ((limits[i-1] + 1) << 1) - bases[i] - } - } - - // getSymbol is the GET_MTF_VAL macro from the C code. - getSymbol := func(c prefix.PrefixCode) (uint32, int) { - v := internal.ReverseUint32(c.Val) - n := c.Len - - zn := minLen - if zn > n { - return 0, statusNeedBits - } - zvec := int32(v >> (32 - zn)) - v <<= zn - for { - if zn > maxLen { - return 0, statusMaxBits - } - if zvec <= limits[zn] { - break - } - zn++ - if zn > n { - return 0, statusNeedBits - } - zvec = (zvec << 1) | int32(v>>31) - v <<= 1 - } - if zvec-bases[zn] < 0 || zvec-bases[zn] >= maxNumSyms { - return 0, statusInvalid - } - return uint32(perms[zvec-bases[zn]]), statusOkay - } - - // Step 1: Create the prefix trees using the C algorithm. - createTables(codes) - - // Step 2: Starting with the shortest bit pattern, explore the whole tree. - // If tree is under-subscribed, the worst-case runtime is O(1< 0 { - codes = append(codes, c) - } - } - return codes -} diff --git a/vendor/github.com/dsnet/compress/bzip2/reader.go b/vendor/github.com/dsnet/compress/bzip2/reader.go deleted file mode 100644 index 86d3f718..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/reader.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package bzip2 - -import ( - "io" - - "github.com/dsnet/compress/internal" - "github.com/dsnet/compress/internal/errors" - "github.com/dsnet/compress/internal/prefix" -) - -type Reader struct { - InputOffset int64 // Total number of bytes read from underlying io.Reader - OutputOffset int64 // Total number of bytes emitted from Read - - rd prefixReader - err error - level int // The current compression level - rdHdrFtr int // Number of times we read the stream header and footer - blkCRC uint32 // CRC-32 IEEE of each block (as stored) - endCRC uint32 // Checksum of all blocks using bzip2's custom method - - crc crc - mtf moveToFront - bwt burrowsWheelerTransform - rle runLengthEncoding - - // These fields are allocated with Reader and re-used later. - treeSels []uint8 - codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode - codes1D [maxNumTrees]prefix.PrefixCodes - trees1D [maxNumTrees]prefix.Decoder - syms []uint16 - - fuzzReader // Exported functionality when fuzz testing -} - -type ReaderConfig struct { - _ struct{} // Blank field to prevent unkeyed struct literals -} - -func NewReader(r io.Reader, conf *ReaderConfig) (*Reader, error) { - zr := new(Reader) - zr.Reset(r) - return zr, nil -} - -func (zr *Reader) Reset(r io.Reader) error { - *zr = Reader{ - rd: zr.rd, - - mtf: zr.mtf, - bwt: zr.bwt, - rle: zr.rle, - - treeSels: zr.treeSels, - trees1D: zr.trees1D, - syms: zr.syms, - } - zr.rd.Init(r) - return nil -} - -func (zr *Reader) Read(buf []byte) (int, error) { - for { - cnt, err := zr.rle.Read(buf) - if err != rleDone && zr.err == nil { - zr.err = err - } - if cnt > 0 { - zr.crc.update(buf[:cnt]) - zr.OutputOffset += int64(cnt) - return cnt, nil - } - if zr.err != nil || len(buf) == 0 { - return 0, zr.err - } - - // Read the next chunk. - zr.rd.Offset = zr.InputOffset - func() { - defer errors.Recover(&zr.err) - if zr.rdHdrFtr%2 == 0 { - // Check if we are already at EOF. - if err := zr.rd.PullBits(1); err != nil { - if err == io.ErrUnexpectedEOF && zr.rdHdrFtr > 0 { - err = io.EOF // EOF is okay if we read at least one stream - } - errors.Panic(err) - } - - // Read stream header. - if zr.rd.ReadBitsBE64(16) != hdrMagic { - panicf(errors.Corrupted, "invalid stream magic") - } - if ver := zr.rd.ReadBitsBE64(8); ver != 'h' { - if ver == '0' { - panicf(errors.Deprecated, "bzip1 format is not supported") - } - panicf(errors.Corrupted, "invalid version: %q", ver) - } - lvl := int(zr.rd.ReadBitsBE64(8)) - '0' - if lvl < BestSpeed || lvl > BestCompression { - panicf(errors.Corrupted, "invalid block size: %d", lvl*blockSize) - } - zr.level = lvl - zr.rdHdrFtr++ - } else { - // Check and update the CRC. - if internal.GoFuzz { - zr.updateChecksum(-1, zr.crc.val) // Update with value - zr.blkCRC = zr.crc.val // Suppress CRC failures - } - if zr.blkCRC != zr.crc.val { - panicf(errors.Corrupted, "mismatching block checksum") - } - zr.endCRC = (zr.endCRC<<1 | zr.endCRC>>31) ^ zr.blkCRC - } - buf := zr.decodeBlock() - zr.rle.Init(buf) - }() - if zr.InputOffset, err = zr.rd.Flush(); zr.err == nil { - zr.err = err - } - if zr.err != nil { - zr.err = errWrap(zr.err, errors.Corrupted) - return 0, zr.err - } - } -} - -func (zr *Reader) Close() error { - if zr.err == io.EOF || zr.err == errClosed { - zr.rle.Init(nil) // Make sure future reads fail - zr.err = errClosed - return nil - } - return zr.err // Return the persistent error -} - -func (zr *Reader) decodeBlock() []byte { - if magic := zr.rd.ReadBitsBE64(48); magic != blkMagic { - if magic == endMagic { - endCRC := uint32(zr.rd.ReadBitsBE64(32)) - if internal.GoFuzz { - zr.updateChecksum(zr.rd.BitsRead()-32, zr.endCRC) - endCRC = zr.endCRC // Suppress CRC failures - } - if zr.endCRC != endCRC { - panicf(errors.Corrupted, "mismatching stream checksum") - } - zr.endCRC = 0 - zr.rd.ReadPads() - zr.rdHdrFtr++ - return nil - } - panicf(errors.Corrupted, "invalid block or footer magic") - } - - zr.crc.val = 0 - zr.blkCRC = uint32(zr.rd.ReadBitsBE64(32)) - if internal.GoFuzz { - zr.updateChecksum(zr.rd.BitsRead()-32, 0) // Record offset only - } - if zr.rd.ReadBitsBE64(1) != 0 { - panicf(errors.Deprecated, "block randomization is not supported") - } - - // Read BWT related fields. - ptr := int(zr.rd.ReadBitsBE64(24)) // BWT origin pointer - - // Read MTF related fields. - var dictArr [256]uint8 - dict := dictArr[:0] - bmapHi := uint16(zr.rd.ReadBits(16)) - for i := 0; i < 256; i, bmapHi = i+16, bmapHi>>1 { - if bmapHi&1 > 0 { - bmapLo := uint16(zr.rd.ReadBits(16)) - for j := 0; j < 16; j, bmapLo = j+1, bmapLo>>1 { - if bmapLo&1 > 0 { - dict = append(dict, uint8(i+j)) - } - } - } - } - - // Step 1: Prefix encoding. - syms := zr.decodePrefix(len(dict)) - - // Step 2: Move-to-front transform and run-length encoding. - zr.mtf.Init(dict, zr.level*blockSize) - buf := zr.mtf.Decode(syms) - - // Step 3: Burrows-Wheeler transformation. - if ptr >= len(buf) { - panicf(errors.Corrupted, "origin pointer (0x%06x) exceeds block size: %d", ptr, len(buf)) - } - zr.bwt.Decode(buf, ptr) - - return buf -} - -func (zr *Reader) decodePrefix(numSyms int) (syms []uint16) { - numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOF symbols - if numSyms < 3 { - panicf(errors.Corrupted, "not enough prefix symbols: %d", numSyms) - } - - // Read information about the trees and tree selectors. - var mtf internal.MoveToFront - numTrees := int(zr.rd.ReadBitsBE64(3)) - if numTrees < minNumTrees || numTrees > maxNumTrees { - panicf(errors.Corrupted, "invalid number of prefix trees: %d", numTrees) - } - numSels := int(zr.rd.ReadBitsBE64(15)) - if cap(zr.treeSels) < numSels { - zr.treeSels = make([]uint8, numSels) - } - treeSels := zr.treeSels[:numSels] - for i := range treeSels { - sym, ok := zr.rd.TryReadSymbol(&decSel) - if !ok { - sym = zr.rd.ReadSymbol(&decSel) - } - if int(sym) >= numTrees { - panicf(errors.Corrupted, "invalid prefix tree selector: %d", sym) - } - treeSels[i] = uint8(sym) - } - mtf.Decode(treeSels) - zr.treeSels = treeSels - - // Initialize prefix codes. - for i := range zr.codes2D[:numTrees] { - zr.codes1D[i] = zr.codes2D[i][:numSyms] - } - zr.rd.ReadPrefixCodes(zr.codes1D[:numTrees], zr.trees1D[:numTrees]) - - // Read prefix encoded symbols of compressed data. - var tree *prefix.Decoder - var blkLen, selIdx int - syms = zr.syms[:0] - for { - if blkLen == 0 { - blkLen = numBlockSyms - if selIdx >= len(treeSels) { - panicf(errors.Corrupted, "not enough prefix tree selectors") - } - tree = &zr.trees1D[treeSels[selIdx]] - selIdx++ - } - blkLen-- - sym, ok := zr.rd.TryReadSymbol(tree) - if !ok { - sym = zr.rd.ReadSymbol(tree) - } - - if int(sym) == numSyms-1 { - break // EOF marker - } - if int(sym) >= numSyms { - panicf(errors.Corrupted, "invalid prefix symbol: %d", sym) - } - if len(syms) >= zr.level*blockSize { - panicf(errors.Corrupted, "number of prefix symbols exceeds block size") - } - syms = append(syms, uint16(sym)) - } - zr.syms = syms - return syms -} diff --git a/vendor/github.com/dsnet/compress/bzip2/rle1.go b/vendor/github.com/dsnet/compress/bzip2/rle1.go deleted file mode 100644 index 1d789f65..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/rle1.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package bzip2 - -import "github.com/dsnet/compress/internal/errors" - -// rleDone is a special "error" to indicate that the RLE stage is done. -var rleDone = errorf(errors.Unknown, "RLE1 stage is completed") - -// runLengthEncoding implements the first RLE stage of bzip2. Every sequence -// of 4..255 duplicated bytes is replaced by only the first 4 bytes, and a -// single byte representing the repeat length. Similar to the C bzip2 -// implementation, the encoder will always terminate repeat sequences with a -// count (even if it is the end of the buffer), and it will also never produce -// run lengths of 256..259. The decoder can handle the latter case. -// -// For example, if the input was: -// input: "AAAAAAABBBBCCCD" -// -// Then the output will be: -// output: "AAAA\x03BBBB\x00CCCD" -type runLengthEncoding struct { - buf []byte - idx int - lastVal byte - lastCnt int -} - -func (rle *runLengthEncoding) Init(buf []byte) { - *rle = runLengthEncoding{buf: buf} -} - -func (rle *runLengthEncoding) Write(buf []byte) (int, error) { - for i, b := range buf { - if rle.lastVal != b { - rle.lastCnt = 0 - } - rle.lastCnt++ - switch { - case rle.lastCnt < 4: - if rle.idx >= len(rle.buf) { - return i, rleDone - } - rle.buf[rle.idx] = b - rle.idx++ - case rle.lastCnt == 4: - if rle.idx+1 >= len(rle.buf) { - return i, rleDone - } - rle.buf[rle.idx] = b - rle.idx++ - rle.buf[rle.idx] = 0 - rle.idx++ - case rle.lastCnt < 256: - rle.buf[rle.idx-1]++ - default: - if rle.idx >= len(rle.buf) { - return i, rleDone - } - rle.lastCnt = 1 - rle.buf[rle.idx] = b - rle.idx++ - } - rle.lastVal = b - } - return len(buf), nil -} - -func (rle *runLengthEncoding) Read(buf []byte) (int, error) { - for i := range buf { - switch { - case rle.lastCnt == -4: - if rle.idx >= len(rle.buf) { - return i, errorf(errors.Corrupted, "missing terminating run-length repeater") - } - rle.lastCnt = int(rle.buf[rle.idx]) - rle.idx++ - if rle.lastCnt > 0 { - break // Break the switch - } - fallthrough // Count was zero, continue the work - case rle.lastCnt <= 0: - if rle.idx >= len(rle.buf) { - return i, rleDone - } - b := rle.buf[rle.idx] - rle.idx++ - if b != rle.lastVal { - rle.lastCnt = 0 - rle.lastVal = b - } - } - buf[i] = rle.lastVal - rle.lastCnt-- - } - return len(buf), nil -} - -func (rle *runLengthEncoding) Bytes() []byte { return rle.buf[:rle.idx] } diff --git a/vendor/github.com/dsnet/compress/bzip2/writer.go b/vendor/github.com/dsnet/compress/bzip2/writer.go deleted file mode 100644 index 5c1a4c66..00000000 --- a/vendor/github.com/dsnet/compress/bzip2/writer.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package bzip2 - -import ( - "io" - - "github.com/dsnet/compress/internal" - "github.com/dsnet/compress/internal/errors" - "github.com/dsnet/compress/internal/prefix" -) - -type Writer struct { - InputOffset int64 // Total number of bytes issued to Write - OutputOffset int64 // Total number of bytes written to underlying io.Writer - - wr prefixWriter - err error - level int // The current compression level - wrHdr bool // Have we written the stream header? - blkCRC uint32 // CRC-32 IEEE of each block - endCRC uint32 // Checksum of all blocks using bzip2's custom method - - crc crc - rle runLengthEncoding - bwt burrowsWheelerTransform - mtf moveToFront - - // These fields are allocated with Writer and re-used later. - buf []byte - treeSels []uint8 - treeSelsMTF []uint8 - codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode - codes1D [maxNumTrees]prefix.PrefixCodes - trees1D [maxNumTrees]prefix.Encoder -} - -type WriterConfig struct { - Level int - - _ struct{} // Blank field to prevent unkeyed struct literals -} - -func NewWriter(w io.Writer, conf *WriterConfig) (*Writer, error) { - var lvl int - if conf != nil { - lvl = conf.Level - } - if lvl == 0 { - lvl = DefaultCompression - } - if lvl < BestSpeed || lvl > BestCompression { - return nil, errorf(errors.Invalid, "compression level: %d", lvl) - } - zw := new(Writer) - zw.level = lvl - zw.Reset(w) - return zw, nil -} - -func (zw *Writer) Reset(w io.Writer) error { - *zw = Writer{ - wr: zw.wr, - level: zw.level, - - rle: zw.rle, - bwt: zw.bwt, - mtf: zw.mtf, - - buf: zw.buf, - treeSels: zw.treeSels, - treeSelsMTF: zw.treeSelsMTF, - trees1D: zw.trees1D, - } - zw.wr.Init(w) - if len(zw.buf) != zw.level*blockSize { - zw.buf = make([]byte, zw.level*blockSize) - } - zw.rle.Init(zw.buf) - return nil -} - -func (zw *Writer) Write(buf []byte) (int, error) { - if zw.err != nil { - return 0, zw.err - } - - cnt := len(buf) - for { - wrCnt, err := zw.rle.Write(buf) - if err != rleDone && zw.err == nil { - zw.err = err - } - zw.crc.update(buf[:wrCnt]) - buf = buf[wrCnt:] - if len(buf) == 0 { - zw.InputOffset += int64(cnt) - return cnt, nil - } - if zw.err = zw.flush(); zw.err != nil { - return 0, zw.err - } - } -} - -func (zw *Writer) flush() error { - vals := zw.rle.Bytes() - if len(vals) == 0 { - return nil - } - zw.wr.Offset = zw.OutputOffset - func() { - defer errors.Recover(&zw.err) - if !zw.wrHdr { - // Write stream header. - zw.wr.WriteBitsBE64(hdrMagic, 16) - zw.wr.WriteBitsBE64('h', 8) - zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) - zw.wrHdr = true - } - zw.encodeBlock(vals) - }() - var err error - if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { - zw.err = err - } - if zw.err != nil { - zw.err = errWrap(zw.err, errors.Internal) - return zw.err - } - zw.endCRC = (zw.endCRC<<1 | zw.endCRC>>31) ^ zw.blkCRC - zw.blkCRC = 0 - zw.rle.Init(zw.buf) - return nil -} - -func (zw *Writer) Close() error { - if zw.err == errClosed { - return nil - } - - // Flush RLE buffer if there is left-over data. - if zw.err = zw.flush(); zw.err != nil { - return zw.err - } - - // Write stream footer. - zw.wr.Offset = zw.OutputOffset - func() { - defer errors.Recover(&zw.err) - if !zw.wrHdr { - // Write stream header. - zw.wr.WriteBitsBE64(hdrMagic, 16) - zw.wr.WriteBitsBE64('h', 8) - zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) - zw.wrHdr = true - } - zw.wr.WriteBitsBE64(endMagic, 48) - zw.wr.WriteBitsBE64(uint64(zw.endCRC), 32) - zw.wr.WritePads(0) - }() - var err error - if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { - zw.err = err - } - if zw.err != nil { - zw.err = errWrap(zw.err, errors.Internal) - return zw.err - } - - zw.err = errClosed - return nil -} - -func (zw *Writer) encodeBlock(buf []byte) { - zw.blkCRC = zw.crc.val - zw.wr.WriteBitsBE64(blkMagic, 48) - zw.wr.WriteBitsBE64(uint64(zw.blkCRC), 32) - zw.wr.WriteBitsBE64(0, 1) - zw.crc.val = 0 - - // Step 1: Burrows-Wheeler transformation. - ptr := zw.bwt.Encode(buf) - zw.wr.WriteBitsBE64(uint64(ptr), 24) - - // Step 2: Move-to-front transform and run-length encoding. - var dictMap [256]bool - for _, c := range buf { - dictMap[c] = true - } - - var dictArr [256]uint8 - var bmapLo [16]uint16 - dict := dictArr[:0] - bmapHi := uint16(0) - for i, b := range dictMap { - if b { - c := uint8(i) - dict = append(dict, c) - bmapHi |= 1 << (c >> 4) - bmapLo[c>>4] |= 1 << (c & 0xf) - } - } - - zw.wr.WriteBits(uint(bmapHi), 16) - for _, m := range bmapLo { - if m > 0 { - zw.wr.WriteBits(uint(m), 16) - } - } - - zw.mtf.Init(dict, len(buf)) - syms := zw.mtf.Encode(buf) - - // Step 3: Prefix encoding. - zw.encodePrefix(syms, len(dict)) -} - -func (zw *Writer) encodePrefix(syms []uint16, numSyms int) { - numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOB symbols - if numSyms < 3 { - panicf(errors.Internal, "unable to encode EOB marker") - } - syms = append(syms, uint16(numSyms-1)) // EOB marker - - // Compute number of prefix trees needed. - numTrees := maxNumTrees - for i, lim := range []int{200, 600, 1200, 2400} { - if len(syms) < lim { - numTrees = minNumTrees + i - break - } - } - - // Compute number of block selectors. - numSels := (len(syms) + numBlockSyms - 1) / numBlockSyms - if cap(zw.treeSels) < numSels { - zw.treeSels = make([]uint8, numSels) - } - treeSels := zw.treeSels[:numSels] - for i := range treeSels { - treeSels[i] = uint8(i % numTrees) - } - - // Initialize prefix codes. - for i := range zw.codes2D[:numTrees] { - pc := zw.codes2D[i][:numSyms] - for j := range pc { - pc[j] = prefix.PrefixCode{Sym: uint32(j)} - } - zw.codes1D[i] = pc - } - - // First cut at assigning prefix trees to each group. - var codes prefix.PrefixCodes - var blkLen, selIdx int - for _, sym := range syms { - if blkLen == 0 { - blkLen = numBlockSyms - codes = zw.codes2D[treeSels[selIdx]][:numSyms] - selIdx++ - } - blkLen-- - codes[sym].Cnt++ - } - - // TODO(dsnet): Use K-means to cluster groups to each prefix tree. - - // Generate lengths and prefixes based on symbol frequencies. - for i := range zw.trees1D[:numTrees] { - pc := prefix.PrefixCodes(zw.codes2D[i][:numSyms]) - pc.SortByCount() - if err := prefix.GenerateLengths(pc, maxPrefixBits); err != nil { - errors.Panic(err) - } - pc.SortBySymbol() - } - - // Write out information about the trees and tree selectors. - var mtf internal.MoveToFront - zw.wr.WriteBitsBE64(uint64(numTrees), 3) - zw.wr.WriteBitsBE64(uint64(numSels), 15) - zw.treeSelsMTF = append(zw.treeSelsMTF[:0], treeSels...) - mtf.Encode(zw.treeSelsMTF) - for _, sym := range zw.treeSelsMTF { - zw.wr.WriteSymbol(uint(sym), &encSel) - } - zw.wr.WritePrefixCodes(zw.codes1D[:numTrees], zw.trees1D[:numTrees]) - - // Write out prefix encoded symbols of compressed data. - var tree *prefix.Encoder - blkLen, selIdx = 0, 0 - for _, sym := range syms { - if blkLen == 0 { - blkLen = numBlockSyms - tree = &zw.trees1D[treeSels[selIdx]] - selIdx++ - } - blkLen-- - ok := zw.wr.TryWriteSymbol(uint(sym), tree) - if !ok { - zw.wr.WriteSymbol(uint(sym), tree) - } - } -} diff --git a/vendor/github.com/dsnet/compress/go.mod b/vendor/github.com/dsnet/compress/go.mod deleted file mode 100644 index 7a0bc001..00000000 --- a/vendor/github.com/dsnet/compress/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module github.com/dsnet/compress - -go 1.9 - -require ( - github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 - github.com/klauspost/compress v1.4.1 - github.com/klauspost/cpuid v1.2.0 // indirect - github.com/ulikunitz/xz v0.5.6 -) diff --git a/vendor/github.com/dsnet/compress/go.sum b/vendor/github.com/dsnet/compress/go.sum deleted file mode 100644 index b6fd40c7..00000000 --- a/vendor/github.com/dsnet/compress/go.sum +++ /dev/null @@ -1,8 +0,0 @@ -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= diff --git a/vendor/github.com/dsnet/compress/internal/common.go b/vendor/github.com/dsnet/compress/internal/common.go deleted file mode 100644 index da4e7034..00000000 --- a/vendor/github.com/dsnet/compress/internal/common.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// Package internal is a collection of common compression algorithms. -// -// For performance reasons, these packages lack strong error checking and -// require that the caller to ensure that strict invariants are kept. -package internal - -var ( - // IdentityLUT returns the input key itself. - IdentityLUT = func() (lut [256]byte) { - for i := range lut { - lut[i] = uint8(i) - } - return lut - }() - - // ReverseLUT returns the input key with its bits reversed. - ReverseLUT = func() (lut [256]byte) { - for i := range lut { - b := uint8(i) - b = (b&0xaa)>>1 | (b&0x55)<<1 - b = (b&0xcc)>>2 | (b&0x33)<<2 - b = (b&0xf0)>>4 | (b&0x0f)<<4 - lut[i] = b - } - return lut - }() -) - -// ReverseUint32 reverses all bits of v. -func ReverseUint32(v uint32) (x uint32) { - x |= uint32(ReverseLUT[byte(v>>0)]) << 24 - x |= uint32(ReverseLUT[byte(v>>8)]) << 16 - x |= uint32(ReverseLUT[byte(v>>16)]) << 8 - x |= uint32(ReverseLUT[byte(v>>24)]) << 0 - return x -} - -// ReverseUint32N reverses the lower n bits of v. -func ReverseUint32N(v uint32, n uint) (x uint32) { - return ReverseUint32(v << (32 - n)) -} - -// ReverseUint64 reverses all bits of v. -func ReverseUint64(v uint64) (x uint64) { - x |= uint64(ReverseLUT[byte(v>>0)]) << 56 - x |= uint64(ReverseLUT[byte(v>>8)]) << 48 - x |= uint64(ReverseLUT[byte(v>>16)]) << 40 - x |= uint64(ReverseLUT[byte(v>>24)]) << 32 - x |= uint64(ReverseLUT[byte(v>>32)]) << 24 - x |= uint64(ReverseLUT[byte(v>>40)]) << 16 - x |= uint64(ReverseLUT[byte(v>>48)]) << 8 - x |= uint64(ReverseLUT[byte(v>>56)]) << 0 - return x -} - -// ReverseUint64N reverses the lower n bits of v. -func ReverseUint64N(v uint64, n uint) (x uint64) { - return ReverseUint64(v << (64 - n)) -} - -// MoveToFront is a data structure that allows for more efficient move-to-front -// transformations. This specific implementation assumes that the alphabet is -// densely packed within 0..255. -type MoveToFront struct { - dict [256]uint8 // Mapping from indexes to values - tail int // Number of tail bytes that are already ordered -} - -func (m *MoveToFront) Encode(vals []uint8) { - copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity - - var max int - for i, val := range vals { - var idx uint8 // Reverse lookup idx in dict - for di, dv := range m.dict { - if dv == val { - idx = uint8(di) - break - } - } - vals[i] = idx - - max |= int(idx) - copy(m.dict[1:], m.dict[:idx]) - m.dict[0] = val - } - m.tail = 256 - max - 1 -} - -func (m *MoveToFront) Decode(idxs []uint8) { - copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity - - var max int - for i, idx := range idxs { - val := m.dict[idx] // Forward lookup val in dict - idxs[i] = val - - max |= int(idx) - copy(m.dict[1:], m.dict[:idx]) - m.dict[0] = val - } - m.tail = 256 - max - 1 -} diff --git a/vendor/github.com/dsnet/compress/internal/debug.go b/vendor/github.com/dsnet/compress/internal/debug.go deleted file mode 100644 index 01df1f89..00000000 --- a/vendor/github.com/dsnet/compress/internal/debug.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// +build debug,!gofuzz - -package internal - -const ( - Debug = true - GoFuzz = false -) diff --git a/vendor/github.com/dsnet/compress/internal/errors/errors.go b/vendor/github.com/dsnet/compress/internal/errors/errors.go deleted file mode 100644 index c631afbd..00000000 --- a/vendor/github.com/dsnet/compress/internal/errors/errors.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2016, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// Package errors implements functions to manipulate compression errors. -// -// In idiomatic Go, it is an anti-pattern to use panics as a form of error -// reporting in the API. Instead, the expected way to transmit errors is by -// returning an error value. Unfortunately, the checking of "err != nil" in -// tight loops commonly found in compression causes non-negligible performance -// degradation. While this may not be idiomatic, the internal packages of this -// repository rely on panics as a normal means to convey errors. In order to -// ensure that these panics do not leak across the public API, the public -// packages must recover from these panics and present an error value. -// -// The Panic and Recover functions in this package provide a safe way to -// recover from errors only generated from within this repository. -// -// Example usage: -// func Foo() (err error) { -// defer errors.Recover(&err) -// -// if rand.Intn(2) == 0 { -// // Unexpected panics will not be caught by Recover. -// io.Closer(nil).Close() -// } else { -// // Errors thrown by Panic will be caught by Recover. -// errors.Panic(errors.New("whoopsie")) -// } -// } -// -package errors - -import "strings" - -const ( - // Unknown indicates that there is no classification for this error. - Unknown = iota - - // Internal indicates that this error is due to an internal bug. - // Users should file a issue report if this type of error is encountered. - Internal - - // Invalid indicates that this error is due to the user misusing the API - // and is indicative of a bug on the user's part. - Invalid - - // Deprecated indicates the use of a deprecated and unsupported feature. - Deprecated - - // Corrupted indicates that the input stream is corrupted. - Corrupted - - // Closed indicates that the handlers are closed. - Closed -) - -var codeMap = map[int]string{ - Unknown: "unknown error", - Internal: "internal error", - Invalid: "invalid argument", - Deprecated: "deprecated format", - Corrupted: "corrupted input", - Closed: "closed handler", -} - -type Error struct { - Code int // The error type - Pkg string // Name of the package where the error originated - Msg string // Descriptive message about the error (optional) -} - -func (e Error) Error() string { - var ss []string - for _, s := range []string{e.Pkg, codeMap[e.Code], e.Msg} { - if s != "" { - ss = append(ss, s) - } - } - return strings.Join(ss, ": ") -} - -func (e Error) CompressError() {} -func (e Error) IsInternal() bool { return e.Code == Internal } -func (e Error) IsInvalid() bool { return e.Code == Invalid } -func (e Error) IsDeprecated() bool { return e.Code == Deprecated } -func (e Error) IsCorrupted() bool { return e.Code == Corrupted } -func (e Error) IsClosed() bool { return e.Code == Closed } - -func IsInternal(err error) bool { return isCode(err, Internal) } -func IsInvalid(err error) bool { return isCode(err, Invalid) } -func IsDeprecated(err error) bool { return isCode(err, Deprecated) } -func IsCorrupted(err error) bool { return isCode(err, Corrupted) } -func IsClosed(err error) bool { return isCode(err, Closed) } - -func isCode(err error, code int) bool { - if cerr, ok := err.(Error); ok && cerr.Code == code { - return true - } - return false -} - -// errWrap is used by Panic and Recover to ensure that only errors raised by -// Panic are recovered by Recover. -type errWrap struct{ e *error } - -func Recover(err *error) { - switch ex := recover().(type) { - case nil: - // Do nothing. - case errWrap: - *err = *ex.e - default: - panic(ex) - } -} - -func Panic(err error) { - panic(errWrap{&err}) -} diff --git a/vendor/github.com/dsnet/compress/internal/gofuzz.go b/vendor/github.com/dsnet/compress/internal/gofuzz.go deleted file mode 100644 index 5035c9d6..00000000 --- a/vendor/github.com/dsnet/compress/internal/gofuzz.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2016, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// +build gofuzz - -package internal - -const ( - Debug = true - GoFuzz = true -) diff --git a/vendor/github.com/dsnet/compress/internal/prefix/debug.go b/vendor/github.com/dsnet/compress/internal/prefix/debug.go deleted file mode 100644 index 04fce70b..00000000 --- a/vendor/github.com/dsnet/compress/internal/prefix/debug.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// +build debug - -package prefix - -import ( - "fmt" - "math" - "strings" -) - -func max(a, b int) int { - if a > b { - return a - } - return b -} - -func lenBase2(n uint) int { - return int(math.Ceil(math.Log2(float64(n + 1)))) -} -func padBase2(v, n uint, m int) string { - s := fmt.Sprintf("%b", 1< 0 { - return strings.Repeat(" ", pad) + s - } - return s -} - -func lenBase10(n int) int { - return int(math.Ceil(math.Log10(float64(n + 1)))) -} -func padBase10(n, m int) string { - s := fmt.Sprintf("%d", n) - if pad := m - len(s); pad > 0 { - return strings.Repeat(" ", pad) + s - } - return s -} - -func (rc RangeCodes) String() string { - var maxLen, maxBase int - for _, c := range rc { - maxLen = max(maxLen, int(c.Len)) - maxBase = max(maxBase, int(c.Base)) - } - - var ss []string - ss = append(ss, "{") - for i, c := range rc { - base := padBase10(int(c.Base), lenBase10(maxBase)) - if c.Len > 0 { - base += fmt.Sprintf("-%d", c.End()-1) - } - ss = append(ss, fmt.Sprintf("\t%s: {len: %s, range: %s},", - padBase10(int(i), lenBase10(len(rc)-1)), - padBase10(int(c.Len), lenBase10(maxLen)), - base, - )) - } - ss = append(ss, "}") - return strings.Join(ss, "\n") -} - -func (pc PrefixCodes) String() string { - var maxSym, maxLen, maxCnt int - for _, c := range pc { - maxSym = max(maxSym, int(c.Sym)) - maxLen = max(maxLen, int(c.Len)) - maxCnt = max(maxCnt, int(c.Cnt)) - } - - var ss []string - ss = append(ss, "{") - for _, c := range pc { - var cntStr string - if maxCnt > 0 { - cnt := int(32*float32(c.Cnt)/float32(maxCnt) + 0.5) - cntStr = fmt.Sprintf("%s |%s", - padBase10(int(c.Cnt), lenBase10(maxCnt)), - strings.Repeat("#", cnt), - ) - } - ss = append(ss, fmt.Sprintf("\t%s: %s, %s", - padBase10(int(c.Sym), lenBase10(maxSym)), - padBase2(uint(c.Val), uint(c.Len), maxLen), - cntStr, - )) - } - ss = append(ss, "}") - return strings.Join(ss, "\n") -} - -func (pd Decoder) String() string { - var ss []string - ss = append(ss, "{") - if len(pd.chunks) > 0 { - ss = append(ss, "\tchunks: {") - for i, c := range pd.chunks { - label := "sym" - if uint(c&countMask) > uint(pd.chunkBits) { - label = "idx" - } - ss = append(ss, fmt.Sprintf("\t\t%s: {%s: %s, len: %s}", - padBase2(uint(i), uint(pd.chunkBits), int(pd.chunkBits)), - label, padBase10(int(c>>countBits), 3), - padBase10(int(c&countMask), 2), - )) - } - ss = append(ss, "\t},") - - for j, links := range pd.links { - ss = append(ss, fmt.Sprintf("\tlinks[%d]: {", j)) - linkBits := lenBase2(uint(pd.linkMask)) - for i, c := range links { - ss = append(ss, fmt.Sprintf("\t\t%s: {sym: %s, len: %s},", - padBase2(uint(i), uint(linkBits), int(linkBits)), - padBase10(int(c>>countBits), 3), - padBase10(int(c&countMask), 2), - )) - } - ss = append(ss, "\t},") - } - } - ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pd.chunkMask)) - ss = append(ss, fmt.Sprintf("\tlinkMask: %b,", pd.linkMask)) - ss = append(ss, fmt.Sprintf("\tchunkBits: %d,", pd.chunkBits)) - ss = append(ss, fmt.Sprintf("\tMinBits: %d,", pd.MinBits)) - ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pd.NumSyms)) - ss = append(ss, "}") - return strings.Join(ss, "\n") -} - -func (pe Encoder) String() string { - var maxLen int - for _, c := range pe.chunks { - maxLen = max(maxLen, int(c&countMask)) - } - - var ss []string - ss = append(ss, "{") - if len(pe.chunks) > 0 { - ss = append(ss, "\tchunks: {") - for i, c := range pe.chunks { - ss = append(ss, fmt.Sprintf("\t\t%s: %s,", - padBase10(i, 3), - padBase2(uint(c>>countBits), uint(c&countMask), maxLen), - )) - } - ss = append(ss, "\t},") - } - ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pe.chunkMask)) - ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pe.NumSyms)) - ss = append(ss, "}") - return strings.Join(ss, "\n") -} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/decoder.go b/vendor/github.com/dsnet/compress/internal/prefix/decoder.go deleted file mode 100644 index a9bc2dcb..00000000 --- a/vendor/github.com/dsnet/compress/internal/prefix/decoder.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package prefix - -import ( - "sort" - - "github.com/dsnet/compress/internal" -) - -// The algorithm used to decode variable length codes is based on the lookup -// method in zlib. If the code is less-than-or-equal to maxChunkBits, -// then the symbol can be decoded using a single lookup into the chunks table. -// Otherwise, the links table will be used for a second level lookup. -// -// The chunks slice is keyed by the contents of the bit buffer ANDed with -// the chunkMask to avoid a out-of-bounds lookup. The value of chunks is a tuple -// that is decoded as follow: -// -// var length = chunks[bitBuffer&chunkMask] & countMask -// var symbol = chunks[bitBuffer&chunkMask] >> countBits -// -// If the decoded length is larger than chunkBits, then an overflow link table -// must be used for further decoding. In this case, the symbol is actually the -// index into the links tables. The second-level links table returned is -// processed in the same way as the chunks table. -// -// if length > chunkBits { -// var index = symbol // Previous symbol is index into links tables -// length = links[index][bitBuffer>>chunkBits & linkMask] & countMask -// symbol = links[index][bitBuffer>>chunkBits & linkMask] >> countBits -// } -// -// See the following: -// http://www.gzip.org/algorithm.txt - -type Decoder struct { - chunks []uint32 // First-level lookup map - links [][]uint32 // Second-level lookup map - chunkMask uint32 // Mask the length of the chunks table - linkMask uint32 // Mask the length of the link table - chunkBits uint32 // Bit-length of the chunks table - - MinBits uint32 // The minimum number of bits to safely make progress - NumSyms uint32 // Number of symbols -} - -// Init initializes Decoder according to the codes provided. -func (pd *Decoder) Init(codes PrefixCodes) { - // Handle special case trees. - if len(codes) <= 1 { - switch { - case len(codes) == 0: // Empty tree (should error if used later) - *pd = Decoder{chunks: pd.chunks[:0], links: pd.links[:0], NumSyms: 0} - case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero) - pd.chunks = append(pd.chunks[:0], codes[0].Sym< c.Len { - minBits = c.Len - } - if maxBits < c.Len { - maxBits = c.Len - } - } - - // Allocate chunks table as needed. - const maxChunkBits = 9 // This can be tuned for better performance - pd.NumSyms = uint32(len(codes)) - pd.MinBits = minBits - pd.chunkBits = maxBits - if pd.chunkBits > maxChunkBits { - pd.chunkBits = maxChunkBits - } - numChunks := 1 << pd.chunkBits - pd.chunks = allocUint32s(pd.chunks, numChunks) - pd.chunkMask = uint32(numChunks - 1) - - // Allocate links tables as needed. - pd.links = pd.links[:0] - pd.linkMask = 0 - if pd.chunkBits < maxBits { - numLinks := 1 << (maxBits - pd.chunkBits) - pd.linkMask = uint32(numLinks - 1) - - var linkIdx uint32 - for i := range pd.chunks { - pd.chunks[i] = 0 // Logic below relies on zero value as uninitialized - } - for _, c := range codes { - if c.Len > pd.chunkBits && pd.chunks[c.Val&pd.chunkMask] == 0 { - pd.chunks[c.Val&pd.chunkMask] = (linkIdx << countBits) | (pd.chunkBits + 1) - linkIdx++ - } - } - - pd.links = extendSliceUint32s(pd.links, int(linkIdx)) - linksFlat := allocUint32s(pd.links[0], numLinks*int(linkIdx)) - for i, j := 0, 0; i < len(pd.links); i, j = i+1, j+numLinks { - pd.links[i] = linksFlat[j : j+numLinks] - } - } - - // Fill out chunks and links tables with values. - for _, c := range codes { - chunk := c.Sym<> countBits - links := pd.links[linkIdx] - skip := 1 << uint(c.Len-pd.chunkBits) - for j := int(c.Val >> pd.chunkBits); j < len(links); j += skip { - links[j] = chunk - } - } - } -} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/encoder.go b/vendor/github.com/dsnet/compress/internal/prefix/encoder.go deleted file mode 100644 index 4424a011..00000000 --- a/vendor/github.com/dsnet/compress/internal/prefix/encoder.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package prefix - -import ( - "sort" - - "github.com/dsnet/compress/internal" -) - -type Encoder struct { - chunks []uint32 // First-level lookup map - chunkMask uint32 // Mask the length of the chunks table - - NumSyms uint32 // Number of symbols -} - -// Init initializes Encoder according to the codes provided. -func (pe *Encoder) Init(codes PrefixCodes) { - // Handle special case trees. - if len(codes) <= 1 { - switch { - case len(codes) == 0: // Empty tree (should error if used later) - *pe = Encoder{chunks: pe.chunks[:0], NumSyms: 0} - case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero) - pe.chunks = append(pe.chunks[:0], codes[0].Val< 0; n >>= 1 { - numChunks <<= 1 - } - pe.NumSyms = uint32(len(codes)) - -retry: - // Allocate and reset chunks. - pe.chunks = allocUint32s(pe.chunks, numChunks) - pe.chunkMask = uint32(numChunks - 1) - for i := range pe.chunks { - pe.chunks[i] = 0 // Logic below relies on zero value as uninitialized - } - - // Insert each symbol, checking that there are no conflicts. - for _, c := range codes { - if pe.chunks[c.Sym&pe.chunkMask] > 0 { - // Collision found our "hash" table, so grow and try again. - numChunks <<= 1 - goto retry - } - pe.chunks[c.Sym&pe.chunkMask] = c.Val<> uint(c.Len) - } - return sum == 0 || len(pc) == 0 -} - -// checkPrefixes reports whether all codes have non-overlapping prefixes. -func (pc PrefixCodes) checkPrefixes() bool { - for i, c1 := range pc { - for j, c2 := range pc { - mask := uint32(1)< 0 { - c.Val = internal.ReverseUint32N(c.Val, uint(c.Len)) - if vals[c.Len].Cnt > 0 && vals[c.Len].Val+1 != c.Val { - return false - } - vals[c.Len].Val = c.Val - vals[c.Len].Cnt++ - } - } - - // Rule 2. - var last PrefixCode - for _, v := range vals { - if v.Cnt > 0 { - curVal := v.Val - v.Cnt + 1 - if last.Cnt != 0 && last.Val >= curVal { - return false - } - last = v - } - } - return true -} - -// GenerateLengths assigns non-zero bit-lengths to all codes. Codes with high -// frequency counts will be assigned shorter codes to reduce bit entropy. -// This function is used primarily by compressors. -// -// The input codes must have the Cnt field populated, be sorted by count. -// Even if a code has a count of 0, a non-zero bit-length will be assigned. -// -// The result will have the Len field populated. The algorithm used guarantees -// that Len <= maxBits and that it is a complete prefix tree. The resulting -// codes will remain sorted by count. -func GenerateLengths(codes PrefixCodes, maxBits uint) error { - if len(codes) <= 1 { - if len(codes) == 1 { - codes[0].Len = 0 - } - return nil - } - - // Verify that the codes are in ascending order by count. - cntLast := codes[0].Cnt - for _, c := range codes[1:] { - if c.Cnt < cntLast { - return errorf(errors.Invalid, "non-monotonically increasing symbol counts") - } - cntLast = c.Cnt - } - - // Construct a Huffman tree used to generate the bit-lengths. - // - // The Huffman tree is a binary tree where each symbol lies as a leaf node - // on this tree. The length of the prefix code to assign is the depth of - // that leaf from the root. The Huffman algorithm, which runs in O(n), - // is used to generate the tree. It assumes that codes are sorted in - // increasing order of frequency. - // - // The algorithm is as follows: - // 1. Start with two queues, F and Q, where F contains all of the starting - // symbols sorted such that symbols with lowest counts come first. - // 2. While len(F)+len(Q) > 1: - // 2a. Dequeue the node from F or Q that has the lowest weight as N0. - // 2b. Dequeue the node from F or Q that has the lowest weight as N1. - // 2c. Create a new node N that has N0 and N1 as its children. - // 2d. Enqueue N into the back of Q. - // 3. The tree's root node is Q[0]. - type node struct { - cnt uint32 - - // n0 or c0 represent the left child of this node. - // Since Go does not have unions, only one of these will be set. - // Similarly, n1 or c1 represent the right child of this node. - // - // If n0 or n1 is set, then it represents a "pointer" to another - // node in the Huffman tree. Since Go's pointer analysis cannot reason - // that these node pointers do not escape (golang.org/issue/13493), - // we use an index to a node in the nodes slice as a pseudo-pointer. - // - // If c0 or c1 is set, then it represents a leaf "node" in the - // Huffman tree. The leaves are the PrefixCode values themselves. - n0, n1 int // Index to child nodes - c0, c1 *PrefixCode - } - var nodeIdx int - var nodeArr [1024]node // Large enough to handle most cases on the stack - nodes := nodeArr[:] - if len(nodes) < len(codes) { - nodes = make([]node, len(codes)) // Number of internal nodes < number of leaves - } - freqs, queue := codes, nodes[:0] - for len(freqs)+len(queue) > 1 { - // These are the two smallest nodes at the front of freqs and queue. - var n node - if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { - n.c0, freqs = &freqs[0], freqs[1:] - n.cnt += n.c0.Cnt - } else { - n.cnt += queue[0].cnt - n.n0 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0] - nodeIdx++ - queue = queue[1:] - } - if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { - n.c1, freqs = &freqs[0], freqs[1:] - n.cnt += n.c1.Cnt - } else { - n.cnt += queue[0].cnt - n.n1 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0] - nodeIdx++ - queue = queue[1:] - } - queue = append(queue, n) - } - rootIdx := nodeIdx - - // Search the whole binary tree, noting when we hit each leaf node. - // We do not care about the exact Huffman tree structure, but rather we only - // care about depth of each of the leaf nodes. That is, the depth determines - // how long each symbol is in bits. - // - // Since the number of leaves is n, there is at most n internal nodes. - // Thus, this algorithm runs in O(n). - var fixBits bool - var explore func(int, uint) - explore = func(rootIdx int, level uint) { - root := &nodes[rootIdx] - - // Explore left branch. - if root.c0 == nil { - explore(root.n0, level+1) - } else { - fixBits = fixBits || (level > maxBits) - root.c0.Len = uint32(level) - } - - // Explore right branch. - if root.c1 == nil { - explore(root.n1, level+1) - } else { - fixBits = fixBits || (level > maxBits) - root.c1.Len = uint32(level) - } - } - explore(rootIdx, 1) - - // Fix the bit-lengths if we violate the maxBits requirement. - if fixBits { - // Create histogram for number of symbols with each bit-length. - var symBitsArr [valueBits + 1]uint32 - symBits := symBitsArr[:] // symBits[nb] indicates number of symbols using nb bits - for _, c := range codes { - for int(c.Len) >= len(symBits) { - symBits = append(symBits, 0) - } - symBits[c.Len]++ - } - - // Fudge the tree such that the largest bit-length is <= maxBits. - // This is accomplish by effectively doing a tree rotation. That is, we - // increase the bit-length of some higher frequency code, so that the - // bit-lengths of lower frequency codes can be decreased. - // - // Visually, this looks like the following transform: - // - // Level Before After - // __ ___ - // / \ / \ - // n-1 X / \ /\ /\ - // n X /\ X X X X - // n+1 X X - // - var treeRotate func(uint) - treeRotate = func(nb uint) { - if symBits[nb-1] == 0 { - treeRotate(nb - 1) - } - symBits[nb-1] -= 1 // Push this node to the level below - symBits[nb] += 3 // This level gets one node from above, two from below - symBits[nb+1] -= 2 // Push two nodes to the level above - } - for i := uint(len(symBits)) - 1; i > maxBits; i-- { - for symBits[i] > 0 { - treeRotate(i - 1) - } - } - - // Assign bit-lengths to each code. Since codes is sorted in increasing - // order of frequency, that means that the most frequently used symbols - // should have the shortest bit-lengths. Thus, we copy symbols to codes - // from the back of codes first. - cs := codes - for nb, cnt := range symBits { - if cnt > 0 { - pos := len(cs) - int(cnt) - cs2 := cs[pos:] - for i := range cs2 { - cs2[i].Len = uint32(nb) - } - cs = cs[:pos] - } - } - if len(cs) != 0 { - panic("not all codes were used up") - } - } - - if internal.Debug && !codes.checkLengths() { - panic("incomplete prefix tree detected") - } - return nil -} - -// GeneratePrefixes assigns a prefix value to all codes according to the -// bit-lengths. This function is used by both compressors and decompressors. -// -// The input codes must have the Sym and Len fields populated and be -// sorted by symbol. The bit-lengths of each code must be properly allocated, -// such that it forms a complete tree. -// -// The result will have the Val field populated and will produce a canonical -// prefix tree. The resulting codes will remain sorted by symbol. -func GeneratePrefixes(codes PrefixCodes) error { - if len(codes) <= 1 { - if len(codes) == 1 { - if codes[0].Len != 0 { - return errorf(errors.Invalid, "degenerate prefix tree with one node") - } - codes[0].Val = 0 - } - return nil - } - - // Compute basic statistics on the symbols. - var bitCnts [valueBits + 1]uint - c0 := codes[0] - bitCnts[c0.Len]++ - minBits, maxBits, symLast := c0.Len, c0.Len, c0.Sym - for _, c := range codes[1:] { - if c.Sym <= symLast { - return errorf(errors.Invalid, "non-unique or non-monotonically increasing symbols") - } - if minBits > c.Len { - minBits = c.Len - } - if maxBits < c.Len { - maxBits = c.Len - } - bitCnts[c.Len]++ // Histogram of bit counts - symLast = c.Sym // Keep track of last symbol - } - if minBits == 0 { - return errorf(errors.Invalid, "invalid prefix bit-length") - } - - // Compute the next code for a symbol of a given bit length. - var nextCodes [valueBits + 1]uint - var code uint - for i := minBits; i <= maxBits; i++ { - code <<= 1 - nextCodes[i] = code - code += bitCnts[i] - } - if code != 1<= n { - return s[:n] - } - return make([]uint32, n, n*3/2) -} - -func extendSliceUint32s(s [][]uint32, n int) [][]uint32 { - if cap(s) >= n { - return s[:n] - } - ss := make([][]uint32, n, n*3/2) - copy(ss, s[:cap(s)]) - return ss -} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/range.go b/vendor/github.com/dsnet/compress/internal/prefix/range.go deleted file mode 100644 index b7eddad5..00000000 --- a/vendor/github.com/dsnet/compress/internal/prefix/range.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package prefix - -type RangeCode struct { - Base uint32 // Starting base offset of the range - Len uint32 // Bit-length of a subsequent integer to add to base offset -} -type RangeCodes []RangeCode - -type RangeEncoder struct { - rcs RangeCodes - lut [1024]uint32 - minBase uint -} - -// End reports the non-inclusive ending range. -func (rc RangeCode) End() uint32 { return rc.Base + (1 << rc.Len) } - -// MakeRangeCodes creates a RangeCodes, where each region is assumed to be -// contiguously stacked, without any gaps, with bit-lengths taken from bits. -func MakeRangeCodes(minBase uint, bits []uint) (rc RangeCodes) { - for _, nb := range bits { - rc = append(rc, RangeCode{Base: uint32(minBase), Len: uint32(nb)}) - minBase += 1 << nb - } - return rc -} - -// Base reports the inclusive starting range for all ranges. -func (rcs RangeCodes) Base() uint32 { return rcs[0].Base } - -// End reports the non-inclusive ending range for all ranges. -func (rcs RangeCodes) End() uint32 { return rcs[len(rcs)-1].End() } - -// checkValid reports whether the RangeCodes is valid. In order to be valid, -// the following must hold true: -// rcs[i-1].Base <= rcs[i].Base -// rcs[i-1].End <= rcs[i].End -// rcs[i-1].End >= rcs[i].Base -// -// Practically speaking, each range must be increasing and must not have any -// gaps in between. It is okay for ranges to overlap. -func (rcs RangeCodes) checkValid() bool { - if len(rcs) == 0 { - return false - } - pre := rcs[0] - for _, cur := range rcs[1:] { - preBase, preEnd := pre.Base, pre.End() - curBase, curEnd := cur.Base, cur.End() - if preBase > curBase || preEnd > curEnd || preEnd < curBase { - return false - } - pre = cur - } - return true -} - -func (re *RangeEncoder) Init(rcs RangeCodes) { - if !rcs.checkValid() { - panic("invalid range codes") - } - *re = RangeEncoder{rcs: rcs, minBase: uint(rcs.Base())} - for sym, rc := range rcs { - base := int(rc.Base) - int(re.minBase) - end := int(rc.End()) - int(re.minBase) - if base >= len(re.lut) { - break - } - if end > len(re.lut) { - end = len(re.lut) - } - for i := base; i < end; i++ { - re.lut[i] = uint32(sym) - } - } -} - -func (re *RangeEncoder) Encode(offset uint) (sym uint) { - if idx := int(offset - re.minBase); idx < len(re.lut) { - return uint(re.lut[idx]) - } - sym = uint(re.lut[len(re.lut)-1]) -retry: - if int(sym) >= len(re.rcs) || re.rcs[sym].Base > uint32(offset) { - return sym - 1 - } - sym++ - goto retry // Avoid for-loop so that this function can be inlined -} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/reader.go b/vendor/github.com/dsnet/compress/internal/prefix/reader.go deleted file mode 100644 index e6252c95..00000000 --- a/vendor/github.com/dsnet/compress/internal/prefix/reader.go +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package prefix - -import ( - "bufio" - "bytes" - "encoding/binary" - "io" - "strings" - - "github.com/dsnet/compress" - "github.com/dsnet/compress/internal" - "github.com/dsnet/compress/internal/errors" -) - -// Reader implements a prefix decoder. If the input io.Reader satisfies the -// compress.ByteReader or compress.BufferedReader interface, then it also -// guarantees that it will never read more bytes than is necessary. -// -// For high performance, provide an io.Reader that satisfies the -// compress.BufferedReader interface. If the input does not satisfy either -// compress.ByteReader or compress.BufferedReader, then it will be internally -// wrapped with a bufio.Reader. -type Reader struct { - Offset int64 // Number of bytes read from the underlying io.Reader - - rd io.Reader - byteRd compress.ByteReader // Set if rd is a ByteReader - bufRd compress.BufferedReader // Set if rd is a BufferedReader - - bufBits uint64 // Buffer to hold some bits - numBits uint // Number of valid bits in bufBits - bigEndian bool // Do we treat input bytes as big endian? - - // These fields are only used if rd is a compress.BufferedReader. - bufPeek []byte // Buffer for the Peek data - discardBits int // Number of bits to discard from reader - fedBits uint // Number of bits fed in last call to PullBits - - // These fields are used to reduce allocations. - bb *buffer - br *bytesReader - sr *stringReader - bu *bufio.Reader -} - -// Init initializes the bit Reader to read from r. If bigEndian is true, then -// bits will be read starting from the most-significant bits of a byte -// (as done in bzip2), otherwise it will read starting from the -// least-significant bits of a byte (such as for deflate and brotli). -func (pr *Reader) Init(r io.Reader, bigEndian bool) { - *pr = Reader{ - rd: r, - bigEndian: bigEndian, - - bb: pr.bb, - br: pr.br, - sr: pr.sr, - bu: pr.bu, - } - switch rr := r.(type) { - case *bytes.Buffer: - if pr.bb == nil { - pr.bb = new(buffer) - } - *pr.bb = buffer{Buffer: rr} - pr.bufRd = pr.bb - case *bytes.Reader: - if pr.br == nil { - pr.br = new(bytesReader) - } - *pr.br = bytesReader{Reader: rr} - pr.bufRd = pr.br - case *strings.Reader: - if pr.sr == nil { - pr.sr = new(stringReader) - } - *pr.sr = stringReader{Reader: rr} - pr.bufRd = pr.sr - case compress.BufferedReader: - pr.bufRd = rr - case compress.ByteReader: - pr.byteRd = rr - default: - if pr.bu == nil { - pr.bu = bufio.NewReader(nil) - } - pr.bu.Reset(r) - pr.rd, pr.bufRd = pr.bu, pr.bu - } -} - -// BitsRead reports the total number of bits emitted from any Read method. -func (pr *Reader) BitsRead() int64 { - offset := 8*pr.Offset - int64(pr.numBits) - if pr.bufRd != nil { - discardBits := pr.discardBits + int(pr.fedBits-pr.numBits) - offset = 8*pr.Offset + int64(discardBits) - } - return offset -} - -// IsBufferedReader reports whether the underlying io.Reader is also a -// compress.BufferedReader. -func (pr *Reader) IsBufferedReader() bool { - return pr.bufRd != nil -} - -// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment. -func (pr *Reader) ReadPads() uint { - nb := pr.numBits % 8 - val := uint(pr.bufBits & uint64(1<>= nb - pr.numBits -= nb - return val -} - -// Read reads bytes into buf. -// The bit-ordering mode does not affect this method. -func (pr *Reader) Read(buf []byte) (cnt int, err error) { - if pr.numBits > 0 { - if pr.numBits%8 != 0 { - return 0, errorf(errors.Invalid, "non-aligned bit buffer") - } - for cnt = 0; len(buf) > cnt && pr.numBits > 0; cnt++ { - if pr.bigEndian { - buf[cnt] = internal.ReverseLUT[byte(pr.bufBits)] - } else { - buf[cnt] = byte(pr.bufBits) - } - pr.bufBits >>= 8 - pr.numBits -= 8 - } - return cnt, nil - } - if _, err := pr.Flush(); err != nil { - return 0, err - } - cnt, err = pr.rd.Read(buf) - pr.Offset += int64(cnt) - return cnt, err -} - -// ReadOffset reads an offset value using the provided RangeCodes indexed by -// the symbol read. -func (pr *Reader) ReadOffset(pd *Decoder, rcs RangeCodes) uint { - rc := rcs[pr.ReadSymbol(pd)] - return uint(rc.Base) + pr.ReadBits(uint(rc.Len)) -} - -// TryReadBits attempts to read nb bits using the contents of the bit buffer -// alone. It returns the value and whether it succeeded. -// -// This method is designed to be inlined for performance reasons. -func (pr *Reader) TryReadBits(nb uint) (uint, bool) { - if pr.numBits < nb { - return 0, false - } - val := uint(pr.bufBits & uint64(1<>= nb - pr.numBits -= nb - return val, true -} - -// ReadBits reads nb bits in from the underlying reader. -func (pr *Reader) ReadBits(nb uint) uint { - if err := pr.PullBits(nb); err != nil { - errors.Panic(err) - } - val := uint(pr.bufBits & uint64(1<>= nb - pr.numBits -= nb - return val -} - -// TryReadSymbol attempts to decode the next symbol using the contents of the -// bit buffer alone. It returns the decoded symbol and whether it succeeded. -// -// This method is designed to be inlined for performance reasons. -func (pr *Reader) TryReadSymbol(pd *Decoder) (uint, bool) { - if pr.numBits < uint(pd.MinBits) || len(pd.chunks) == 0 { - return 0, false - } - chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] - nb := uint(chunk & countMask) - if nb > pr.numBits || nb > uint(pd.chunkBits) { - return 0, false - } - pr.bufBits >>= nb - pr.numBits -= nb - return uint(chunk >> countBits), true -} - -// ReadSymbol reads the next symbol using the provided prefix Decoder. -func (pr *Reader) ReadSymbol(pd *Decoder) uint { - if len(pd.chunks) == 0 { - panicf(errors.Invalid, "decode with empty prefix tree") - } - - nb := uint(pd.MinBits) - for { - if err := pr.PullBits(nb); err != nil { - errors.Panic(err) - } - chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] - nb = uint(chunk & countMask) - if nb > uint(pd.chunkBits) { - linkIdx := chunk >> countBits - chunk = pd.links[linkIdx][uint32(pr.bufBits>>pd.chunkBits)&pd.linkMask] - nb = uint(chunk & countMask) - } - if nb <= pr.numBits { - pr.bufBits >>= nb - pr.numBits -= nb - return uint(chunk >> countBits) - } - } -} - -// Flush updates the read offset of the underlying ByteReader. -// If reader is a compress.BufferedReader, then this calls Discard to update -// the read offset. -func (pr *Reader) Flush() (int64, error) { - if pr.bufRd == nil { - return pr.Offset, nil - } - - // Update the number of total bits to discard. - pr.discardBits += int(pr.fedBits - pr.numBits) - pr.fedBits = pr.numBits - - // Discard some bytes to update read offset. - var err error - nd := (pr.discardBits + 7) / 8 // Round up to nearest byte - nd, err = pr.bufRd.Discard(nd) - pr.discardBits -= nd * 8 // -7..0 - pr.Offset += int64(nd) - - // These are invalid after Discard. - pr.bufPeek = nil - return pr.Offset, err -} - -// PullBits ensures that at least nb bits exist in the bit buffer. -// If the underlying reader is a compress.BufferedReader, then this will fill -// the bit buffer with as many bits as possible, relying on Peek and Discard to -// properly advance the read offset. Otherwise, it will use ReadByte to fill the -// buffer with just the right number of bits. -func (pr *Reader) PullBits(nb uint) error { - if pr.bufRd != nil { - pr.discardBits += int(pr.fedBits - pr.numBits) - for { - if len(pr.bufPeek) == 0 { - pr.fedBits = pr.numBits // Don't discard bits just added - if _, err := pr.Flush(); err != nil { - return err - } - - // Peek no more bytes than necessary. - // The computation for cntPeek computes the minimum number of - // bytes to Peek to fill nb bits. - var err error - cntPeek := int(nb+(-nb&7)) / 8 - if cntPeek < pr.bufRd.Buffered() { - cntPeek = pr.bufRd.Buffered() - } - pr.bufPeek, err = pr.bufRd.Peek(cntPeek) - pr.bufPeek = pr.bufPeek[int(pr.numBits/8):] // Skip buffered bits - if len(pr.bufPeek) == 0 { - if pr.numBits >= nb { - break - } - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return err - } - } - - n := int(64-pr.numBits) / 8 // Number of bytes to copy to bit buffer - if len(pr.bufPeek) >= 8 { - // Starting with Go 1.7, the compiler should use a wide integer - // load here if the architecture supports it. - u := binary.LittleEndian.Uint64(pr.bufPeek) - if pr.bigEndian { - // Swap all the bits within each byte. - u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 - u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 - u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 - } - - pr.bufBits |= u << pr.numBits - pr.numBits += uint(n * 8) - pr.bufPeek = pr.bufPeek[n:] - break - } else { - if n > len(pr.bufPeek) { - n = len(pr.bufPeek) - } - for _, c := range pr.bufPeek[:n] { - if pr.bigEndian { - c = internal.ReverseLUT[c] - } - pr.bufBits |= uint64(c) << pr.numBits - pr.numBits += 8 - } - pr.bufPeek = pr.bufPeek[n:] - if pr.numBits > 56 { - break - } - } - } - pr.fedBits = pr.numBits - } else { - for pr.numBits < nb { - c, err := pr.byteRd.ReadByte() - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return err - } - if pr.bigEndian { - c = internal.ReverseLUT[c] - } - pr.bufBits |= uint64(c) << pr.numBits - pr.numBits += 8 - pr.Offset++ - } - } - return nil -} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/wrap.go b/vendor/github.com/dsnet/compress/internal/prefix/wrap.go deleted file mode 100644 index 49906d4a..00000000 --- a/vendor/github.com/dsnet/compress/internal/prefix/wrap.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package prefix - -import ( - "bytes" - "io" - "strings" -) - -// For some of the common Readers, we wrap and extend them to satisfy the -// compress.BufferedReader interface to improve performance. - -type buffer struct { - *bytes.Buffer -} - -type bytesReader struct { - *bytes.Reader - pos int64 - buf []byte - arr [512]byte -} - -type stringReader struct { - *strings.Reader - pos int64 - buf []byte - arr [512]byte -} - -func (r *buffer) Buffered() int { - return r.Len() -} - -func (r *buffer) Peek(n int) ([]byte, error) { - b := r.Bytes() - if len(b) < n { - return b, io.EOF - } - return b[:n], nil -} - -func (r *buffer) Discard(n int) (int, error) { - b := r.Next(n) - if len(b) < n { - return len(b), io.EOF - } - return n, nil -} - -func (r *bytesReader) Buffered() int { - r.update() - if r.Len() > len(r.buf) { - return len(r.buf) - } - return r.Len() -} - -func (r *bytesReader) Peek(n int) ([]byte, error) { - if n > len(r.arr) { - return nil, io.ErrShortBuffer - } - - // Return sub-slice of local buffer if possible. - r.update() - if len(r.buf) >= n { - return r.buf[:n], nil - } - - // Fill entire local buffer, and return appropriate sub-slice. - cnt, err := r.ReadAt(r.arr[:], r.pos) - r.buf = r.arr[:cnt] - if cnt < n { - return r.arr[:cnt], err - } - return r.arr[:n], nil -} - -func (r *bytesReader) Discard(n int) (int, error) { - var err error - if n > r.Len() { - n, err = r.Len(), io.EOF - } - r.Seek(int64(n), io.SeekCurrent) - return n, err -} - -// update reslices the internal buffer to be consistent with the read offset. -func (r *bytesReader) update() { - pos, _ := r.Seek(0, io.SeekCurrent) - if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { - r.buf, r.pos = r.buf[off:], pos - } else { - r.buf, r.pos = nil, pos - } -} - -func (r *stringReader) Buffered() int { - r.update() - if r.Len() > len(r.buf) { - return len(r.buf) - } - return r.Len() -} - -func (r *stringReader) Peek(n int) ([]byte, error) { - if n > len(r.arr) { - return nil, io.ErrShortBuffer - } - - // Return sub-slice of local buffer if possible. - r.update() - if len(r.buf) >= n { - return r.buf[:n], nil - } - - // Fill entire local buffer, and return appropriate sub-slice. - cnt, err := r.ReadAt(r.arr[:], r.pos) - r.buf = r.arr[:cnt] - if cnt < n { - return r.arr[:cnt], err - } - return r.arr[:n], nil -} - -func (r *stringReader) Discard(n int) (int, error) { - var err error - if n > r.Len() { - n, err = r.Len(), io.EOF - } - r.Seek(int64(n), io.SeekCurrent) - return n, err -} - -// update reslices the internal buffer to be consistent with the read offset. -func (r *stringReader) update() { - pos, _ := r.Seek(0, io.SeekCurrent) - if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { - r.buf, r.pos = r.buf[off:], pos - } else { - r.buf, r.pos = nil, pos - } -} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/writer.go b/vendor/github.com/dsnet/compress/internal/prefix/writer.go deleted file mode 100644 index c9783905..00000000 --- a/vendor/github.com/dsnet/compress/internal/prefix/writer.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package prefix - -import ( - "encoding/binary" - "io" - - "github.com/dsnet/compress/internal/errors" -) - -// Writer implements a prefix encoder. For performance reasons, Writer will not -// write bytes immediately to the underlying stream. -type Writer struct { - Offset int64 // Number of bytes written to the underlying io.Writer - - wr io.Writer - bufBits uint64 // Buffer to hold some bits - numBits uint // Number of valid bits in bufBits - bigEndian bool // Are bits written in big-endian order? - - buf [512]byte - cntBuf int -} - -// Init initializes the bit Writer to write to w. If bigEndian is true, then -// bits will be written starting from the most-significant bits of a byte -// (as done in bzip2), otherwise it will write starting from the -// least-significant bits of a byte (such as for deflate and brotli). -func (pw *Writer) Init(w io.Writer, bigEndian bool) { - *pw = Writer{wr: w, bigEndian: bigEndian} - return -} - -// BitsWritten reports the total number of bits issued to any Write method. -func (pw *Writer) BitsWritten() int64 { - return 8*pw.Offset + 8*int64(pw.cntBuf) + int64(pw.numBits) -} - -// WritePads writes 0-7 bits to the bit buffer to achieve byte-alignment. -func (pw *Writer) WritePads(v uint) { - nb := -pw.numBits & 7 - pw.bufBits |= uint64(v) << pw.numBits - pw.numBits += nb -} - -// Write writes bytes from buf. -// The bit-ordering mode does not affect this method. -func (pw *Writer) Write(buf []byte) (cnt int, err error) { - if pw.numBits > 0 || pw.cntBuf > 0 { - if pw.numBits%8 != 0 { - return 0, errorf(errors.Invalid, "non-aligned bit buffer") - } - if _, err := pw.Flush(); err != nil { - return 0, err - } - } - cnt, err = pw.wr.Write(buf) - pw.Offset += int64(cnt) - return cnt, err -} - -// WriteOffset writes ofs in a (sym, extra) fashion using the provided prefix -// Encoder and RangeEncoder. -func (pw *Writer) WriteOffset(ofs uint, pe *Encoder, re *RangeEncoder) { - sym := re.Encode(ofs) - pw.WriteSymbol(sym, pe) - rc := re.rcs[sym] - pw.WriteBits(ofs-uint(rc.Base), uint(rc.Len)) -} - -// TryWriteBits attempts to write nb bits using the contents of the bit buffer -// alone. It reports whether it succeeded. -// -// This method is designed to be inlined for performance reasons. -func (pw *Writer) TryWriteBits(v, nb uint) bool { - if 64-pw.numBits < nb { - return false - } - pw.bufBits |= uint64(v) << pw.numBits - pw.numBits += nb - return true -} - -// WriteBits writes nb bits of v to the underlying writer. -func (pw *Writer) WriteBits(v, nb uint) { - if _, err := pw.PushBits(); err != nil { - errors.Panic(err) - } - pw.bufBits |= uint64(v) << pw.numBits - pw.numBits += nb -} - -// TryWriteSymbol attempts to encode the next symbol using the contents of the -// bit buffer alone. It reports whether it succeeded. -// -// This method is designed to be inlined for performance reasons. -func (pw *Writer) TryWriteSymbol(sym uint, pe *Encoder) bool { - chunk := pe.chunks[uint32(sym)&pe.chunkMask] - nb := uint(chunk & countMask) - if 64-pw.numBits < nb { - return false - } - pw.bufBits |= uint64(chunk>>countBits) << pw.numBits - pw.numBits += nb - return true -} - -// WriteSymbol writes the symbol using the provided prefix Encoder. -func (pw *Writer) WriteSymbol(sym uint, pe *Encoder) { - if _, err := pw.PushBits(); err != nil { - errors.Panic(err) - } - chunk := pe.chunks[uint32(sym)&pe.chunkMask] - nb := uint(chunk & countMask) - pw.bufBits |= uint64(chunk>>countBits) << pw.numBits - pw.numBits += nb -} - -// Flush flushes all complete bytes from the bit buffer to the byte buffer, and -// then flushes all bytes in the byte buffer to the underlying writer. -// After this call, the bit Writer is will only withhold 7 bits at most. -func (pw *Writer) Flush() (int64, error) { - if pw.numBits < 8 && pw.cntBuf == 0 { - return pw.Offset, nil - } - if _, err := pw.PushBits(); err != nil { - return pw.Offset, err - } - cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) - pw.cntBuf -= cnt - pw.Offset += int64(cnt) - return pw.Offset, err -} - -// PushBits pushes as many bytes as possible from the bit buffer to the byte -// buffer, reporting the number of bits pushed. -func (pw *Writer) PushBits() (uint, error) { - if pw.cntBuf >= len(pw.buf)-8 { - cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) - pw.cntBuf -= cnt - pw.Offset += int64(cnt) - if err != nil { - return 0, err - } - } - - u := pw.bufBits - if pw.bigEndian { - // Swap all the bits within each byte. - u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 - u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 - u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 - } - // Starting with Go 1.7, the compiler should use a wide integer - // store here if the architecture supports it. - binary.LittleEndian.PutUint64(pw.buf[pw.cntBuf:], u) - - nb := pw.numBits / 8 // Number of bytes to copy from bit buffer - pw.cntBuf += int(nb) - pw.bufBits >>= 8 * nb - pw.numBits -= 8 * nb - return 8 * nb, nil -} diff --git a/vendor/github.com/dsnet/compress/internal/release.go b/vendor/github.com/dsnet/compress/internal/release.go deleted file mode 100644 index 0990be1c..00000000 --- a/vendor/github.com/dsnet/compress/internal/release.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -// +build !debug,!gofuzz - -package internal - -// Debug indicates whether the debug build tag was set. -// -// If set, programs may choose to print with more human-readable -// debug information and also perform sanity checks that would otherwise be too -// expensive to run in a release build. -const Debug = false - -// GoFuzz indicates whether the gofuzz build tag was set. -// -// If set, programs may choose to disable certain checks (like checksums) that -// would be nearly impossible for gofuzz to properly get right. -// If GoFuzz is set, it implies that Debug is set as well. -const GoFuzz = false diff --git a/vendor/github.com/dsnet/compress/zbench.sh b/vendor/github.com/dsnet/compress/zbench.sh deleted file mode 100755 index 0205920d..00000000 --- a/vendor/github.com/dsnet/compress/zbench.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Copyright 2017, Joe Tsai. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE.md file. - -# zbench wraps internal/tool/bench and is useful for comparing benchmarks from -# the implementations in this repository relative to other implementations. -# -# See internal/tool/bench/main.go for more details. -cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/bench -go run $(go list -f '{{ join .GoFiles "\n" }}') "$@" diff --git a/vendor/github.com/dsnet/compress/zfuzz.sh b/vendor/github.com/dsnet/compress/zfuzz.sh deleted file mode 100755 index 42958ed4..00000000 --- a/vendor/github.com/dsnet/compress/zfuzz.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# -# Copyright 2017, Joe Tsai. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE.md file. - -# zfuzz wraps internal/tool/fuzz and is useful for fuzz testing each of -# the implementations in this repository. -cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/fuzz -./fuzz.sh "$@" diff --git a/vendor/github.com/dsnet/compress/zprof.sh b/vendor/github.com/dsnet/compress/zprof.sh deleted file mode 100755 index 3cd535be..00000000 --- a/vendor/github.com/dsnet/compress/zprof.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -# Copyright 2017, Joe Tsai. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE.md file. - -if [ $# == 0 ]; then - echo "Usage: $0 PKG_PATH TEST_ARGS..." - echo "" - echo "Runs coverage and performance benchmarks for a given package." - echo "The results are stored in the _zprof_ directory." - echo "" - echo "Example:" - echo " $0 flate -test.bench=Decode/Twain/Default" - exit 1 -fi - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PKG_PATH=$1 -PKG_NAME=$(basename $PKG_PATH) -shift - -TMPDIR=$(mktemp -d) -trap "rm -rf $TMPDIR $PKG_PATH/$PKG_NAME.test" SIGINT SIGTERM EXIT - -( - cd $DIR/$PKG_PATH - - # Print the go version. - go version - - # Perform coverage profiling. - go test github.com/dsnet/compress/$PKG_PATH -coverprofile $TMPDIR/cover.profile - if [ $? != 0 ]; then exit 1; fi - go tool cover -html $TMPDIR/cover.profile -o cover.html - - # Perform performance profiling. - if [ $# != 0 ]; then - go test -c github.com/dsnet/compress/$PKG_PATH - if [ $? != 0 ]; then exit 1; fi - ./$PKG_NAME.test -test.cpuprofile $TMPDIR/cpu.profile -test.memprofile $TMPDIR/mem.profile -test.run - "$@" - PPROF="go tool pprof" - $PPROF -output=cpu.svg -web $PKG_NAME.test $TMPDIR/cpu.profile 2> /dev/null - $PPROF -output=cpu.html -weblist=. $PKG_NAME.test $TMPDIR/cpu.profile 2> /dev/null - $PPROF -output=mem_objects.svg -alloc_objects -web $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null - $PPROF -output=mem_objects.html -alloc_objects -weblist=. $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null - $PPROF -output=mem_space.svg -alloc_space -web $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null - $PPROF -output=mem_space.html -alloc_space -weblist=. $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null - fi - - rm -rf $DIR/_zprof_/$PKG_NAME - mkdir -p $DIR/_zprof_/$PKG_NAME - mv *.html *.svg $DIR/_zprof_/$PKG_NAME 2> /dev/null -) diff --git a/vendor/github.com/dsnet/compress/ztest.sh b/vendor/github.com/dsnet/compress/ztest.sh deleted file mode 100755 index 15c4c00b..00000000 --- a/vendor/github.com/dsnet/compress/ztest.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -# Copyright 2017, Joe Tsai. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE.md file. - -cd $(go list -f '{{ .Dir }}' github.com/dsnet/compress) - -BOLD="\x1b[1mRunning: " -PASS="\x1b[32mPASS" -FAIL="\x1b[31mFAIL" -RESET="\x1b[0m" - -echo -e "${BOLD}fmt${RESET}" -RET_FMT=$(find . -name "*.go" | egrep -v "/(_.*_|\..*|testdata)/" | xargs gofmt -d) -if [[ ! -z "$RET_FMT" ]]; then echo "$RET_FMT"; echo; fi - -echo -e "${BOLD}test${RESET}" -RET_TEST=$(go test -race ./... | egrep -v "^(ok|[?])\s+") -if [[ ! -z "$RET_TEST" ]]; then echo "$RET_TEST"; echo; fi - -echo -e "${BOLD}staticcheck${RESET}" -RET_SCHK=$(staticcheck \ - -ignore " - github.com/dsnet/compress/brotli/*.go:SA4016 - github.com/dsnet/compress/brotli/*.go:S1023 - github.com/dsnet/compress/brotli/*.go:U1000 - github.com/dsnet/compress/bzip2/*.go:S1023 - github.com/dsnet/compress/flate/*.go:U1000 - github.com/dsnet/compress/internal/cgo/lzma/*.go:SA4000 - github.com/dsnet/compress/internal/prefix/*.go:S1004 - github.com/dsnet/compress/internal/prefix/*.go:S1023 - github.com/dsnet/compress/internal/prefix/*.go:SA4016 - github.com/dsnet/compress/internal/tool/bench/*.go:S1007 - github.com/dsnet/compress/xflate/internal/meta/*.go:S1023 - " ./... 2>&1) -if [[ ! -z "$RET_SCHK" ]]; then echo "$RET_SCHK"; echo; fi - -echo -e "${BOLD}lint${RESET}" -RET_LINT=$(golint ./... 2>&1 | - egrep -v "^vendor/" | - egrep -v "should have comment(.*)or be unexported" | - egrep -v "^(.*)type name will be used as(.*)by other packages" | - egrep -v "^brotli/transform.go:(.*)replace i [+]= 1 with i[+]{2}" | - egrep -v "^internal/prefix/prefix.go:(.*)replace symBits(.*) [-]= 1 with symBits(.*)[-]{2}" | - egrep -v "^xflate/common.go:(.*)NoCompression should be of the form" | - egrep -v "^exit status") -if [[ ! -z "$RET_LINT" ]]; then echo "$RET_LINT"; echo; fi - -if [[ ! -z "$RET_FMT" ]] || [ ! -z "$RET_TEST" ] || [[ ! -z "$RET_SCHK" ]] || [[ ! -z "$RET_LINT" ]]; then - echo -e "${FAIL}${RESET}"; exit 1 -else - echo -e "${PASS}${RESET}"; exit 0 -fi diff --git a/vendor/github.com/icedream/go-bsdiff/.gitignore b/vendor/github.com/icedream/go-bsdiff/.gitignore deleted file mode 100644 index b99edc31..00000000 --- a/vendor/github.com/icedream/go-bsdiff/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -go-bsdiff/go-bsdiff -go-bspatch/go-bspatch \ No newline at end of file diff --git a/vendor/github.com/icedream/go-bsdiff/README.md b/vendor/github.com/icedream/go-bsdiff/README.md deleted file mode 100644 index e17c8616..00000000 --- a/vendor/github.com/icedream/go-bsdiff/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# bsdiff for Go - -This wrapper implementation for Golang reuses the existing -C version of bsdiff as provided by @mendsley and wraps it -into a Go package, abstracting away all the cgo work that -would need to be done otherwise. - -## Installation - -The library and the helper binaries `go-bsdiff` and `go-bspatch` can be installed like this: - - go get -v github.com/icedream/go-bsdiff/... - -## Usage in application code - -For exact documentation of the library check out [GoDoc](https://godoc.org/github.com/icedream/go-bsdiff). - -Library functionality is provided both as a package `bsdiff` containing both -methods `Diff` and `Patch`, or as subpackages `diff` and `patch` which each -only link the wanted functionality. - -Below example will generate a patch and apply it again in one go. This code -is not safe against errors but it shows how to use the provided routines: - -```go -package main - -import ( - "os" - "github.com/icedream/go-bsdiff" - // Or use the subpackages to only link what you need: - //"github.com/icedream/go-bsdiff/diff" - //"github.com/icedream/go-bsdiff/patch" -) - -const ( - oldFilePath = "your_old_file.dat" - newFilePath = "your_new_file.dat" - patchFilePath = "the_generated.patch" -) - -func generatePatch() error { - oldFile, _ := os.Open(oldFilePath) - defer oldFile.Close() - newFile, _ := os.Open(newFilePath) - defer newFile.Close() - patchFile, _ := os.Create(patchFilePath) - defer patchFile.Close() - - return bsdiff.Diff(oldFile, newFile, patchFile) -} - -func applyPatch() error { - oldFile, _ := os.Open(oldFilePath) - defer oldFile.Close() - newFile, _ := os.Create(newFilePath) - defer newFile.Close() - patchFile, _ := os.Open(patchFilePath) - defer patchFile.Close() - - return bsdiff.Patch(oldFile, newFile, patchFile) -} - -func main() { - generatePatch() - applyPatch() -} -``` - -## Usage of the tools - -The tools `go-bsdiff` and `go-bspatch` both provide a `--help` flag to print -out all information but in their simplest form, they can be used like this: - -```sh -# Creates a patch file $the_generated with differences from -# $your_old_file to $your_new_file. -go-bsdiff "$your_old_file" "$your_new_file" "$the_generated" - -# Applies a patch file $the_generated on $your_old_file -# and saves the new file to $your_new_file. -go-bspatch "$your_old_file" "$your_new_file" "$the_generated" -``` - -## Motivation - -There is [a Go implementation of an older version of bsdiff called binarydist](https://github.com/kr/binarydist). The original bsdiff tool has since been updated so patches generating using the original tool are no longer compatible with the Go implementation. I don't know what the changes between the versions are and unfortunately I don't have the time to search for these changes and port them over as a pull request, otherwise I'd have done that instead. - -Additionally, @mendsley has already done the extra work of rewriting the code to be embeddable in any application code as a library. So why not make use of cgo, which I was going to look into in more detail at some point anyways? diff --git a/vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE b/vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE deleted file mode 100644 index 5d406657..00000000 --- a/vendor/github.com/icedream/go-bsdiff/bsdiff/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ - Copyright 2003-2005 Colin Percival - Copyright 2012 Matthew Endsley - All rights reserved - - Redistribution and use in source and binary forms, with or without - modification, are permitted providing that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/icedream/go-bsdiff/diff/diff.go b/vendor/github.com/icedream/go-bsdiff/diff/diff.go deleted file mode 100644 index 1ed4cda5..00000000 --- a/vendor/github.com/icedream/go-bsdiff/diff/diff.go +++ /dev/null @@ -1,34 +0,0 @@ -package diff - -import ( - "io" - "io/ioutil" - - "github.com/dsnet/compress/bzip2" - "github.com/icedream/go-bsdiff/internal" - "github.com/icedream/go-bsdiff/internal/native" -) - -func Diff(oldReader, newReader io.Reader, patchWriter io.Writer) (err error) { - oldBytes, err := ioutil.ReadAll(oldReader) - if err != nil { - return - } - newBytes, err := ioutil.ReadAll(newReader) - if err != nil { - return - } - - if err = internal.WriteHeader(patchWriter, uint64(len(newBytes))); err != nil { - return - } - - // Compression - bz2Writer, err := bzip2.NewWriter(patchWriter, nil) - if err != nil { - return - } - defer bz2Writer.Close() - - return native.Diff(oldBytes, newBytes, bz2Writer) -} diff --git a/vendor/github.com/icedream/go-bsdiff/go.mod b/vendor/github.com/icedream/go-bsdiff/go.mod deleted file mode 100644 index d072de8e..00000000 --- a/vendor/github.com/icedream/go-bsdiff/go.mod +++ /dev/null @@ -1,11 +0,0 @@ -module github.com/icedream/go-bsdiff - -require ( - github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect - github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/testify v1.2.2 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 -) diff --git a/vendor/github.com/icedream/go-bsdiff/go.sum b/vendor/github.com/icedream/go-bsdiff/go.sum deleted file mode 100644 index 81f370ec..00000000 --- a/vendor/github.com/icedream/go-bsdiff/go.sum +++ /dev/null @@ -1,14 +0,0 @@ -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 h1:eX+pdPPlD279OWgdx7f6KqIRSONuK7egk+jDx7OM3Ac= -github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76/go.mod h1:KjxHHirfLaw19iGT70HvVjHQsL1vq1SRQB4yOsAfy2s= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/vendor/github.com/icedream/go-bsdiff/internal/magic.go b/vendor/github.com/icedream/go-bsdiff/internal/magic.go deleted file mode 100644 index 21766eb9..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/magic.go +++ /dev/null @@ -1,39 +0,0 @@ -package internal - -import ( - "encoding/binary" - "errors" - "io" -) - -var ( - ErrInvalidMagic = errors.New("Invalid magic") - - sizeEncoding = binary.BigEndian - - magicText = []byte("ENDSLEY/BSDIFF43") -) - -func WriteHeader(w io.Writer, size uint64) (err error) { - if _, err = w.Write(magicText); err != nil { - return - } - err = binary.Write(w, sizeEncoding, size) - return -} - -func ReadHeader(r io.Reader) (size uint64, err error) { - magicBuf := make([]byte, len(magicText)) - n, err := r.Read(magicBuf) - if err != nil { - return - } - if n < len(magicText) { - err = ErrInvalidMagic - return - } - - err = binary.Read(r, sizeEncoding, &size) - - return -} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c deleted file mode 100644 index 576101c2..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.c +++ /dev/null @@ -1,56 +0,0 @@ -#include "cgo.h" - -#include "bsdiff.h" -#include "bspatch.h" - -extern int cgo_write_buffer(int bufferIndex, void* buf, int size); - -int cgo_write(struct bsdiff_stream* stream, - const void* buf, int size) { - struct buffer_table_index* bufferEntry; - - bufferEntry = (struct buffer_table_index*)stream->opaque; - - return cgo_write_buffer(bufferEntry->index, (void*)buf, size); -} - -extern int cgo_read_buffer(int bufferIndex, void* buf, int size); - -int cgo_read(const struct bspatch_stream* stream, - void* buf, int size) { - struct buffer_table_index* bufferEntry; - - bufferEntry = (struct buffer_table_index*)stream->opaque; - - return cgo_read_buffer(bufferEntry->index, buf, size) ; -} - -int bsdiff_cgo(uint8_t* oldptr, int64_t oldsize, - uint8_t* newptr, int64_t newsize, - int bufferIndex) -{ - struct bsdiff_stream stream; - stream.malloc = malloc; - stream.free = free; - stream.write = cgo_write; - - struct buffer_table_index bufferEntry; - bufferEntry.index = bufferIndex; - stream.opaque = &bufferEntry; - - return bsdiff(oldptr, oldsize, newptr, newsize, &stream); -} - -int bspatch_cgo(uint8_t* oldptr, int64_t oldsize, - uint8_t* newptr, int64_t newsize, - int bufferIndex) -{ - struct bspatch_stream stream; - stream.read = cgo_read; - - struct buffer_table_index bufferEntry; - bufferEntry.index = bufferIndex; - stream.opaque = &bufferEntry; - - return bspatch(oldptr, oldsize, newptr, newsize, &stream); -} \ No newline at end of file diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h deleted file mode 100644 index cf692e83..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo.h +++ /dev/null @@ -1,15 +0,0 @@ -#include -#include "stdint.h" - -struct buffer_table_index -{ - int index; -}; - -int bsdiff_cgo(uint8_t* oldptr, int64_t oldsize, - uint8_t* newptr, int64_t newsize, - int bufferIndex); - -int bspatch_cgo(uint8_t* oldptr, int64_t oldsize, - uint8_t* newptr, int64_t newsize, - int bufferIndex); diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go deleted file mode 100644 index 4e4967a9..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_read.go +++ /dev/null @@ -1,43 +0,0 @@ -package native - -/* -#include "bspatch.h" -*/ -import "C" -import ( - "io" - "log" - "unsafe" -) - -//export cgo_read_buffer -func cgo_read_buffer(bufferIndex C.int, - bufPtr unsafe.Pointer, length C.int) C.int { - goLength := int(length) - - if goLength == 0 { - return 0 - } - - sourceBuffer := readers.Get(int(bufferIndex)) - targetBuffer := cPtrToSlice(bufPtr, goLength) - - errCode := 0 - offset := 0 - for offset < goLength { - n, err := sourceBuffer.Read(targetBuffer) - - if err == io.EOF { - break - } else if err != nil { - log.Println("cgo_read_buffer failed:", err) - errCode = 1 - break - } - - offset += n - targetBuffer = targetBuffer[n:] - } - - return C.int(errCode) -} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go b/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go deleted file mode 100644 index 1a4e6bd1..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/cgo_write.go +++ /dev/null @@ -1,18 +0,0 @@ -package native - -/* -#include "bsdiff.h" -*/ -import "C" -import "unsafe" - -//export cgo_write_buffer -func cgo_write_buffer(bufferIndex C.int, - dataPtr unsafe.Pointer, size C.int) C.int { - buffer := writers.Get(int(bufferIndex)) - errCode := 0 - if _, err := buffer.Write(cPtrToSlice(dataPtr, int(size))); err != nil { - errCode = 1 - } - return C.int(errCode) -} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/diff.go b/vendor/github.com/icedream/go-bsdiff/internal/native/diff.go deleted file mode 100644 index 13606d46..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/diff.go +++ /dev/null @@ -1,29 +0,0 @@ -package native - -/* -#cgo CFLAGS: -I../../bsdiff - -#include "bsdiff.h" -#include "cgo.h" -*/ -import "C" -import ( - "errors" - "io" -) - -func Diff(oldbytes, newbytes []byte, patch io.Writer) (err error) { - oldptr, oldsize := bytesToUint8PtrAndSize(oldbytes) - newptr, newsize := bytesToUint8PtrAndSize(newbytes) - - bufferIndex := writers.Add(patch) - defer writers.Free(bufferIndex) - - errCode := int(C.bsdiff_cgo(oldptr, oldsize, newptr, newsize, C.int(bufferIndex))) - if errCode != 0 { - err = errors.New("bsdiff failed") - return - } - - return -} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c b/vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c deleted file mode 100644 index a9558c10..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/ext_bsdiff.c +++ /dev/null @@ -1,2 +0,0 @@ -#include "bsdiff.c" -#include "bspatch.c" \ No newline at end of file diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/native.go b/vendor/github.com/icedream/go-bsdiff/internal/native/native.go deleted file mode 100644 index 8a9348f9..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/native.go +++ /dev/null @@ -1,31 +0,0 @@ -package native - -/* -#include -*/ -import "C" -import ( - "reflect" - "unsafe" -) - -var ( - writers = writerTable{} - readers = readerTable{} -) - -func bytesToUint8PtrAndSize(bytes []byte) (ptr *C.uint8_t, size C.int64_t) { - ptr = (*C.uint8_t)(unsafe.Pointer(&bytes[0])) - size = C.int64_t(int64(len(bytes))) - return -} - -func cPtrToSlice(ptr unsafe.Pointer, size int) []byte { - var slice []byte - sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - sliceHeader.Cap = size - sliceHeader.Len = size - sliceHeader.Data = uintptr(ptr) - - return slice -} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/patch.go b/vendor/github.com/icedream/go-bsdiff/internal/native/patch.go deleted file mode 100644 index 78f59d11..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/patch.go +++ /dev/null @@ -1,30 +0,0 @@ -package native - -/* -#cgo CFLAGS: -I../../bsdiff - -#include "bspatch.h" -#include "cgo.h" -*/ -import "C" -import ( - "errors" - "io" - "strconv" -) - -func Patch(oldbytes, newbytes []byte, patch io.Reader) (err error) { - oldptr, oldsize := bytesToUint8PtrAndSize(oldbytes) - newptr, newsize := bytesToUint8PtrAndSize(newbytes) - - bufferIndex := readers.Add(patch) - defer readers.Free(bufferIndex) - - errCode := int(C.bspatch_cgo(oldptr, oldsize, newptr, newsize, C.int(bufferIndex))) - if errCode != 0 { - err = errors.New("bspatch failed with code " + strconv.Itoa(errCode)) - return - } - - return -} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go b/vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go deleted file mode 100644 index a5efa2ce..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/table_reader.go +++ /dev/null @@ -1,44 +0,0 @@ -package native - -import ( - "io" - "sync" -) - -type readerTable struct { - nextIndex int - table map[int]io.Reader - mutex sync.Mutex -} - -func (bt *readerTable) Add(reader io.Reader) (index int) { - bt.mutex.Lock() - defer bt.mutex.Unlock() - - if bt.table == nil { - bt.table = map[int]io.Reader{} - } - - index = bt.nextIndex - bt.table[index] = reader - - // TODO - Handle int overflow - - bt.nextIndex++ - - return -} - -func (bt *readerTable) Get(index int) io.Reader { - bt.mutex.Lock() - defer bt.mutex.Unlock() - - return bt.table[index] -} - -func (bt *readerTable) Free(index int) { - bt.mutex.Lock() - defer bt.mutex.Unlock() - - delete(bt.table, index) -} diff --git a/vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go b/vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go deleted file mode 100644 index ceaffd1c..00000000 --- a/vendor/github.com/icedream/go-bsdiff/internal/native/table_writer.go +++ /dev/null @@ -1,44 +0,0 @@ -package native - -import ( - "io" - "sync" -) - -type writerTable struct { - nextIndex int - table map[int]io.Writer - mutex sync.Mutex -} - -func (bt *writerTable) Add(writer io.Writer) (index int) { - bt.mutex.Lock() - defer bt.mutex.Unlock() - - if bt.table == nil { - bt.table = map[int]io.Writer{} - } - - index = bt.nextIndex - bt.table[index] = writer - - // TODO - Handle int overflow - - bt.nextIndex++ - - return -} - -func (bt *writerTable) Get(index int) io.Writer { - bt.mutex.Lock() - defer bt.mutex.Unlock() - - return bt.table[index] -} - -func (bt *writerTable) Free(index int) { - bt.mutex.Lock() - defer bt.mutex.Unlock() - - delete(bt.table, index) -} diff --git a/vendor/github.com/icedream/go-bsdiff/main.go b/vendor/github.com/icedream/go-bsdiff/main.go deleted file mode 100644 index 2d0e7f35..00000000 --- a/vendor/github.com/icedream/go-bsdiff/main.go +++ /dev/null @@ -1,16 +0,0 @@ -package bsdiff - -import ( - "io" - - "github.com/icedream/go-bsdiff/diff" - "github.com/icedream/go-bsdiff/patch" -) - -func Diff(oldReader, newReader io.Reader, patchWriter io.Writer) (err error) { - return diff.Diff(oldReader, newReader, patchWriter) -} - -func Patch(oldReader io.Reader, newWriter io.Writer, patchReader io.Reader) (err error) { - return patch.Patch(oldReader, newWriter, patchReader) -} diff --git a/vendor/github.com/icedream/go-bsdiff/patch/patch.go b/vendor/github.com/icedream/go-bsdiff/patch/patch.go deleted file mode 100644 index eb623082..00000000 --- a/vendor/github.com/icedream/go-bsdiff/patch/patch.go +++ /dev/null @@ -1,31 +0,0 @@ -package patch - -import ( - "compress/bzip2" - "io" - "io/ioutil" - - "github.com/icedream/go-bsdiff/internal" - "github.com/icedream/go-bsdiff/internal/native" -) - -func Patch(oldReader io.Reader, newWriter io.Writer, patchReader io.Reader) (err error) { - oldBytes, err := ioutil.ReadAll(oldReader) - if err != nil { - return - } - - newLen, err := internal.ReadHeader(patchReader) - if err != nil { - return - } - newBytes := make([]byte, newLen) - - // Decompression - bz2Reader := bzip2.NewReader(patchReader) - - err = native.Patch(oldBytes, newBytes, bz2Reader) - - newWriter.Write(newBytes) - return -} From ec28399677d464cad04301e67988817e0ba4cc59 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 19:52:24 +0100 Subject: [PATCH 62/74] fix: updated gatt library with latest fixes --- Gopkg.lock | 4 ++-- vendor/github.com/bettercap/gatt/adv.go | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 3a04de16..3ec72fc5 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -27,7 +27,7 @@ [[projects]] branch = "master" - digest = "1:a2c142e6c2aa1c71796c748bbe42d224e23d6638fd5b3ae153e70a4b08a8da4e" + digest = "1:881bb9d751b9408f038b83e9331ce3c57603710f3546f16e7d43b5c24e974f6d" name = "github.com/bettercap/gatt" packages = [ ".", @@ -40,7 +40,7 @@ "xpc", ] pruneopts = "UT" - revision = "277ee0d0ef94d26e3190252c59fa34dde0df4f26" + revision = "d1a17475747afe7c0d78813596d4e95801a5d592" [[projects]] branch = "master" diff --git a/vendor/github.com/bettercap/gatt/adv.go b/vendor/github.com/bettercap/gatt/adv.go index aa5e204c..40e8937d 100644 --- a/vendor/github.com/bettercap/gatt/adv.go +++ b/vendor/github.com/bettercap/gatt/adv.go @@ -1981,6 +1981,13 @@ func (a *Advertisement) unmarshall(b []byte) error { // Utility function for creating a list of uuids. uuidList := func(u []UUID, d []byte, w int) []UUID { + // https://github.com/bettercap/gatt/issues/8 + defer func() { + if recover() != nil { + + } + }() + for len(d) > 0 { u = append(u, UUID{d[:w]}) d = d[w:] From c3f0e3598bb2f400122d8391474e291a936207d3 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 20:03:46 +0100 Subject: [PATCH 63/74] misc: small fix or general refactoring i did not bother commenting --- modules/api_rest/api_rest_replay.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/api_rest/api_rest_replay.go b/modules/api_rest/api_rest_replay.go index b7072d3e..7376f6e3 100644 --- a/modules/api_rest/api_rest_replay.go +++ b/modules/api_rest/api_rest_replay.go @@ -26,6 +26,10 @@ func (mod *RestAPI) startReplay(filename string) (err error) { } mod.State.Store("load_progress", 0) + defer func() { + mod.State.Store("load_progress", 100.0) + }() + mod.loading = true defer func() { mod.loading = false From 1eec682aeb9ca1f91cae7e5a5631f4e422a71c47 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 20:16:41 +0100 Subject: [PATCH 64/74] new: new ble.device parameter to set HCI index (closes #519) --- modules/ble/ble_recon.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/ble/ble_recon.go b/modules/ble/ble_recon.go index afb2e216..d30bf03d 100644 --- a/modules/ble/ble_recon.go +++ b/modules/ble/ble_recon.go @@ -20,6 +20,7 @@ import ( type BLERecon struct { session.SessionModule + deviceId int gattDevice gatt.Device currDevice *network.BLEDevice writeUUID *gatt.UUID @@ -34,6 +35,7 @@ type BLERecon struct { func NewBLERecon(s *session.Session) *BLERecon { mod := &BLERecon{ SessionModule: session.NewSessionModule("ble.recon", s), + deviceId: -1, gattDevice: nil, quit: make(chan bool), done: make(chan bool), @@ -110,6 +112,10 @@ func NewBLERecon(s *session.Session) *BLERecon { mod.AddHandler(write) + mod.AddParam(session.NewIntParameter("ble.device", + fmt.Sprintf("%d", mod.deviceId), + "Index of the HCI device to use, -1 to autodetect.")) + return mod } @@ -142,11 +148,21 @@ func (mod *BLERecon) Configure() (err error) { if mod.Running() { return session.ErrAlreadyStarted(mod.Name()) } else if mod.gattDevice == nil { - mod.Debug("initializing device ...") + if err, mod.deviceId = mod.IntParam("ble.device"); err != nil { + return err + } + + mod.Debug("initializing device (id:%d) ...", mod.deviceId) golog.SetFlags(0) golog.SetOutput(dummyWriter{mod}) - if mod.gattDevice, err = gatt.NewDevice(defaultBLEClientOptions...); err != nil { + + options := []gatt.Option{ + gatt.LnxMaxConnections(255), + gatt.LnxDeviceID(mod.deviceId, true), + } + + if mod.gattDevice, err = gatt.NewDevice(options...); err != nil { mod.Debug("error while creating new gatt device: %v", err) return err } From 5fb4cd6a5a9398dd58d10ab4efe5df2772e285e1 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 21:23:08 +0100 Subject: [PATCH 65/74] misc: small fix or general refactoring i did not bother commenting --- modules/gps/gps.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/gps/gps.go b/modules/gps/gps.go index 8e46e331..845e0090 100644 --- a/modules/gps/gps.go +++ b/modules/gps/gps.go @@ -126,6 +126,8 @@ func (mod *GPS) Start() error { return mod.SetRunning(true, func() { defer mod.serial.Close() + mod.Info("started on port %s ...", mod.serialPort) + for mod.Running() { if line, err := mod.readLine(); err == nil { if s, err := nmea.Parse(line); err == nil { From 054d8a5a3e9395066feb91eaa547f9b09a0a98f1 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 21:45:43 +0100 Subject: [PATCH 66/74] misc: keeping frames in memory as compressed --- modules/api_rest/record.go | 74 ++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/modules/api_rest/record.go b/modules/api_rest/record.go index 4693ce2a..6e061cd6 100644 --- a/modules/api_rest/record.go +++ b/modules/api_rest/record.go @@ -15,6 +15,36 @@ import ( "github.com/kr/binarydist" ) +func compress(data []byte) (error, []byte) { + compressed := new(bytes.Buffer) + compress := gzip.NewWriter(compressed) + + if _, err := compress.Write(data); err != nil { + return err, nil + } else if err = compress.Flush(); err != nil { + return err, nil + } else if err = compress.Close(); err != nil { + return err, nil + } + + return nil, compressed.Bytes() +} + +func decompress(data []byte) (error, []byte) { + decompress, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err, nil + } + defer decompress.Close() + + raw, err := ioutil.ReadAll(decompress) + if err != nil { + return err, nil + } + + return nil, raw +} + type patch []byte type frame []byte @@ -79,7 +109,7 @@ func (e *RecordEntry) Reset() { e.CurState = 0 } -func (e *RecordEntry) Compile() error { +func (e *RecordEntry) Compile() (err error) { e.Lock() defer e.Unlock() @@ -90,7 +120,10 @@ func (e *RecordEntry) Compile() error { e.frames = make([]frame, e.NumStates+1) // first is the master frame - e.frames[0] = frame(e.Data) + if err, e.frames[0] = compress(e.Data); err != nil { + return + } + // precompute frames so they can be accessed by index for i := 0; i < e.NumStates; i++ { patch := e.States[i] @@ -103,7 +136,10 @@ func (e *RecordEntry) Compile() error { } e.Cur = newWriter.Bytes() - e.frames[i+1] = e.Cur + + if err, e.frames[i+1] = compress(e.Cur); err != nil { + return err + } e.progress(1) } @@ -143,7 +179,15 @@ func (e *RecordEntry) Next() []byte { defer e.Unlock() cur := e.CurState e.CurState++ - return e.frames[cur] + + zFrame := e.frames[cur] + + err, data := decompress(zFrame) + if err != nil { + panic(err) + } + + return data } // the Record object represents a recorded session @@ -187,16 +231,10 @@ func LoadRecord(fileName string, mod *session.SessionModule) (*Record, error) { return nil, fmt.Errorf("error while reading %s: %s", fileName, err) } - decompress, err := gzip.NewReader(bytes.NewReader(compressed)) + err, raw := decompress(compressed) if err != nil { return nil, fmt.Errorf("error while reading gzip file %s: %s", fileName, err) } - defer decompress.Close() - - raw, err := ioutil.ReadAll(decompress) - if err != nil { - return nil, fmt.Errorf("error while decompressing %s: %s", fileName, err) - } rec := &Record{} @@ -244,20 +282,12 @@ func (r *Record) save() error { return err } - data := buf.Bytes() - - compressed := new(bytes.Buffer) - compress := gzip.NewWriter(compressed) - - if _, err := compress.Write(data); err != nil { - return err - } else if err = compress.Flush(); err != nil { - return err - } else if err = compress.Close(); err != nil { + err, data := compress(buf.Bytes()) + if err != nil { return err } - return ioutil.WriteFile(r.fileName, compressed.Bytes(), os.ModePerm) + return ioutil.WriteFile(r.fileName, data, os.ModePerm) } func (r *Record) Flush() error { From 2f53e40f98f1eedb890fe1007e19932713e2879c Mon Sep 17 00:00:00 2001 From: evilsocket Date: Fri, 29 Mar 2019 21:52:03 +0100 Subject: [PATCH 67/74] Revert "misc: keeping frames in memory as compressed" This reverts commit 054d8a5a3e9395066feb91eaa547f9b09a0a98f1. --- modules/api_rest/record.go | 74 ++++++++++++-------------------------- 1 file changed, 22 insertions(+), 52 deletions(-) diff --git a/modules/api_rest/record.go b/modules/api_rest/record.go index 6e061cd6..4693ce2a 100644 --- a/modules/api_rest/record.go +++ b/modules/api_rest/record.go @@ -15,36 +15,6 @@ import ( "github.com/kr/binarydist" ) -func compress(data []byte) (error, []byte) { - compressed := new(bytes.Buffer) - compress := gzip.NewWriter(compressed) - - if _, err := compress.Write(data); err != nil { - return err, nil - } else if err = compress.Flush(); err != nil { - return err, nil - } else if err = compress.Close(); err != nil { - return err, nil - } - - return nil, compressed.Bytes() -} - -func decompress(data []byte) (error, []byte) { - decompress, err := gzip.NewReader(bytes.NewReader(data)) - if err != nil { - return err, nil - } - defer decompress.Close() - - raw, err := ioutil.ReadAll(decompress) - if err != nil { - return err, nil - } - - return nil, raw -} - type patch []byte type frame []byte @@ -109,7 +79,7 @@ func (e *RecordEntry) Reset() { e.CurState = 0 } -func (e *RecordEntry) Compile() (err error) { +func (e *RecordEntry) Compile() error { e.Lock() defer e.Unlock() @@ -120,10 +90,7 @@ func (e *RecordEntry) Compile() (err error) { e.frames = make([]frame, e.NumStates+1) // first is the master frame - if err, e.frames[0] = compress(e.Data); err != nil { - return - } - + e.frames[0] = frame(e.Data) // precompute frames so they can be accessed by index for i := 0; i < e.NumStates; i++ { patch := e.States[i] @@ -136,10 +103,7 @@ func (e *RecordEntry) Compile() (err error) { } e.Cur = newWriter.Bytes() - - if err, e.frames[i+1] = compress(e.Cur); err != nil { - return err - } + e.frames[i+1] = e.Cur e.progress(1) } @@ -179,15 +143,7 @@ func (e *RecordEntry) Next() []byte { defer e.Unlock() cur := e.CurState e.CurState++ - - zFrame := e.frames[cur] - - err, data := decompress(zFrame) - if err != nil { - panic(err) - } - - return data + return e.frames[cur] } // the Record object represents a recorded session @@ -231,10 +187,16 @@ func LoadRecord(fileName string, mod *session.SessionModule) (*Record, error) { return nil, fmt.Errorf("error while reading %s: %s", fileName, err) } - err, raw := decompress(compressed) + decompress, err := gzip.NewReader(bytes.NewReader(compressed)) if err != nil { return nil, fmt.Errorf("error while reading gzip file %s: %s", fileName, err) } + defer decompress.Close() + + raw, err := ioutil.ReadAll(decompress) + if err != nil { + return nil, fmt.Errorf("error while decompressing %s: %s", fileName, err) + } rec := &Record{} @@ -282,12 +244,20 @@ func (r *Record) save() error { return err } - err, data := compress(buf.Bytes()) - if err != nil { + data := buf.Bytes() + + compressed := new(bytes.Buffer) + compress := gzip.NewWriter(compressed) + + if _, err := compress.Write(data); err != nil { + return err + } else if err = compress.Flush(); err != nil { + return err + } else if err = compress.Close(); err != nil { return err } - return ioutil.WriteFile(r.fileName, data, os.ModePerm) + return ioutil.WriteFile(r.fileName, compressed.Bytes(), os.ModePerm) } func (r *Record) Flush() error { From 50d01429cdb6cdd52b5cb1ad0315f0ca9f0966a5 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 30 Mar 2019 00:26:38 +0100 Subject: [PATCH 68/74] new: added Updated field to session.GPS --- modules/gps/gps.go | 2 ++ session/session.go | 1 + 2 files changed, 3 insertions(+) diff --git a/modules/gps/gps.go b/modules/gps/gps.go index 845e0090..b71b67fd 100644 --- a/modules/gps/gps.go +++ b/modules/gps/gps.go @@ -133,6 +133,7 @@ func (mod *GPS) Start() error { if s, err := nmea.Parse(line); err == nil { // http://aprs.gids.nl/nmea/#gga if m, ok := s.(nmea.GNGGA); ok { + mod.Session.GPS.Updated = time.Now() mod.Session.GPS.Latitude = m.Latitude mod.Session.GPS.Longitude = m.Longitude mod.Session.GPS.FixQuality = m.FixQuality @@ -141,6 +142,7 @@ func (mod *GPS) Start() error { mod.Session.GPS.Altitude = m.Altitude mod.Session.GPS.Separation = m.Separation } else if m, ok := s.(nmea.GPGGA); ok { + mod.Session.GPS.Updated = time.Now() mod.Session.GPS.Latitude = m.Latitude mod.Session.GPS.Longitude = m.Longitude mod.Session.GPS.FixQuality = m.FixQuality diff --git a/session/session.go b/session/session.go index e4661086..c9aeedc8 100644 --- a/session/session.go +++ b/session/session.go @@ -52,6 +52,7 @@ func ErrAlreadyStopped(name string) error { type UnknownCommandCallback func(cmd string) bool type GPS struct { + Updated time.Time Latitude float64 // Latitude. Longitude float64 // Longitude. FixQuality string // Quality of fix. From 54116f7fbe75e9bfb9a44a0a3332397e251ea03e Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 30 Mar 2019 13:59:08 +0100 Subject: [PATCH 69/74] fix: fixed replay time computation using actual dates instead of the assumption of one frame per second --- modules/api_rest/api_rest.go | 4 ++++ modules/api_rest/api_rest_controller.go | 2 +- modules/api_rest/api_rest_replay.go | 10 +++++++- modules/api_rest/record.go | 31 +++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index b20e8030..28d66565 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -35,6 +35,8 @@ type RestAPI struct { recordFileName string recordWait *sync.WaitGroup record *Record + recStarted time.Time + recStopped time.Time } func NewRestAPI(s *session.Session) *RestAPI { @@ -65,6 +67,8 @@ func NewRestAPI(s *session.Session) *RestAPI { mod.State.Store("rec_filename", &mod.recordFileName) mod.State.Store("rec_frames", 0) mod.State.Store("rec_cur_frame", 0) + mod.State.Store("rec_started", &mod.recStarted) + mod.State.Store("rec_stopped", &mod.recStopped) mod.AddParam(session.NewStringParameter("api.rest.address", "127.0.0.1", diff --git a/modules/api_rest/api_rest_controller.go b/modules/api_rest/api_rest_controller.go index 72e7c43e..da9b4d63 100644 --- a/modules/api_rest/api_rest_controller.go +++ b/modules/api_rest/api_rest_controller.go @@ -36,7 +36,7 @@ func (mod *RestAPI) setAuthFailed(w http.ResponseWriter, r *http.Request) { func (mod *RestAPI) toJSON(w http.ResponseWriter, o interface{}) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(o); err != nil { - fmt.Printf("error while encoding object to JSON: %v\n", err) + mod.Warning("error while encoding object to JSON: %v", err) } } diff --git a/modules/api_rest/api_rest_replay.go b/modules/api_rest/api_rest_replay.go index 7376f6e3..b04c9ef4 100644 --- a/modules/api_rest/api_rest_replay.go +++ b/modules/api_rest/api_rest_replay.go @@ -50,10 +50,18 @@ func (mod *RestAPI) startReplay(filename string) (err error) { } } + mod.recStarted = mod.record.Session.StartedAt() + mod.recStopped = mod.record.Session.StoppedAt() + duration := mod.recStopped.Sub(mod.recStarted) + mod.recTime = int(duration.Seconds()) mod.replaying = true mod.recording = false - mod.Info("loaded %d frames in %s, started replaying ...", mod.record.Session.Frames(), loadedIn) + mod.Info("loaded %s of recording (%d frames) started at %s in %s, started replaying ...", + duration, + mod.record.Session.Frames(), + mod.recStarted, + loadedIn) return nil } diff --git a/modules/api_rest/record.go b/modules/api_rest/record.go index 4693ce2a..cd52765f 100644 --- a/modules/api_rest/record.go +++ b/modules/api_rest/record.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "sync" + "time" "github.com/bettercap/bettercap/session" @@ -146,6 +147,36 @@ func (e *RecordEntry) Next() []byte { return e.frames[cur] } +func (e *RecordEntry) TimeOf(idx int) time.Time { + e.Lock() + defer e.Unlock() + + buf := e.frames[idx] + frame := make(map[string]interface{}) + + if err := json.Unmarshal(buf, &frame); err != nil { + fmt.Printf("%v\n", err) + return time.Time{} + } else if tm, err := time.Parse(time.RFC3339, frame["polled_at"].(string)); err != nil { + fmt.Printf("%v\n", err) + return time.Time{} + } else { + return tm + } +} + +func (e *RecordEntry) StartedAt() time.Time { + return e.TimeOf(0) +} + +func (e *RecordEntry) StoppedAt() time.Time { + return e.TimeOf(e.NumStates) +} + +func (e *RecordEntry) Duration() time.Duration { + return e.StoppedAt().Sub(e.StartedAt()) +} + // the Record object represents a recorded session type Record struct { sync.Mutex From afe300cd8ae61d6a8139ee68c3b3e6765422ded6 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 30 Mar 2019 16:17:26 +0100 Subject: [PATCH 70/74] fix: gracefully handling wifi device disconnection --- core/core.go | 11 +------- modules/wifi/wifi.go | 11 ++++++++ modules/wifi/wifi_hopping.go | 51 ++++++++++++++++++++++++++++-------- network/net.go | 2 +- 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/core/core.go b/core/core.go index f79e6ffc..3761a6c7 100644 --- a/core/core.go +++ b/core/core.go @@ -1,7 +1,6 @@ package core import ( - "fmt" "os/exec" "sort" @@ -34,7 +33,7 @@ func HasBinary(executable string) bool { return true } -func ExecSilent(executable string, args []string) (string, error) { +func Exec(executable string, args []string) (string, error) { path, err := exec.LookPath(executable) if err != nil { return "", err @@ -47,11 +46,3 @@ func ExecSilent(executable string, args []string) (string, error) { return str.Trim(string(raw)), nil } } - -func Exec(executable string, args []string) (string, error) { - out, err := ExecSilent(executable, args) - if err != nil { - fmt.Printf("ERROR for '%s %s': %s\n", executable, args, err) - } - return out, err -} diff --git a/modules/wifi/wifi.go b/modules/wifi/wifi.go index 4ac37e34..2a48b4a1 100644 --- a/modules/wifi/wifi.go +++ b/modules/wifi/wifi.go @@ -561,6 +561,17 @@ func (mod *WiFiModule) Start() error { return nil } +func (mod *WiFiModule) forcedStop() error { + return mod.SetRunning(false, func() { + // signal the main for loop we want to exit + if !mod.pktSourceChanClosed { + mod.pktSourceChan <- nil + } + // close the pcap handle to make the main for exit + mod.handle.Close() + }) +} + func (mod *WiFiModule) Stop() error { return mod.SetRunning(false, func() { // wait any pending write operation diff --git a/modules/wifi/wifi_hopping.go b/modules/wifi/wifi_hopping.go index 79fea79e..99fdbf07 100644 --- a/modules/wifi/wifi_hopping.go +++ b/modules/wifi/wifi_hopping.go @@ -1,11 +1,47 @@ package wifi import ( + "net" "time" "github.com/bettercap/bettercap/network" ) +func (mod *WiFiModule) isInterfaceConnected() bool { + ifaces, err := net.Interfaces() + if err != nil { + mod.Error("error while enumerating interfaces: %s", err) + return false + } + + for _, iface := range ifaces { + if mod.iface.HwAddress == network.NormalizeMac(iface.HardwareAddr.String()) { + return true + } + } + + return false +} + +func (mod *WiFiModule) hop(channel int) (mustStop bool) { + mod.chanLock.Lock() + defer mod.chanLock.Unlock() + + mod.Debug("hopping on channel %d", channel) + + if err := network.SetInterfaceChannel(mod.iface.Name(), channel); err != nil { + // check if the device has been disconnected + if mod.isInterfaceConnected() == false { + mod.Error("interface %s disconnected, stopping module", mod.iface.Name()) + mustStop = true + } else { + mod.Warning("error while hopping to channel %d: %s", channel, err) + } + } + + return +} + func (mod *WiFiModule) onChannel(channel int, cb func()) { mod.chanLock.Lock() defer mod.chanLock.Unlock() @@ -13,11 +49,7 @@ func (mod *WiFiModule) onChannel(channel int, cb func()) { prev := mod.stickChan mod.stickChan = channel - if err := network.SetInterfaceChannel(mod.iface.Name(), channel); err != nil { - mod.Warning("error while hopping to channel %d: %s", channel, err) - } else { - mod.Debug("hopped on channel %d", channel) - } + mod.hop(channel) cb() @@ -50,13 +82,10 @@ func (mod *WiFiModule) channelHopper() { channel = mod.stickChan } - mod.Debug("hopping on channel %d", channel) - - mod.chanLock.Lock() - if err := network.SetInterfaceChannel(mod.iface.Name(), channel); err != nil { - mod.Warning("error while hopping to channel %d: %s", channel, err) + if stop := mod.hop(channel); stop { + mod.forcedStop() + return } - mod.chanLock.Unlock() select { case _ = <-mod.hopChanges: diff --git a/network/net.go b/network/net.go index 37123537..47be68d8 100644 --- a/network/net.go +++ b/network/net.go @@ -286,7 +286,7 @@ func ActivateInterface(name string) error { func SetInterfaceTxPower(name string, txpower int) error { if core.HasBinary("iwconfig") { - if out, err := core.ExecSilent("iwconfig", []string{name, "txpower", fmt.Sprintf("%d", txpower)}); err != nil { + if out, err := core.Exec("iwconfig", []string{name, "txpower", fmt.Sprintf("%d", txpower)}); err != nil { return err } else if out != "" { return fmt.Errorf("unexpected output while setting txpower to %d for interface %s: %s", txpower, name, out) From 0113286b4fc5832144df8a2532b032ee65f6702c Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 30 Mar 2019 16:27:56 +0100 Subject: [PATCH 71/74] fix: gracefully handling hid receiver disconnection --- modules/hid/hid.go | 10 ++++++++++ modules/hid/hid_recon.go | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/modules/hid/hid.go b/modules/hid/hid.go index 424267f2..49b239c5 100644 --- a/modules/hid/hid.go +++ b/modules/hid/hid.go @@ -204,11 +204,21 @@ func (mod *HIDRecon) Configure() error { return nil } +func (mod *HIDRecon) forceStop() error { + return mod.SetRunning(false, func() { + if mod.dongle != nil { + mod.dongle.Close() + mod.dongle = nil + mod.Debug("device closed") + } + }) +} func (mod *HIDRecon) Stop() error { return mod.SetRunning(false, func() { mod.waitGroup.Wait() if mod.dongle != nil { mod.dongle.Close() + mod.dongle = nil mod.Debug("device closed") } }) diff --git a/modules/hid/hid_recon.go b/modules/hid/hid_recon.go index b4771a03..fc5d29e1 100644 --- a/modules/hid/hid_recon.go +++ b/modules/hid/hid_recon.go @@ -4,6 +4,7 @@ import ( "time" "github.com/bettercap/nrf24" + "github.com/google/gousb" ) func (mod *HIDRecon) doHopping() { @@ -26,7 +27,13 @@ func (mod *HIDRecon) doHopping() { mod.channel = 1 } if err := mod.dongle.SetChannel(mod.channel); err != nil { - mod.Warning("error hopping on channel %d: %v", mod.channel, err) + if err == gousb.ErrorNoDevice { + mod.Error("device disconnected, stopping module") + mod.forceStop() + return + } else { + mod.Warning("error hopping on channel %d: %v", mod.channel, err) + } } else { mod.lastHop = time.Now() } @@ -107,6 +114,11 @@ func (mod *HIDRecon) Start() error { buf, err := mod.dongle.ReceivePayload() if err != nil { + if err == gousb.ErrorNoDevice { + mod.Error("device disconnected, stopping module") + mod.forceStop() + return + } mod.Warning("error receiving payload from channel %d: %v", mod.channel, err) continue } From 460bd9b159a81946c844b86df9db11ed7d037393 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 30 Mar 2019 17:32:53 +0100 Subject: [PATCH 72/74] new: api.rest.record.clock parameter to decide delay per sample --- modules/api_rest/api_rest.go | 11 +++++++++-- modules/api_rest/api_rest_record.go | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/modules/api_rest/api_rest.go b/modules/api_rest/api_rest.go index 28d66565..c3b6f082 100644 --- a/modules/api_rest/api_rest.go +++ b/modules/api_rest/api_rest.go @@ -28,6 +28,7 @@ type RestAPI struct { upgrader websocket.Upgrader quit chan bool + recClock int recording bool recTime int loading bool @@ -50,6 +51,7 @@ func NewRestAPI(s *session.Session) *RestAPI { ReadBufferSize: 1024, WriteBufferSize: 1024, }, + recClock: 1, recording: false, recTime: 0, loading: false, @@ -60,6 +62,7 @@ func NewRestAPI(s *session.Session) *RestAPI { } mod.State.Store("recording", &mod.recording) + mod.State.Store("rec_clock", &mod.recClock) mod.State.Store("replaying", &mod.replaying) mod.State.Store("loading", &mod.loading) mod.State.Store("load_progress", 0) @@ -122,6 +125,10 @@ func NewRestAPI(s *session.Session) *RestAPI { return mod.Stop() })) + mod.AddParam(session.NewIntParameter("api.rest.record.clock", + "1", + "Number of seconds to wait while recording with api.rest.record between one sample and the next one.")) + mod.AddHandler(session.NewModuleHandler("api.rest.record off", "", "Stop recording the session.", func(args []string) error { @@ -129,7 +136,7 @@ func NewRestAPI(s *session.Session) *RestAPI { })) mod.AddHandler(session.NewModuleHandler("api.rest.record FILENAME", `api\.rest\.record (.+)`, - "Start polling the rest API every second recording each sample as a session file that can be replayed.", + "Start polling the rest API periodically recording each sample in a compressed file that can be later replayed.", func(args []string) error { return mod.startRecording(args[0]) })) @@ -141,7 +148,7 @@ func NewRestAPI(s *session.Session) *RestAPI { })) mod.AddHandler(session.NewModuleHandler("api.rest.replay FILENAME", `api\.rest\.replay (.+)`, - "Start the rest API module in replay mode using FILENAME as the recorded session file.", + "Start the rest API module in replay mode using FILENAME as the recorded session file, will revert to normal mode once the replay is over.", func(args []string) error { return mod.startReplay(args[0]) })) diff --git a/modules/api_rest/api_rest_record.go b/modules/api_rest/api_rest_record.go index 22f03cd6..fd5d7c09 100644 --- a/modules/api_rest/api_rest_record.go +++ b/modules/api_rest/api_rest_record.go @@ -40,17 +40,21 @@ func (mod *RestAPI) recordState() error { } func (mod *RestAPI) recorder() { + clock := time.Duration(mod.recClock) * time.Second + mod.recTime = 0 mod.recording = true mod.replaying = false mod.record = NewRecord(mod.recordFileName, &mod.SessionModule) - mod.Info("started recording to %s ...", mod.recordFileName) + mod.Info("started recording to %s (clock %s) ...", mod.recordFileName, clock) mod.recordWait.Add(1) defer mod.recordWait.Done() tick := time.NewTicker(1 * time.Second) + lastSampled := time.Time{} + for range tick.C { if !mod.recording { break @@ -58,10 +62,13 @@ func (mod *RestAPI) recorder() { mod.recTime++ - if err := mod.recordState(); err != nil { - mod.Error("error while recording: %s", err) - mod.recording = false - break + if time.Since(lastSampled) >= clock { + lastSampled = time.Now() + if err := mod.recordState(); err != nil { + mod.Error("error while recording: %s", err) + mod.recording = false + break + } } } @@ -73,6 +80,8 @@ func (mod *RestAPI) startRecording(filename string) (err error) { return mod.errAlreadyRecording() } else if mod.replaying { return mod.errAlreadyReplaying() + } else if err, mod.recClock = mod.IntParam("api.rest.record.clock"); err != nil { + return err } else if mod.recordFileName, err = fs.Expand(filename); err != nil { return err } From f9ebb1a77edca5313180911670165fcf3d4adbf1 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 30 Mar 2019 17:33:02 +0100 Subject: [PATCH 73/74] fix --- modules/hid/hid.go | 2 -- modules/hid/hid_recon.go | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/hid/hid.go b/modules/hid/hid.go index 49b239c5..1ae5b3b4 100644 --- a/modules/hid/hid.go +++ b/modules/hid/hid.go @@ -208,7 +208,6 @@ func (mod *HIDRecon) forceStop() error { return mod.SetRunning(false, func() { if mod.dongle != nil { mod.dongle.Close() - mod.dongle = nil mod.Debug("device closed") } }) @@ -218,7 +217,6 @@ func (mod *HIDRecon) Stop() error { mod.waitGroup.Wait() if mod.dongle != nil { mod.dongle.Close() - mod.dongle = nil mod.Debug("device closed") } }) diff --git a/modules/hid/hid_recon.go b/modules/hid/hid_recon.go index fc5d29e1..13df827a 100644 --- a/modules/hid/hid_recon.go +++ b/modules/hid/hid_recon.go @@ -27,7 +27,7 @@ func (mod *HIDRecon) doHopping() { mod.channel = 1 } if err := mod.dongle.SetChannel(mod.channel); err != nil { - if err == gousb.ErrorNoDevice { + if err == gousb.ErrorNoDevice || err == gousb.TransferStall { mod.Error("device disconnected, stopping module") mod.forceStop() return @@ -114,7 +114,7 @@ func (mod *HIDRecon) Start() error { buf, err := mod.dongle.ReceivePayload() if err != nil { - if err == gousb.ErrorNoDevice { + if err == gousb.ErrorNoDevice || err == gousb.TransferStall { mod.Error("device disconnected, stopping module") mod.forceStop() return From 5ef330f80be7d2b3f3f3549ae1955dfae9696ee5 Mon Sep 17 00:00:00 2001 From: evilsocket Date: Sat, 30 Mar 2019 18:19:34 +0100 Subject: [PATCH 74/74] misc: small fix or general refactoring i did not bother commenting --- firewall/firewall_darwin.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/firewall/firewall_darwin.go b/firewall/firewall_darwin.go index ac91b370..8a83255e 100644 --- a/firewall/firewall_darwin.go +++ b/firewall/firewall_darwin.go @@ -41,7 +41,7 @@ func Make(iface *network.Endpoint) FirewallManager { } func (f PfFirewall) sysCtlRead(param string) (string, error) { - if out, err := core.ExecSilent("sysctl", []string{param}); err != nil { + if out, err := core.Exec("sysctl", []string{param}); err != nil { return "", err } else if m := sysCtlParser.FindStringSubmatch(out); len(m) == 3 && m[1] == param { return m[2], nil @@ -52,7 +52,7 @@ func (f PfFirewall) sysCtlRead(param string) (string, error) { func (f PfFirewall) sysCtlWrite(param string, value string) (string, error) { args := []string{"-w", fmt.Sprintf("%s=%s", param, value)} - _, err := core.ExecSilent("sysctl", args) + _, err := core.Exec("sysctl", args) if err != nil { return "", err } @@ -115,9 +115,9 @@ func (f PfFirewall) generateRule(r *Redirection) string { func (f *PfFirewall) enable(enabled bool) { f.enabled = enabled if enabled { - core.ExecSilent("pfctl", []string{"-e"}) + core.Exec("pfctl", []string{"-e"}) } else { - core.ExecSilent("pfctl", []string{"-d"}) + core.Exec("pfctl", []string{"-d"}) } } @@ -139,7 +139,7 @@ func (f PfFirewall) EnableRedirection(r *Redirection, enabled bool) error { f.enable(true) // load the rule - if _, err := core.ExecSilent("pfctl", []string{"-f", f.filename}); err != nil { + if _, err := core.Exec("pfctl", []string{"-f", f.filename}); err != nil { return err } } else {