From 684141d7aeb15da9c95452e4d9db2428c27683c1 Mon Sep 17 00:00:00 2001 From: trietend <212042+trietend@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:09:15 +0200 Subject: [PATCH 1/6] Added config for psql --- Responder.conf | 11 +++++++++++ settings.py | 12 +++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Responder.conf b/Responder.conf index a9eac42..528c28d 100755 --- a/Responder.conf +++ b/Responder.conf @@ -27,10 +27,21 @@ MQTT = On ; Use "Random" for generating a random challenge for each requests (Default) Challenge = Random +; Database Management System +; Use "sqlite" or "psql" +Dbms = sqlite + ; SQLite Database file ; Delete this file to re-capture previously captured hashes Database = Responder.db +; Psql Database +PsqlHost = 127.0.0.1 +PsqlPort = 5432 +PsqlUser= dbuser +PsqlPassword = dbpass +PsqlDatabase = responder + ; Default log file SessionLog = Responder-Session.log diff --git a/settings.py b/settings.py index ee96190..258ee7c 100644 --- a/settings.py +++ b/settings.py @@ -138,9 +138,19 @@ class Settings: self.Krb_On_Off = self.toBool(config.get('Responder Core', 'Kerberos')) self.SNMP_On_Off = self.toBool(config.get('Responder Core', 'SNMP')) - # Db File + # Db + self.Dbms = config.get('Responder Core', 'Dbms') + + # Db File Sqlite self.DatabaseFile = os.path.join(self.ResponderPATH, config.get('Responder Core', 'Database')) + # Db Psql + self.PsqlHost = config.get('Responder Core', 'PsqlHost') + self.PsqlPort = config.get('Responder Core', 'PsqlPort') + self.PsqlUser = config.get('Responder Core', 'PsqlUser') + self.PsqlPassword = config.get('Responder Core', 'PsqlPassword') + self.PsqlDatabase = config.get('Responder Core', 'PsqlDatabase') + # Log Files self.LogDir = os.path.join(self.ResponderPATH, 'logs') From 2b26aa8dfdc0f7961c001b533e03fdadec7d1761 Mon Sep 17 00:00:00 2001 From: trietend <212042+trietend@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:00:34 +0200 Subject: [PATCH 2/6] Added implementation for psql --- utils.py | 157 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 127 insertions(+), 30 deletions(-) diff --git a/utils.py b/utils.py index 38a72b1..1341dd2 100644 --- a/utils.py +++ b/utils.py @@ -25,6 +25,7 @@ import datetime import codecs import struct import random +import psycopg try: import netifaces except: @@ -313,7 +314,26 @@ def NetworkRecvBufferPython2or3(data): else: return str(data.decode('latin-1')) -def CreateResponderDb(): +def PsqlConnect(): + conn = psycopg.connect(dbname=settings.Config.PsqlDatabase, + host=settings.Config.PsqlHost, + user=settings.Config.PsqlUser, + password=settings.Config.PsqlPassword, + port=settings.Config.PsqlPort) + return conn + +def _CreateResponderDbPsql(): + conn = PsqlConnect() + cursor = conn.cursor() + cursor.execute('CREATE TABLE IF NOT EXISTS Poisoned (timestamp TEXT, Poisoner TEXT, SentToIp TEXT, ForName TEXT, AnalyzeMode TEXT)') + conn.commit() + cursor.execute('CREATE TABLE IF NOT EXISTS responder (timestamp TEXT, module TEXT, type TEXT, client TEXT, hostname TEXT, username TEXT, cleartext TEXT, hash TEXT, fullhash TEXT)') + conn.commit() + cursor.execute('CREATE TABLE IF NOT EXISTS DHCP (timestamp TEXT, MAC TEXT, IP TEXT, RequestedIP TEXT)') + conn.commit() + conn.close() + +def _CreateResponderDbSqlite(): if not os.path.exists(settings.Config.DatabaseFile): cursor = sqlite3.connect(settings.Config.DatabaseFile) cursor.execute('CREATE TABLE Poisoned (timestamp TEXT, Poisoner TEXT, SentToIp TEXT, ForName TEXT, AnalyzeMode TEXT)') @@ -324,6 +344,58 @@ def CreateResponderDb(): cursor.commit() cursor.close() +def CreateResponderDb(): + if settings.Config.Dbms.lower() == 'psql': + _CreateResponderDbPsql() + else: + _CreateResponderDbSqlite() + + +def _SaveToDbPsql(result): + conn = PsqlConnect() + cursor = conn.cursor() + + if len(result['cleartext']): + cursor.execute("SELECT COUNT(*) AS count FROM responder WHERE module=%s AND type=%s AND client=%s AND LOWER(username)=LOWER(%s) AND cleartext=%s", (result['module'], result['type'], result['client'], result['user'], result['cleartext'])) + else: + cursor.execute("SELECT COUNT(*) AS count FROM responder WHERE module=%s AND type=%s AND client=%s AND LOWER(username)=LOWER(%s)", (result['module'], result['type'], result['client'], result['user'])) + + (count,) = cursor.fetchone() + + if not count: + cursor.execute("INSERT INTO responder VALUES(NOW(), %s, %s, %s, %s, %s, %s, %s, %s)", (result['module'], result['type'], result['client'], result['hostname'], result['user'], result['cleartext'], result['hash'], result['fullhash'])) + conn.commit() + + if count and not settings.Config.Verbose and not len(result['cleartext']): + print(color('[*] Skipping previously captured hash for %s' % result['user'], 3, 1)) + text('[*] Skipping previously captured hash for %s' % result['user']) + cursor.execute("UPDATE responder SET timestamp=NOW() WHERE user=%s AND client=%s", (result['user'], result['client'])) + conn.commit() + cursor.close() + return count + +def _SaveToDbSqlite(result): + cursor = sqlite3.connect(settings.Config.DatabaseFile) + cursor.text_factory = sqlite3.Binary # We add a text factory to support different charsets + + if len(result['cleartext']): + res = cursor.execute("SELECT COUNT(*) AS count FROM responder WHERE module=? AND type=? AND client=? AND LOWER(user)=LOWER(?) AND cleartext=?", (result['module'], result['type'], result['client'], result['user'], result['cleartext'])) + else: + res = cursor.execute("SELECT COUNT(*) AS count FROM responder WHERE module=? AND type=? AND client=? AND LOWER(user)=LOWER(?)", (result['module'], result['type'], result['client'], result['user'])) + + (count,) = res.fetchone() + if not count: + cursor.execute("INSERT INTO responder VALUES(datetime('now'), ?, ?, ?, ?, ?, ?, ?, ?)", (result['module'], result['type'], result['client'], result['hostname'], result['user'], result['cleartext'], result['hash'], result['fullhash'])) + cursor.commit() + + if count and not settings.Config.Verbose and not len(result['cleartext']): + print(color('[*] Skipping previously captured hash for %s' % result['user'], 3, 1)) + text('[*] Skipping previously captured hash for %s' % result['user']) + cursor.execute("UPDATE responder SET timestamp=datetime('now') WHERE user=? AND client=?", (result['user'], result['client'])) + cursor.commit() + cursor.close() + return count + def SaveToDb(result): for k in [ 'module', 'type', 'client', 'hostname', 'user', 'cleartext', 'hash', 'fullhash' ]: @@ -335,23 +407,16 @@ def SaveToDb(result): text("[*] Skipping one character username: %s" % result['user']) return - cursor = sqlite3.connect(settings.Config.DatabaseFile) - cursor.text_factory = sqlite3.Binary # We add a text factory to support different charsets - + if settings.Config.Dbms.lower() == 'psql': + count = _SaveToDbPsql(result) + else: + count = _SaveToDbSqlite(result) + if len(result['cleartext']): fname = '%s-%s-ClearText-%s.txt' % (result['module'], result['type'], result['client']) - res = cursor.execute("SELECT COUNT(*) AS count FROM responder WHERE module=? AND type=? AND client=? AND LOWER(user)=LOWER(?) AND cleartext=?", (result['module'], result['type'], result['client'], result['user'], result['cleartext'])) else: fname = '%s-%s-%s.txt' % (result['module'], result['type'], result['client']) - res = cursor.execute("SELECT COUNT(*) AS count FROM responder WHERE module=? AND type=? AND client=? AND LOWER(user)=LOWER(?)", (result['module'], result['type'], result['client'], result['user'])) - - (count,) = res.fetchone() logfile = os.path.join(settings.Config.ResponderPATH, 'logs', fname) - - if not count: - cursor.execute("INSERT INTO responder VALUES(datetime('now'), ?, ?, ?, ?, ?, ?, ?, ?)", (result['module'], result['type'], result['client'], result['hostname'], result['user'], result['cleartext'], result['hash'], result['fullhash'])) - cursor.commit() - if not count or settings.Config.CaptureMultipleHashFromSameHost: with open(logfile,"a") as outf: if len(result['cleartext']): # If we obtained cleartext credentials, write them to file @@ -387,11 +452,30 @@ def SaveToDb(result): elif len(result['cleartext']): print(color('[*] Skipping previously captured cleartext password for %s' % result['user'], 3, 1)) text('[*] Skipping previously captured cleartext password for %s' % result['user']) - else: - print(color('[*] Skipping previously captured hash for %s' % result['user'], 3, 1)) - text('[*] Skipping previously captured hash for %s' % result['user']) - cursor.execute("UPDATE responder SET timestamp=datetime('now') WHERE user=? AND client=?", (result['user'], result['client'])) + +def _SavePoisonersToDbPsql(result): + conn = PsqlConnect() + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) AS count FROM Poisoned WHERE Poisoner=%s AND SentToIp=%s AND ForName=%s AND AnalyzeMode=%s", (result['Poisoner'], result['SentToIp'], result['ForName'], result['AnalyzeMode'])) + (count,) = cursor.fetchone() + + if not count: + cursor.execute("INSERT INTO Poisoned VALUES(NOW(), %s, %s, %s, %s)", (result['Poisoner'], result['SentToIp'], result['ForName'], result['AnalyzeMode'])) + conn.commit() + + cursor.close() + +def _SavePoisonersToDbSqlite(result): + cursor = sqlite3.connect(settings.Config.DatabaseFile) + cursor.text_factory = sqlite3.Binary # We add a text factory to support different charsets + res = cursor.execute("SELECT COUNT(*) AS count FROM Poisoned WHERE Poisoner=? AND SentToIp=? AND ForName=? AND AnalyzeMode=?", (result['Poisoner'], result['SentToIp'], result['ForName'], result['AnalyzeMode'])) + (count,) = res.fetchone() + + if not count: + cursor.execute("INSERT INTO Poisoned VALUES(datetime('now'), ?, ?, ?, ?)", (result['Poisoner'], result['SentToIp'], result['ForName'], result['AnalyzeMode'])) cursor.commit() + cursor.close() def SavePoisonersToDb(result): @@ -400,13 +484,32 @@ def SavePoisonersToDb(result): if not k in result: result[k] = '' result['SentToIp'] = result['SentToIp'].replace("::ffff:","") + + if settings.Config.Dbms.lower() == 'psql': + _SavePoisonersToDbPsql(result) + else: + _SavePoisonersToDbSqlite(result) + +def _SaveDHCPToDbPsql(result): + conn = PsqlConnect() + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) AS count FROM DHCP WHERE MAC=%s AND IP=%s AND RequestedIP=%s", (result['MAC'], result['IP'], result['RequestedIP'])) + (count,) = cursor.fetchone() + + if not count: + cursor.execute("INSERT INTO DHCP VALUES(NOW(), %s, %s, %s)", (result['MAC'], result['IP'], result['RequestedIP'])) + conn.commit() + + cursor.close() + +def _SaveDHCPToDbSqlite(result): cursor = sqlite3.connect(settings.Config.DatabaseFile) cursor.text_factory = sqlite3.Binary # We add a text factory to support different charsets - res = cursor.execute("SELECT COUNT(*) AS count FROM Poisoned WHERE Poisoner=? AND SentToIp=? AND ForName=? AND AnalyzeMode=?", (result['Poisoner'], result['SentToIp'], result['ForName'], result['AnalyzeMode'])) + res = cursor.execute("SELECT COUNT(*) AS count FROM DHCP WHERE MAC=? AND IP=? AND RequestedIP=?", (result['MAC'], result['IP'], result['RequestedIP'])) (count,) = res.fetchone() - + if not count: - cursor.execute("INSERT INTO Poisoned VALUES(datetime('now'), ?, ?, ?, ?)", (result['Poisoner'], result['SentToIp'], result['ForName'], result['AnalyzeMode'])) + cursor.execute("INSERT INTO DHCP VALUES(datetime('now'), ?, ?, ?)", (result['MAC'], result['IP'], result['RequestedIP'])) cursor.commit() cursor.close() @@ -416,17 +519,11 @@ def SaveDHCPToDb(result): if not k in result: result[k] = '' - cursor = sqlite3.connect(settings.Config.DatabaseFile) - cursor.text_factory = sqlite3.Binary # We add a text factory to support different charsets - res = cursor.execute("SELECT COUNT(*) AS count FROM DHCP WHERE MAC=? AND IP=? AND RequestedIP=?", (result['MAC'], result['IP'], result['RequestedIP'])) - (count,) = res.fetchone() - - if not count: - cursor.execute("INSERT INTO DHCP VALUES(datetime('now'), ?, ?, ?)", (result['MAC'], result['IP'], result['RequestedIP'])) - cursor.commit() + if settings.Config.Dbms.lower() == 'psql': + _SaveDHCPToDbPsql(result) + else: + _SaveDHCPToDbSqlite(result) - cursor.close() - def Parse_IPV6_Addr(data): if data[len(data)-4:len(data)] == b'\x00\x1c\x00\x01': return 'IPv6' From a18eb0768e58913541eab85075a77377b9213cb4 Mon Sep 17 00:00:00 2001 From: trietend <212042+trietend@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:25:24 +0200 Subject: [PATCH 3/6] Added requirements for psql --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7823774..28c752d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ netifaces>=0.10.4 +psycopg>=3.2.3 From d186395f27c8107525d5a38f54728bc25a250a8b Mon Sep 17 00:00:00 2001 From: trietend <212042+trietend@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:17:04 +0200 Subject: [PATCH 4/6] improve psql table --- utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils.py b/utils.py index 1341dd2..e2e50ab 100644 --- a/utils.py +++ b/utils.py @@ -325,11 +325,11 @@ def PsqlConnect(): def _CreateResponderDbPsql(): conn = PsqlConnect() cursor = conn.cursor() - cursor.execute('CREATE TABLE IF NOT EXISTS Poisoned (timestamp TEXT, Poisoner TEXT, SentToIp TEXT, ForName TEXT, AnalyzeMode TEXT)') + cursor.execute('CREATE TABLE IF NOT EXISTS Poisoned (timestamp TIMESTAMP, Poisoner TEXT, SentToIp TEXT, ForName TEXT, AnalyzeMode TEXT)') conn.commit() - cursor.execute('CREATE TABLE IF NOT EXISTS responder (timestamp TEXT, module TEXT, type TEXT, client TEXT, hostname TEXT, username TEXT, cleartext TEXT, hash TEXT, fullhash TEXT)') + cursor.execute('CREATE TABLE IF NOT EXISTS responder (timestamp TIMESTAMP, module TEXT, type TEXT, client TEXT, hostname TEXT, username TEXT, cleartext TEXT, hash TEXT, fullhash TEXT)') conn.commit() - cursor.execute('CREATE TABLE IF NOT EXISTS DHCP (timestamp TEXT, MAC TEXT, IP TEXT, RequestedIP TEXT)') + cursor.execute('CREATE TABLE IF NOT EXISTS DHCP (timestamp TIMESTAMP, MAC TEXT, IP TEXT, RequestedIP TEXT)') conn.commit() conn.close() From bde0ba95d6a77b5fee66393c642ce00f5edc6093 Mon Sep 17 00:00:00 2001 From: trietend <212042+trietend@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:27:29 +0200 Subject: [PATCH 5/6] further psql table improvements --- utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils.py b/utils.py index e2e50ab..e861b7e 100644 --- a/utils.py +++ b/utils.py @@ -325,11 +325,11 @@ def PsqlConnect(): def _CreateResponderDbPsql(): conn = PsqlConnect() cursor = conn.cursor() - cursor.execute('CREATE TABLE IF NOT EXISTS Poisoned (timestamp TIMESTAMP, Poisoner TEXT, SentToIp TEXT, ForName TEXT, AnalyzeMode TEXT)') + cursor.execute('CREATE TABLE IF NOT EXISTS Poisoned (timestamp TIMESTAMP, Poisoner TEXT, SentToIp CIDR, ForName TEXT, AnalyzeMode TEXT)') conn.commit() - cursor.execute('CREATE TABLE IF NOT EXISTS responder (timestamp TIMESTAMP, module TEXT, type TEXT, client TEXT, hostname TEXT, username TEXT, cleartext TEXT, hash TEXT, fullhash TEXT)') + cursor.execute('CREATE TABLE IF NOT EXISTS responder (timestamp TIMESTAMP, module TEXT, type TEXT, client CIDR, hostname TEXT, username TEXT, cleartext TEXT, hash TEXT, fullhash TEXT)') conn.commit() - cursor.execute('CREATE TABLE IF NOT EXISTS DHCP (timestamp TIMESTAMP, MAC TEXT, IP TEXT, RequestedIP TEXT)') + cursor.execute('CREATE TABLE IF NOT EXISTS DHCP (timestamp TIMESTAMP, MAC MACADDR8, IP CIDR, RequestedIP CIDR)') conn.commit() conn.close() From 85bafe1670f23939290bae89113507cfea280925 Mon Sep 17 00:00:00 2001 From: trietend <212042+trietend@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:36:29 +0100 Subject: [PATCH 6/6] fixed encoding issues --- utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils.py b/utils.py index e861b7e..61de23b 100644 --- a/utils.py +++ b/utils.py @@ -401,6 +401,11 @@ def SaveToDb(result): for k in [ 'module', 'type', 'client', 'hostname', 'user', 'cleartext', 'hash', 'fullhash' ]: if not k in result: result[k] = '' + if isinstance(result[k], str) and '\x00' in result[k]: + # some strings have a mixed encoding in a single string + # correct decoding is not possible + result[k] = result[k].replace("\x00","") + result['client'] = result['client'].replace("::ffff:","") if len(result['user']) < 2: print(color('[*] Skipping one character username: %s' % result['user'], 3, 1))