Week of 256MB remake

This commit is contained in:
Fierelier 2021-10-28 08:08:33 +02:00
parent b00d217400
commit fb26b7ac72
10 changed files with 437 additions and 484 deletions

3
.gitignore vendored
View File

@ -1 +1,2 @@
/textServer/
/modules/
server.db

View File

@ -1,330 +0,0 @@
#!/usr/bin/env python3
import sys
oldexcepthook = sys.excepthook
def newexcepthook(type,value,traceback):
oldexcepthook(type,value,traceback)
input("Press ENTER to quit.")
sys.excepthook = newexcepthook
import os
p = os.path.join
pUp = os.path.dirname
s = False
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
s = os.path.realpath(sys.executable)
else:
s = os.path.realpath(__file__)
sp = pUp(s)
# script start
import threading
import queue
import socket
import traceback
import time
import colorama
colorama.init()
maxConnections = 10000
maxConnectionsPerIp = 10
maxQueueSize = 1000
maxRequestSize = 4096
pauseBetweenCommands = 0.1
serverAddr = ("127.0.0.1",21779)
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connectionsLock = threading.Lock()
connections = {}
connectionsId = 0
heartbeatTime = 600
threadCount = 0
threadCountLock = threading.Lock()
fileLock = threading.Lock()
commands = {}
def commandlineToList(cmd):
args = []
cArg = ""
escape = False
quoted = False
for letter in cmd:
if escape == True:
cArg += letter
escape = False
continue
if letter == "\\":
escape = True
continue
#if quoted == False and letter == ",":
if letter == ",":
if cArg == "": continue
args.append(cArg)
cArg = ""
continue
#if letter == '"':
# quoted = not quoted
# continue
cArg += letter
args.append(cArg)
return args
def listToCommandline(lst):
cmd = ""
for arg in lst:
arg = arg.replace("\\","\\\\")
arg = arg.replace(",","\\,")
#arg = arg.replace('"','\\"')
#if " " in arg: arg = '"' +arg+ '"'
cmd += arg + ","
return cmd[:-1]
printLock = threading.Lock()
def tprint(st):
with printLock:
print(st)
def runCode(str, lcs = False, description = "loose-code"):
if lcs == False: lcs = {}
code = compile(str,description,"exec")
exec(code,globals(),lcs)
return lcs
def runScript(sf, lcs = False):
if lcs == False: lcs = {}
with open(sf) as script:
runCode(script.read(),lcs,sf)
return lcs
def getModlist(path):
modList = []
for root,dirs,files in os.walk(path):
for file in dirs:
ffile = p(root,file)
lfile = ffile.replace(path + os.path.sep,"",1)
if lfile[0] == "-": continue
if lfile[0] == "[" and lfile[-1] == "]":
modList = modList + sorted(getModlist(ffile))
continue
modList.append(ffile)
break
return modList
modulesLoaded = []
modulePath = p(sp,"modules")
def moduleRun(localModule):
if not localModule in modulesLoaded: modulesLoaded.append(localModule)
print("> " +localModule+ "...")
runScript(p(modulePath,localModule,"module.py"))
def moduleDepends(localModules):
if type(localModules) == str: localModules = [localModules]
for localModule in localModules:
if localModule in modulesLoaded: return
print("depend ",end="")
moduleRun(localModule)
def addThread():
global threadCount
with threadCountLock:
threadCount += 1
tprint(colorama.Fore.YELLOW + colorama.Style.BRIGHT + "Thread opened. Threads: " +str(threadCount)+ " (Actual: " +str(threading.active_count())+ ")" + colorama.Style.RESET_ALL)
def removeThread():
global threadCount
with threadCountLock:
threadCount -= 1
tprint(colorama.Fore.YELLOW + colorama.Style.BRIGHT + "Thread closed. Threads: " +str(threadCount)+ " (Actual: " +str(threading.active_count())+ ")" + colorama.Style.RESET_ALL)
def sendResponse(connection,data):
connection.sendall(len(data).to_bytes(4,"big") + b"\x00" + data)
def getResponse(connection):
data = b''
data = connection.recv(4)
if not data: return False
nul = connection.recv(1)
if not nul: return False
if nul != b"\x00": return False
requestLength = int.from_bytes(data,"big")
if requestLength > maxRequestSize: raise Exception("security","request_too_large")
return connection.recv(requestLength)
def closeConnection(connectionId):
if not connectionId in connections: return False
try:
connections[connectionId]["connection"].close()
except Exception as e:
tprint("Failed to close connection: " +str(e))
pass
try:
connections[connectionId]["threadOut"].queue.put(False)
except:
with printLock:
print(colorama.Fore.GREEN + colorama.Style.BRIGHT)
traceback.print_exc()
print(colorama.Style.RESET_ALL)
del connections[connectionId]
return True
class connectionThreadOut(threading.Thread):
def __init__(self,connectionId):
threading.Thread.__init__(self)
self.queue = queue.Queue()
self.connectionId = connectionId
def getConnection(self):
with connectionsLock:
if self.connectionId in connections:
return connections[self.connectionId]["connection"]
return False
def run(self):
try:
while True:
data = self.queue.get(timeout=heartbeatTime)
if data == False: return
connection = self.getConnection()
if not connection:
with connectionsLock: closeConnection(self.connectionId)
return
sendResponse(connection,data)
except Exception as e:
with connectionsLock: closeConnection(self.connectionId)
with printLock:
print(colorama.Fore.GREEN + colorama.Style.BRIGHT)
traceback.print_exc()
print(colorama.Style.RESET_ALL)
finally:
removeThread()
class connectionThreadIn(threading.Thread):
def __init__(self,connectionId):
threading.Thread.__init__(self)
self.connectionId = connectionId
def getConnection(self):
with connectionsLock:
if self.connectionId in connections:
return connections[self.connectionId]["connection"]
return False
def runCommand(self,cmd):
command = False
if not cmd[0] in commands:
return ["error","nonfatal","command_not_found"]
command = commands[cmd[0]]
rtn = command["function"](self,cmd)
sleep = pauseBetweenCommands
if "sleep" in command:
sleep = command["sleep"]
if sleep > 0: time.sleep(sleep)
return rtn
def run(self):
try:
while True:
connection = self.getConnection()
if not connection:
with connectionsLock: closeConnection(self.connectionId)
return
data = getResponse(connection)
if data == False:
with connectionsLock: closeConnection(self.connectionId)
return
queue = False
with connectionsLock:
queue = connections[self.connectionId]["threadOut"].queue
if queue.qsize() >= maxQueueSize:
closeConnection(self.connectionId)
return
dataString = data.decode(encoding="utf-8")
commandList = commandlineToList(dataString)
queue.put(listToCommandline(self.runCommand(commandList)).encode(encoding="utf-8"))
except Exception as e:
with connectionsLock: closeConnection(self.connectionId)
with printLock:
print(colorama.Fore.GREEN + colorama.Style.BRIGHT)
traceback.print_exc()
print(colorama.Style.RESET_ALL)
finally:
removeThread()
def main():
print("Loading modules...")
for path in getModlist(modulePath):
if os.path.isfile(p(path,"module.py")):
localModule = path.replace(modulePath + os.path.sep,"",1)
if not localModule in modulesLoaded:
moduleRun(localModule)
global connectionsId
serverSocket.bind(serverAddr)
serverSocket.listen(65535)
while True:
connection,address = serverSocket.accept()
connection.settimeout(heartbeatTime)
with connectionsLock:
# Count connections
connectionsCount = 0
connectionsCountIp = 0
for connectionId in connections:
connectionsCount += 1
if connections[connectionId]["address"][0] == address[0]:
connectionsCountIp += 1
if connectionsCount >= maxConnections:
tprint("Connection closed - too many clients.")
closeConnection(connectionId)
continue
if connectionsCountIp >= maxConnectionsPerIp:
tprint("Connection closed - same IP connected too many times.")
closeConnection(connectionId)
continue
# Create connection
connectionsId += 1
threadIn = connectionThreadIn(str(connectionsId))
threadOut = connectionThreadOut(str(connectionsId))
connections[str(connectionsId)] = {
"connection": connection,
"address": address,
"threadOut": threadOut,
"threadIn": threadIn,
"user": False
}
addThread()
addThread()
threadOut.start()
threadIn.start()
if __name__ == '__main__':
main()

93
dumbclient.py Normal file
View File

@ -0,0 +1,93 @@
#!/usr/bin/env python3
import sys
stopOnException = False
oldexcepthook = sys.excepthook
def newexcepthook(type,value,traceback):
oldexcepthook(type,value,traceback)
if stopOnException: input("Press ENTER to quit.")
sys.excepthook = newexcepthook
import os
p = os.path.join
pUp = os.path.dirname
s = False
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
s = os.path.realpath(sys.executable)
else:
s = os.path.realpath(__file__)
sp = pUp(s)
import socket
import threading
import queue
import dumbconsole
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.settimeout(600)
connection.connect(("127.0.0.1",1337))
class receiverThread(threading.Thread):
def __init__(self,connection):
threading.Thread.__init__(self)
self.connection = connection
def run(self):
while True:
response = getResponse(connection).decode("utf-8")
dumbconsole.inputQueue.put(response)
def getResponse(connection):
data = b''
data = connection.recv(4)
if not data: return False
nul = connection.recv(1)
if not nul: return False
if nul != b"\x00": return False
requestLength = int.from_bytes(data,"big")
return connection.recv(requestLength)
def sendResponse(connection,data):
connection.sendall(len(data).to_bytes(4,"big") + b"\x00" + data)
def commandlineToList(cmd):
args = []
cArg = ""
escape = False
quoted = False
for letter in cmd:
if escape == True:
cArg += letter
escape = False
continue
if letter == "\\":
escape = True
continue
if letter == ",":
if cArg == "": continue
args.append(cArg)
cArg = ""
continue
cArg += letter
args.append(cArg)
return args
def listToCommandline(lst):
cmd = ""
for arg in lst:
arg = arg.replace("\\","\\\\")
arg = arg.replace(",","\\,")
cmd += arg + ","
return cmd[:-1]
def dumbSend(text):
sendResponse(connection,text.encode("utf-8"))
thread = receiverThread(connection)
thread.start()
dumbconsole.init(dumbSend)

73
dumbconsole.py Normal file
View File

@ -0,0 +1,73 @@
import qtpy
#import qtpy.QtGui as QtGui
from qtpy.QtGui import *
from qtpy.QtWidgets import *
from qtpy.QtCore import *
#from qtpy.QtMultimedia import QSound
import sys
import queue
inputQueue = queue.Queue()
class consoleWindow(QMainWindow):
def __init__(self,outputFunc,*args,**kwargs):
super().__init__(*args,**kwargs)
self.cOutputFunc = outputFunc
self.cWidth = 640
self.cHeight = 480
self.cCommandEditHeight = 22
self.resize(self.cWidth,self.cHeight)
self.cCreateElements()
self.cTaskTimer = QTimer()
self.cTaskTimer.setInterval(100)
self.cTaskTimer.timeout.connect(self.cRunTasks)
self.cTaskTimer.start()
def cCreateElements(self):
self.cTextBox = QTextBrowser(self)
self.cCommandEdit = QLineEdit("",self)
self.cCommandEdit.returnPressed.connect(self.cSend)
self.cButtonSend = QPushButton("Go",self)
self.cButtonSend.clicked.connect(self.cSend)
self.cResizeElements()
self.show()
self.cCommandEdit.setFocus()
def cResizeElements(self):
self.cTextBox.move(0,0)
self.cTextBox.resize(self.cWidth,self.cHeight - self.cCommandEditHeight)
self.cCommandEdit.move(0,self.cHeight - self.cCommandEditHeight)
self.cCommandEdit.resize(self.cWidth - 50,self.cCommandEditHeight)
self.cButtonSend.move(self.cWidth - 50,self.cHeight - self.cCommandEditHeight)
self.cButtonSend.resize(50,self.cCommandEditHeight)
def resizeEvent(self,event):
self.cWidth = self.width()
self.cHeight = self.height()
self.cResizeElements()
def cSend(self):
text = self.cCommandEdit.text()
self.cCommandEdit.clear()
self.cOutput(">" +text)
self.cOutputFunc(text)
def cOutput(self,text):
self.cTextBox.append(text)
def cRunTasks(self):
try:
while True:
text = inputQueue.get(False)
self.cOutput(text)
except queue.Empty:
return
def init(outputFunc):
global app
global window
app = QApplication(sys.argv)
window = consoleWindow(outputFunc)
app.exec_()

17
modules.txt Normal file
View File

@ -0,0 +1,17 @@
# base modules
servercaps.py
filelock.py
db.py
# base extras
spamprotection.py
# userland base
account.py
messageid.py
history.py
# userland addons
send.py
req.py
nop.py

View File

@ -1,5 +0,0 @@
global commands
commands["nop"] = {}
def f(self,cmd):
return [""]
commands["nop"]["function"] = f

View File

@ -1,10 +0,0 @@
global commands
commands["req"] = {}
def f(self,cmd):
if len(cmd) < 3:
return ["error","nonfatal","syntax","need at least 3 arguments"]
rtn = cmd[:2] + self.runCommand(cmd[2:])
return rtn
commands["req"]["function"] = f
commands["req"]["sleep"] = 0

View File

@ -1,61 +0,0 @@
#!/usr/bin/env python3
import sys
oldexcepthook = sys.excepthook
def newexcepthook(type,value,traceback):
oldexcepthook(type,value,traceback)
input("Press ENTER to quit.")
sys.excepthook = newexcepthook
import os
p = os.path.join
pUp = os.path.dirname
s = False
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
s = os.path.realpath(sys.executable)
else:
s = os.path.realpath(__file__)
sp = pUp(s)
# script start
import socket
def sendRequest(connection,data):
connection.sendall(len(data).to_bytes(4,"big") + data)
def getResponse(connection):
data = b''
data = connection.recv(4)
if not data:
connection.close()
return
requestLength = int.from_bytes(data,"big")
data = connection.recv(requestLength)
return data
def main():
global connection
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect(("127.0.0.1",21779))
while True:
text = input("data: ")
data = text.encode("utf-8")
sendRequest(connection,data)
response = getResponse(connection).decode("utf-8")
print("server: " +response)
if text == "exit":
connection.close()
break
if text == "close":
connection.close()
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect(("127.0.0.1",21779))
connection.close()
break
if __name__ == '__main__':
main()

252
server.py Normal file
View File

@ -0,0 +1,252 @@
#!/usr/bin/env python3
import sys
stopOnException = True
oldexcepthook = sys.excepthook
def newexcepthook(type,value,traceback):
oldexcepthook(type,value,traceback)
if stopOnException: input("Press ENTER to quit.")
sys.excepthook = newexcepthook
import os
p = os.path.join
pUp = os.path.dirname
s = False
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
s = os.path.realpath(sys.executable)
else:
s = os.path.realpath(__file__)
sp = pUp(s)
import socket
import threading
import queue
import traceback
# SETTINGS
serverAddr = ("127.0.0.1",1337)
moduleDir = p(sp,"modules")
heartbeatTime = 300
maxRequestSize = 4096
# SETTINGS END
connections = {}
connectionsLock = threading.Lock()
connectionId = 0
handlers = {}
handlers["modulesLoaded"] = []
handlers["preConnect"] = []
handlers["connect"] = []
handlers["preCommand"] = []
handlers["command"] = []
commands = {}
def runCode(str, lcs = False, description = "loose-code"):
if lcs == False: lcs = {}
code = compile(str,description,"exec")
exec(code,globals(),lcs)
return lcs
def runScript(sf, lcs = False):
if lcs == False: lcs = {}
with open(sf) as script:
runCode(script.read(),lcs,sf)
return lcs
def callHandler(handlerName,env = {},*args,**kwargs):
if handlerName in handlers:
for handlerFunc in handlers[handlerName]:
if handlerFunc(env,*args,**kwargs):
return True
return False
def getResponse(connection):
data = b''
data = connection.recv(4)
if not data: return False
nul = connection.recv(1)
if not nul: return False
if nul != b"\x00": return False
requestLength = int.from_bytes(data,"big")
if requestLength > maxRequestSize: return False
return connection.recv(requestLength)
def sendResponse(connection,data):
connection.sendall(len(data).to_bytes(4,"big") + b"\x00" + data)
def commandlineToList(cmd):
args = []
cArg = ""
escape = False
quoted = False
for letter in cmd:
if escape == True:
cArg += letter
escape = False
continue
if letter == "\\":
escape = True
continue
if letter == ",":
if cArg == "": continue
args.append(cArg)
cArg = ""
continue
cArg += letter
args.append(cArg)
return args
def listToCommandline(lst):
cmd = ""
for arg in lst:
arg = arg.replace("\\","\\\\")
arg = arg.replace(",","\\,")
cmd += arg + ","
return cmd[:-1]
def runCommand(self,command,*args):
callHandler("preCommand",locals())
if not command in commands:
rtn = ["error","nonfatal","invalid_command","Command does not exist"]
callHandler("command",locals())
return rtn
rtn = commands[command](self,command,*args)
if not rtn: rtn = ["OK"]
callHandler("command",locals())
return rtn
class connectionThreadIn(threading.Thread):
def __init__(self,cid,connection,address):
threading.Thread.__init__(self)
self.cid = cid
self.connection = connection
self.address = address
def routine(self):
while True:
data = getResponse(self.connection)
if data == False: return
commandList = commandlineToList(data.decode("utf-8"))
rtn = runCommand(self,*commandList)
with connectionsLock:
connections[self.cid]["threadOut"].queue.put(listToCommandline(rtn))
def run(self):
try:
self.routine()
except:
print(traceback.format_exc())
try:
self.connection.close()
except:
pass
with connectionsLock:
try:
connections[self.cid]["threadOut"].queue.put(False)
except:
pass
try:
del connections[self.cid]
except:
pass
class connectionThreadOut(threading.Thread):
def __init__(self,cid,connection,address):
threading.Thread.__init__(self)
self.cid = cid
self.connection = connection
self.address = address
self.queue = queue.Queue()
def routine(self):
while True:
data = self.queue.get(timeout=heartbeatTime)
if data == False:
return
sendResponse(self.connection,data.encode("utf-8"))
def run(self):
try:
self.routine()
except:
print(traceback.format_exc())
try:
self.connection.close()
except:
pass
try:
with connectionsLock:
del connections[self.cid]
except:
pass
def main():
if os.path.isfile("modules.txt"):
print("Loading modules...")
with open("modules.txt","r") as modulesFile:
for line in modulesFile:
line = line.split("#",1)[0].strip(" \t\r\n")
if line == "": continue
print("> " +line+ " ...")
moduleFile = p(moduleDir,line)
runScript(moduleFile,locals())
print("OK.\n")
callHandler("modulesLoaded",locals())
global connectionId
global serverSocket
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serverSocket.bind(serverAddr)
serverSocket.listen(65535)
print("Serving at " +str(serverAddr[0])+ ":" +str(serverAddr[1])+ ".")
while True:
connection = False
try:
connection,address = serverSocket.accept()
connection.settimeout(heartbeatTime)
if callHandler("preConnect",locals()):
connection.close()
continue
with connectionsLock:
connectionId += 1
threadIn = connectionThreadIn(str(connectionId),connection,address)
threadOut = connectionThreadOut(str(connectionId),connection,address)
connections[str(connectionId)] = {
"connection": connection,
"address": address,
"threadIn": threadIn,
"threadOut": threadOut
}
threadIn.start()
threadOut.start()
callHandler("connect",locals())
except:
print(traceback.format_exc())
try:
connection.close()
except:
pass
if __name__ == '__main__':
main()

View File

@ -1,77 +0,0 @@
#!/usr/bin/env python3
import sys
oldexcepthook = sys.excepthook
def newexcepthook(type,value,traceback):
oldexcepthook(type,value,traceback)
input("Press ENTER to quit.")
sys.excepthook = newexcepthook
import os
p = os.path.join
pUp = os.path.dirname
s = False
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
s = os.path.realpath(sys.executable)
else:
s = os.path.realpath(__file__)
sp = pUp(s)
# script start
import socket
import threading
class receiverThread(threading.Thread):
def __init__(self,connection):
threading.Thread.__init__(self)
self.connection = connection
def run(self):
while True:
response = getResponse(connection).decode("utf-8")
print("server: " +response)
def sendRequest(connection,data):
connection.sendall(len(data).to_bytes(4,"big") + b"\x00" + data)
def getResponse(connection):
data = b''
data = connection.recv(4)
if not data:
connection.close()
return
nul = connection.recv(1)
if not nul:
connection.close()
return
if nul != b"\x00":
connection.close()
return
requestLength = int.from_bytes(data,"big")
data = connection.recv(requestLength)
return data
def main():
global connection
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect((sys.argv[1],int(sys.argv[2])))
thread = receiverThread(connection)
thread.start()
while True:
text = input()
data = text.encode("utf-8")
connection.settimeout(15)
sendRequest(connection,data)
connection.settimeout(None)
if text == "exit":
connection.close()
break
if __name__ == '__main__':
main()