From dd780f8d806afae29db5c7173d3e3a9eb1bcfb2e Mon Sep 17 00:00:00 2001 From: sandakersmann Date: Thu, 7 Dec 2017 00:27:59 +0100 Subject: [PATCH 01/18] This type of data is called metadata --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a161d04b2..0ea6144b8a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Bitmessage is a P2P communications protocol used to send encrypted messages to another person or to many subscribers. It is decentralized and trustless, meaning that you need-not inherently trust any entities like root certificate authorities. It uses strong authentication, which means that the sender of a -message cannot be spoofed, and it aims to hide "non-content" data, like the +message cannot be spoofed, and it aims to hide metadata, like the sender and receiver of messages, from passive eavesdroppers like those running warrantless wiretapping programs. From 395812c0f8b50bcc56b411d4c293b7735382c32f Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Wed, 20 Dec 2017 09:20:24 +0100 Subject: [PATCH 02/18] Systemd config file - tested on Debian 9, you may have to adjust paths/uids if your deployment differs --- packages/systemd/bitmessage.service | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 packages/systemd/bitmessage.service diff --git a/packages/systemd/bitmessage.service b/packages/systemd/bitmessage.service new file mode 100644 index 0000000000..1a9f7f4770 --- /dev/null +++ b/packages/systemd/bitmessage.service @@ -0,0 +1,18 @@ +[Unit] +Description=Bitmessage Daemon +After=network.target auditd.service + +[Service] +ExecStart=/usr/bin/python2 /usr/src/PyBitmessage/src/bitmessagemain.py +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure +Type=forking +PIDFile=/var/lib/bitmessage/.config/PyBitmessage/singleton.lock +User=bitmessage +Group=nogroup +WorkingDirectory=/var/lib/bitmessage +Environment="HOME=/var/lib/bitmessage" + +[Install] +WantedBy=multi-user.target From 6fb5a751c624efce8bc41736d219b0acda7cebaa Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Wed, 20 Dec 2017 09:41:36 +0100 Subject: [PATCH 03/18] Add collectd monitoring script --- packages/collectd/pybitmessagestatus.py | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 packages/collectd/pybitmessagestatus.py diff --git a/packages/collectd/pybitmessagestatus.py b/packages/collectd/pybitmessagestatus.py new file mode 100644 index 0000000000..1db9f5b13d --- /dev/null +++ b/packages/collectd/pybitmessagestatus.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python2.7 + +import collectd +import json +import xmlrpclib + +pybmurl = "" +api = "" + +def init_callback(): + global api + api = xmlrpclib.ServerProxy(pybmurl) + collectd.info('pybitmessagestatus.py init done') + +def config_callback(ObjConfiguration): + global pybmurl + apiUsername = "" + apiPassword = "" + apiInterface = "127.0.0.1" + apiPort = 8445 + for node in ObjConfiguration.children: + key = node.key.lower() + if key.lower() == "apiusername" and node.values: + apiUsername = node.values[0] + elif key.lower() == "apipassword" and node.values: + apiPassword = node.values[0] + elif key.lower() == "apiinterface" and node.values: + apiInterface = node.values[0] + elif key.lower() == "apiport" and node.values: + apiPort = node.values[0] + pybmurl = "http://" + apiUsername + ":" + apiPassword + "@" + apiInterface+ ":" + str(int(apiPort)) + "/" + collectd.info('pybitmessagestatus.py config done') + +def read_callback(): + try: + clientStatus = json.loads(api.clientStatus()) + except: + collectd.info("Exception loading or parsing JSON") + return + + for i in ["networkConnections", "numberOfPubkeysProcessed", "numberOfMessagesProcessed", "numberOfBroadcastsProcessed"]: + metric = collectd.Values() + metric.plugin = "pybitmessagestatus" + if i[0:6] == "number": + metric.type = 'counter' + else: + metric.type = 'gauge' + metric.type_instance = i.lower() + try: + metric.values = [clientStatus[i]] + except: + collectd.info("Value for %s missing" % (i)) + metric.dispatch() + +if __name__ == "__main__": + main() +else: + collectd.register_init(init_callback) + collectd.register_config(config_callback) + collectd.register_read(read_callback) From 3cb9547389e41bbf8b7cd8e258e91db57614e4b6 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Thu, 21 Dec 2017 14:26:51 +0100 Subject: [PATCH 04/18] Only write PID after last fork - should fix systemd integration --- src/bitmessagemain.py | 2 +- src/singleinstance.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 83a419198f..91032fe578 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -363,7 +363,7 @@ def daemonize(self): # fork not implemented pass else: - shared.thisapp.lock() # relock + shared.thisapp.lock(True) # relock and write pid shared.thisapp.lockPid = None # indicate we're the final child sys.stdout.flush() sys.stderr.flush() diff --git a/src/singleinstance.py b/src/singleinstance.py index 7a02594514..883c83ccff 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -36,7 +36,7 @@ def __init__(self, flavor_id="", daemon=False): self.initialized = True atexit.register(self.cleanup) - def lock(self): + def lock(self, writePid = False): if self.lockPid is None: self.lockPid = os.getpid() if sys.platform == 'win32': @@ -68,9 +68,10 @@ def lock(self): sys.exit(-1) else: pidLine = "%i\n" % self.lockPid - self.fp.truncate(0) - self.fp.write(pidLine) - self.fp.flush() + if writePid: + self.fp.truncate(0) + self.fp.write(pidLine) + self.fp.flush() def cleanup(self): if not self.initialized: From 02490e3286ad1d865bdc9ec4552766a0304f6b37 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Fri, 29 Dec 2017 08:41:15 +0100 Subject: [PATCH 05/18] Don't break if over 50k messages - typo if there were over 50k messages in inventory caused PyBM to stall --- src/network/tcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network/tcp.py b/src/network/tcp.py index 70e22e0887..922aa79d13 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -180,7 +180,7 @@ def sendChunk(): payload += hash objectCount += 1 if objectCount >= BMProto.maxObjectCount: - self.sendChunk() + sendChunk() payload = b'' objectCount = 0 From e9b1aa48a91624db922d832a1d342e63ad896b4e Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Fri, 29 Dec 2017 08:49:08 +0100 Subject: [PATCH 06/18] Protocol error handler fixes - was broken if there was no error message in "raise" - added default texts for network exceptions --- src/network/bmobject.py | 17 +++++++++++------ src/network/bmproto.py | 9 ++++++--- src/network/proxy.py | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 249ec2ab33..267cac5873 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -8,23 +8,28 @@ import protocol import state -class BMObjectInsufficientPOWError(Exception): pass +class BMObjectInsufficientPOWError(Exception): + errorCodes = ("Insufficient proof of work") -class BMObjectInvalidDataError(Exception): pass +class BMObjectInvalidDataError(Exception): + errorCodes = ("Data invalid") -class BMObjectExpiredError(Exception): pass +class BMObjectExpiredError(Exception): + errorCodes = ("Object expired") -class BMObjectUnwantedStreamError(Exception): pass +class BMObjectUnwantedStreamError(Exception): + errorCodes = ("Object in unwanted stream") -class BMObjectInvalidError(Exception): pass +class BMObjectInvalidError(Exception): + errorCodes = ("Invalid object") class BMObjectAlreadyHaveError(Exception): - pass + errorCodes = ("Already have this object") class BMObject(object): diff --git a/src/network/bmproto.py b/src/network/bmproto.py index 17b2c76144..47c6c85857 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -24,13 +24,16 @@ import state import protocol -class BMProtoError(ProxyError): pass +class BMProtoError(ProxyError): + errorCodes = ("Protocol error") -class BMProtoInsufficientDataError(BMProtoError): pass +class BMProtoInsufficientDataError(BMProtoError): + errorCodes = ("Insufficient data") -class BMProtoExcessiveDataError(BMProtoError): pass +class BMProtoExcessiveDataError(BMProtoError): + errorCodes = ("Too much data") class BMProto(AdvancedDispatcher, ObjectTracker): diff --git a/src/network/proxy.py b/src/network/proxy.py index 7d46cd86fa..96930c189a 100644 --- a/src/network/proxy.py +++ b/src/network/proxy.py @@ -10,7 +10,7 @@ class ProxyError(Exception): errorCodes = ("UnknownError") - def __init__(self, code): + def __init__(self, code=-1): self.code = code try: self.message = self.__class__.errorCodes[self.code] From bcc5a210a484dd76b7e9ad3b033f47e704026770 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Fri, 29 Dec 2017 09:13:41 +0100 Subject: [PATCH 07/18] Fix PID file if not daemonized --- src/singleinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singleinstance.py b/src/singleinstance.py index 883c83ccff..fdb5ee9881 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -31,7 +31,7 @@ def __init__(self, flavor_id="", daemon=False): import bitmessageqt bitmessageqt.init() - self.lock() + self.lock(not daemon) self.initialized = True atexit.register(self.cleanup) From 1864762a0a3af4f32ba7fc5994bcb51d76643da3 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 12:49:08 +0100 Subject: [PATCH 08/18] Apply bandwidth limits without restart - also minor style fixes --- src/bitmessageqt/__init__.py | 10 +++++----- src/network/asyncore_pollchoose.py | 4 ++-- src/network/connectionpool.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index c731b469d5..e6ff0cd0d2 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -69,7 +69,7 @@ import shutdown import state from statusbar import BMStatusBar -import throttle +from network.asyncore_pollchoose import set_rates from version import softwareVersion import sound @@ -2288,16 +2288,16 @@ def click_actionSettings(self): int(float(self.settingsDialogInstance.ui.lineEditMaxDownloadRate.text())))) BMConfigParser().set('bitmessagesettings', 'maxuploadrate', str( int(float(self.settingsDialogInstance.ui.lineEditMaxUploadRate.text())))) - except: + except ValueError: QMessageBox.about(self, _translate("MainWindow", "Number needed"), _translate( "MainWindow", "Your maximum download and upload rate must be numbers. Ignoring what you typed.")) + else: + set_rates(BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate"), + BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate")) BMConfigParser().set('bitmessagesettings', 'maxoutboundconnections', str( int(float(self.settingsDialogInstance.ui.lineEditMaxOutboundConnections.text())))) - throttle.SendThrottle().resetLimit() - throttle.ReceiveThrottle().resetLimit() - BMConfigParser().set('bitmessagesettings', 'namecoinrpctype', self.settingsDialogInstance.getNamecoinType()) BMConfigParser().set('bitmessagesettings', 'namecoinrpchost', str( diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index caa9d650e8..c5586a91f7 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -129,8 +129,8 @@ def write(obj): def set_rates(download, upload): global maxDownloadRate, maxUploadRate, downloadBucket, uploadBucket, downloadTimestamp, uploadTimestamp - maxDownloadRate = float(download) - maxUploadRate = float(upload) + maxDownloadRate = float(download) * 1024 + maxUploadRate = float(upload) * 1024 downloadBucket = maxDownloadRate uploadBucket = maxUploadRate downloadTimestamp = time.time() diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 2f937a1558..3b817e6574 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -22,8 +22,8 @@ class BMConnectionPool(object): def __init__(self): asyncore.set_rates( - BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate") * 1024, - BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate") * 1024) + BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate"), + BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate")) self.outboundConnections = {} self.inboundConnections = {} self.listeningSockets = {} From 36cc5b9cf5e122b736854079b78e6b0dfb903bbc Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 12:51:35 +0100 Subject: [PATCH 09/18] Download optimisations - don't make empty download requests - use smaller chunks when they can be spread across multiple connections --- src/network/downloadthread.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/network/downloadthread.py b/src/network/downloadthread.py index 5fe1ee254f..35616f1bfb 100644 --- a/src/network/downloadthread.py +++ b/src/network/downloadthread.py @@ -43,7 +43,7 @@ def run(self): connections = BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values() random.shuffle(connections) try: - requestChunk = max(int(DownloadThread.maxRequestChunk / len(connections)), 1) + requestChunk = max(int(min(DownloadThread.maxRequestChunk, len(missingObjects)) / len(connections)), 1) except ZeroDivisionError: requestChunk = 1 for i in connections: @@ -63,15 +63,14 @@ def run(self): except KeyError: continue random.shuffle(request) + if len(request) > requestChunk - downloadPending: + request = request[:max(1, requestChunk - downloadPending)] if not request: continue - if len(request) > requestChunk - downloadPending: - request = request[:requestChunk - downloadPending] # mark them as pending for k in request: i.objectsNewToMe[k] = False missingObjects[k] = now - payload = bytearray() payload.extend(addresses.encodeVarint(len(request))) for chunk in request: From baba0ae206b6cc8f6d81725fb1184a5d70225ed9 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 13:04:58 +0100 Subject: [PATCH 10/18] Remove obsolete code --- src/shutdown.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/shutdown.py b/src/shutdown.py index 278759e571..49c2fb9b55 100644 --- a/src/shutdown.py +++ b/src/shutdown.py @@ -9,14 +9,12 @@ from helper_threading import StoppableThread from knownnodes import saveKnownNodes from inventory import Inventory -import protocol from queues import addressGeneratorQueue, objectProcessorQueue, UISignalQueue, workerQueue import shared import state def doCleanShutdown(): state.shutdown = 1 #Used to tell proof of work worker threads and the objectProcessorThread to exit. - protocol.broadcastToSendDataQueues((0, 'shutdown', 'no data')) objectProcessorQueue.put(('checkShutdownVariable', 'no data')) for thread in threading.enumerate(): if thread.isAlive() and isinstance(thread, StoppableThread): From d9a42630830eef597021179924351bba3b959386 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 13:08:12 +0100 Subject: [PATCH 11/18] Daemonising fixes - change forking exit order as systemd expects (wait until child is ready, then exit parent, then grandparent) - fix signal handler if prctl not installed - revert recent PID file changes --- src/bitmessagemain.py | 19 ++++++++++++++++++- src/helper_generic.py | 2 +- src/singleinstance.py | 18 +++++++++++++----- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 91032fe578..6f79693928 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -342,13 +342,21 @@ def start(self): sleep(1) def daemonize(self): + grandfatherPid = os.getpid() + parentPid = None try: if os.fork(): + # unlock + shared.thisapp.cleanup() + # wait until grandchild ready + while True: + sleep(1) os._exit(0) except AttributeError: # fork not implemented pass else: + parentPid = os.getpid() shared.thisapp.lock() # relock os.umask(0) try: @@ -358,12 +366,17 @@ def daemonize(self): pass try: if os.fork(): + # unlock + shared.thisapp.cleanup() + # wait until child ready + while True: + sleep(1) os._exit(0) except AttributeError: # fork not implemented pass else: - shared.thisapp.lock(True) # relock and write pid + shared.thisapp.lock() # relock shared.thisapp.lockPid = None # indicate we're the final child sys.stdout.flush() sys.stderr.flush() @@ -374,6 +387,10 @@ def daemonize(self): os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) + if parentPid: + # signal ready + os.kill(parentPid, signal.SIGTERM) + os.kill(grandfatherPid, signal.SIGTERM) def setSignalHandler(self): signal.signal(signal.SIGINT, helper_generic.signal_handler) diff --git a/src/helper_generic.py b/src/helper_generic.py index b750e51915..4f7a12996c 100644 --- a/src/helper_generic.py +++ b/src/helper_generic.py @@ -51,7 +51,7 @@ def signal_handler(signal, frame): raise SystemExit if "PoolWorker" in current_process().name: raise SystemExit - if current_thread().name != "PyBitmessage": + if current_thread().name not in ("PyBitmessage", "MainThread"): return logger.error("Got signal %i", signal) if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'): diff --git a/src/singleinstance.py b/src/singleinstance.py index fdb5ee9881..d7cc0ab302 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -36,7 +36,7 @@ def __init__(self, flavor_id="", daemon=False): self.initialized = True atexit.register(self.cleanup) - def lock(self, writePid = False): + def lock(self): if self.lockPid is None: self.lockPid = os.getpid() if sys.platform == 'win32': @@ -68,16 +68,24 @@ def lock(self, writePid = False): sys.exit(-1) else: pidLine = "%i\n" % self.lockPid - if writePid: - self.fp.truncate(0) - self.fp.write(pidLine) - self.fp.flush() + self.fp.truncate(0) + self.fp.write(pidLine) + self.fp.flush() def cleanup(self): if not self.initialized: return if self.daemon and self.lockPid == os.getpid(): # these are the two initial forks while daemonizing + try: + if sys.platform == 'win32': + if hasattr(self, 'fd'): + os.close(self.fd) + else: + fcntl.lockf(self.fp, fcntl.LOCK_UN) + except Exception, e: + pass + return print "Cleaning up lockfile" try: From 6b54a4ab0e1ddfc39d06dca2d5aea010be7e8a5a Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 13:10:19 +0100 Subject: [PATCH 12/18] Daemonize fix - forgot to revert a line in previous commit --- src/singleinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singleinstance.py b/src/singleinstance.py index d7cc0ab302..ed1048bae8 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -31,7 +31,7 @@ def __init__(self, flavor_id="", daemon=False): import bitmessageqt bitmessageqt.init() - self.lock(not daemon) + self.lock() self.initialized = True atexit.register(self.cleanup) From bb5f1d6f98b3dabec7a432bcde399bbec12c0b8a Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 10:29:21 +0100 Subject: [PATCH 13/18] Setup.py typo - surprisingly, it only was broken on some systems, e.g. Debian 8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0851b78f20..ba34f6df5a 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def run(self): 'qrcode': ['qrcode'], 'pyopencl': ['pyopencl'], 'notify2': ['notify2'], - 'sound:platform_system=="Windows"': ['winsound'] + 'sound;platform_system=="Windows"': ['winsound'] }, classifiers=[ "License :: OSI Approved :: MIT License" From 9b58f35b80e1ece4f6d8fdefcc8ec13fce687b08 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 13:56:03 +0100 Subject: [PATCH 14/18] App name in version --- src/version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/version.py b/src/version.py index bd06ca9b44..694f71e419 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1,2 @@ +softwareName = 'PyBitmessage' softwareVersion = '0.6.2' From 8788f2d3497e061d14e9ae4bd0864c6ea2373b39 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 14:29:21 +0100 Subject: [PATCH 15/18] Server full and duplicate handling - will try to report "Server full" over protocol for 10 extra connections over limit, instead of simply dropping them - if connected to the same host inbound and outbound, handle as server full (prevents duplicate connections) --- src/network/bmproto.py | 14 ++++++++++++++ src/network/tcp.py | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/network/bmproto.py b/src/network/bmproto.py index 47c6c85857..21ec692cfc 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -497,6 +497,20 @@ def peerValidityChecks(self): return False except: pass + if not self.isOutbound: + # incoming from a peer we're connected to as outbound, or server full + # report the same error to counter deanonymisation + if state.Peer(self.destination.host, self.peerNode.port) in \ + network.connectionpool.BMConnectionPool().inboundConnections or \ + len(network.connectionpool.BMConnectionPool().inboundConnections) + \ + len(network.connectionpool.BMConnectionPool().outboundConnections) > \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): + self.append_write_buf(protocol.assembleErrorMessage(fatal=2, + errorText="Server full, please try again later.")) + logger.debug ("Closed connection to %s due to server full or duplicate inbound/outbound.", + str(self.destination)) + return False if network.connectionpool.BMConnectionPool().isAlreadyConnected(self.nonce): self.append_write_buf(protocol.assembleErrorMessage(fatal=2, errorText="I'm connected to myself. Closing connection.")) diff --git a/src/network/tcp.py b/src/network/tcp.py index 922aa79d13..5a27aca365 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -292,7 +292,10 @@ def handle_accept(self): if len(network.connectionpool.BMConnectionPool().inboundConnections) + \ len(network.connectionpool.BMConnectionPool().outboundConnections) > \ BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ - BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): + BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections") + 10: + # 10 is a sort of buffer, in between it will go through the version handshake + # and return an error to the peer + logger.warning("Server full, dropping connection") sock.close() return try: From 4086253730e2551a9b3aa3df5af13ff9be322bee Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 15:24:47 +0100 Subject: [PATCH 16/18] Bandwidth limit optimisation - should be slightly more accurate and use slightly fewer resources --- src/network/advanceddispatcher.py | 4 ++-- src/network/asyncore_pollchoose.py | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/network/advanceddispatcher.py b/src/network/advanceddispatcher.py index c50b6a43ad..3e84ed8522 100644 --- a/src/network/advanceddispatcher.py +++ b/src/network/advanceddispatcher.py @@ -75,7 +75,7 @@ def set_state(self, state, length=0, expectBytes=0): def writable(self): self.uploadChunk = AdvancedDispatcher._buf_len if asyncore.maxUploadRate > 0: - self.uploadChunk = asyncore.uploadBucket + self.uploadChunk = int(asyncore.uploadBucket) self.uploadChunk = min(self.uploadChunk, len(self.write_buf)) return asyncore.dispatcher.writable(self) and \ (self.connecting or (self.connected and self.uploadChunk > 0)) @@ -83,7 +83,7 @@ def writable(self): def readable(self): self.downloadChunk = AdvancedDispatcher._buf_len if asyncore.maxDownloadRate > 0: - self.downloadChunk = asyncore.downloadBucket + self.downloadChunk = int(asyncore.downloadBucket) try: if self.expectBytes > 0 and not self.fullyEstablished: self.downloadChunk = min(self.downloadChunk, self.expectBytes - len(self.read_buf)) diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index c5586a91f7..5717ff782d 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -112,6 +112,8 @@ class ExitNow(Exception): sentBytes = 0 def read(obj): + if not can_receive(): + return try: obj.handle_read_event() except _reraised_exceptions: @@ -120,6 +122,8 @@ def read(obj): obj.handle_error() def write(obj): + if not can_send(): + return try: obj.handle_write_event() except _reraised_exceptions: @@ -136,12 +140,18 @@ def set_rates(download, upload): downloadTimestamp = time.time() uploadTimestamp = time.time() +def can_receive(): + return maxDownloadRate == 0 or downloadBucket > 0 + +def can_send(): + return maxUploadRate == 0 or uploadBucket > 0 + def update_received(download=0): global receivedBytes, downloadBucket, downloadTimestamp currentTimestamp = time.time() receivedBytes += download if maxDownloadRate > 0: - bucketIncrease = int(maxDownloadRate * (currentTimestamp - downloadTimestamp)) + bucketIncrease = maxDownloadRate * (currentTimestamp - downloadTimestamp) downloadBucket += bucketIncrease if downloadBucket > maxDownloadRate: downloadBucket = int(maxDownloadRate) @@ -153,7 +163,7 @@ def update_sent(upload=0): currentTimestamp = time.time() sentBytes += upload if maxUploadRate > 0: - bucketIncrease = int(maxUploadRate * (currentTimestamp - uploadTimestamp)) + bucketIncrease = maxUploadRate * (currentTimestamp - uploadTimestamp) uploadBucket += bucketIncrease if uploadBucket > maxUploadRate: uploadBucket = int(maxUploadRate) @@ -170,9 +180,9 @@ def _exception(obj): def readwrite(obj, flags): try: - if flags & select.POLLIN: + if flags & select.POLLIN and can_receive(): obj.handle_read_event() - if flags & select.POLLOUT: + if flags & select.POLLOUT and can_send(): obj.handle_write_event() if flags & select.POLLPRI: obj.handle_expt_event() From f74f82e54f97a9075e95ae59474e4948ce1d6e82 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 22:20:33 +0100 Subject: [PATCH 17/18] Start downloading earlier --- src/network/objectracker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/network/objectracker.py b/src/network/objectracker.py index f846e7d5d7..62f01e4f1c 100644 --- a/src/network/objectracker.py +++ b/src/network/objectracker.py @@ -29,6 +29,7 @@ class ObjectTracker(object): invInitialCapacity = 50000 invErrorRate = 0.03 trackingExpires = 3600 + initialTimeOffset = 60 def __init__(self): self.objectsNewToMe = {} @@ -87,7 +88,7 @@ def handleReceivedInventory(self, hashId): if hashId in Dandelion().hashMap: Dandelion().fluffTrigger(hashId) if hashId not in missingObjects: - missingObjects[hashId] = time.time() + missingObjects[hashId] = time.time() - ObjectTracker.initialTimeOffset with self.objectsNewToMeLock: self.objectsNewToMe[hashId] = True From c9851b9f41da4db4365ef080b1533ab845992bd1 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 22:23:03 +0100 Subject: [PATCH 18/18] Connection lookups invalid data handling - shouldn't throw an exception if argument is a string rather than Peer --- src/network/connectionpool.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 3b817e6574..44534a7668 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -65,12 +65,18 @@ def connectToStream(self, streamNumber): def getConnectionByAddr(self, addr): if addr in self.inboundConnections: return self.inboundConnections[addr] - if addr.host in self.inboundConnections: - return self.inboundConnections[addr.host] + try: + if addr.host in self.inboundConnections: + return self.inboundConnections[addr.host] + except AttributeError: + pass if addr in self.outboundConnections: return self.outboundConnections[addr] - if addr.host in self.udpSockets: - return self.udpSockets[addr.host] + try: + if addr.host in self.udpSockets: + return self.udpSockets[addr.host] + except AttributeError: + pass raise KeyError def isAlreadyConnected(self, nodeid):