diff --git a/README.md b/README.md index 7a161d04b2..0ea6144b8a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Bitmessage is a P2P communications protocol used to send encrypted messages to another person or to many subscribers. It is decentralized and trustless, meaning that you need-not inherently trust any entities like root certificate authorities. It uses strong authentication, which means that the sender of a -message cannot be spoofed, and it aims to hide "non-content" data, like the +message cannot be spoofed, and it aims to hide metadata, like the sender and receiver of messages, from passive eavesdroppers like those running warrantless wiretapping programs. diff --git a/packages/collectd/pybitmessagestatus.py b/packages/collectd/pybitmessagestatus.py new file mode 100644 index 0000000000..1db9f5b13d --- /dev/null +++ b/packages/collectd/pybitmessagestatus.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python2.7 + +import collectd +import json +import xmlrpclib + +pybmurl = "" +api = "" + +def init_callback(): + global api + api = xmlrpclib.ServerProxy(pybmurl) + collectd.info('pybitmessagestatus.py init done') + +def config_callback(ObjConfiguration): + global pybmurl + apiUsername = "" + apiPassword = "" + apiInterface = "127.0.0.1" + apiPort = 8445 + for node in ObjConfiguration.children: + key = node.key.lower() + if key.lower() == "apiusername" and node.values: + apiUsername = node.values[0] + elif key.lower() == "apipassword" and node.values: + apiPassword = node.values[0] + elif key.lower() == "apiinterface" and node.values: + apiInterface = node.values[0] + elif key.lower() == "apiport" and node.values: + apiPort = node.values[0] + pybmurl = "http://" + apiUsername + ":" + apiPassword + "@" + apiInterface+ ":" + str(int(apiPort)) + "/" + collectd.info('pybitmessagestatus.py config done') + +def read_callback(): + try: + clientStatus = json.loads(api.clientStatus()) + except: + collectd.info("Exception loading or parsing JSON") + return + + for i in ["networkConnections", "numberOfPubkeysProcessed", "numberOfMessagesProcessed", "numberOfBroadcastsProcessed"]: + metric = collectd.Values() + metric.plugin = "pybitmessagestatus" + if i[0:6] == "number": + metric.type = 'counter' + else: + metric.type = 'gauge' + metric.type_instance = i.lower() + try: + metric.values = [clientStatus[i]] + except: + collectd.info("Value for %s missing" % (i)) + metric.dispatch() + +if __name__ == "__main__": + main() +else: + collectd.register_init(init_callback) + collectd.register_config(config_callback) + collectd.register_read(read_callback) diff --git a/packages/systemd/bitmessage.service b/packages/systemd/bitmessage.service new file mode 100644 index 0000000000..1a9f7f4770 --- /dev/null +++ b/packages/systemd/bitmessage.service @@ -0,0 +1,18 @@ +[Unit] +Description=Bitmessage Daemon +After=network.target auditd.service + +[Service] +ExecStart=/usr/bin/python2 /usr/src/PyBitmessage/src/bitmessagemain.py +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure +Type=forking +PIDFile=/var/lib/bitmessage/.config/PyBitmessage/singleton.lock +User=bitmessage +Group=nogroup +WorkingDirectory=/var/lib/bitmessage +Environment="HOME=/var/lib/bitmessage" + +[Install] +WantedBy=multi-user.target diff --git a/setup.py b/setup.py index 0851b78f20..ba34f6df5a 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def run(self): 'qrcode': ['qrcode'], 'pyopencl': ['pyopencl'], 'notify2': ['notify2'], - 'sound:platform_system=="Windows"': ['winsound'] + 'sound;platform_system=="Windows"': ['winsound'] }, classifiers=[ "License :: OSI Approved :: MIT License" diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 83a419198f..6f79693928 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -342,13 +342,21 @@ def start(self): sleep(1) def daemonize(self): + grandfatherPid = os.getpid() + parentPid = None try: if os.fork(): + # unlock + shared.thisapp.cleanup() + # wait until grandchild ready + while True: + sleep(1) os._exit(0) except AttributeError: # fork not implemented pass else: + parentPid = os.getpid() shared.thisapp.lock() # relock os.umask(0) try: @@ -358,6 +366,11 @@ def daemonize(self): pass try: if os.fork(): + # unlock + shared.thisapp.cleanup() + # wait until child ready + while True: + sleep(1) os._exit(0) except AttributeError: # fork not implemented @@ -374,6 +387,10 @@ def daemonize(self): os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) + if parentPid: + # signal ready + os.kill(parentPid, signal.SIGTERM) + os.kill(grandfatherPid, signal.SIGTERM) def setSignalHandler(self): signal.signal(signal.SIGINT, helper_generic.signal_handler) diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index c731b469d5..e6ff0cd0d2 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -69,7 +69,7 @@ import shutdown import state from statusbar import BMStatusBar -import throttle +from network.asyncore_pollchoose import set_rates from version import softwareVersion import sound @@ -2288,16 +2288,16 @@ def click_actionSettings(self): int(float(self.settingsDialogInstance.ui.lineEditMaxDownloadRate.text())))) BMConfigParser().set('bitmessagesettings', 'maxuploadrate', str( int(float(self.settingsDialogInstance.ui.lineEditMaxUploadRate.text())))) - except: + except ValueError: QMessageBox.about(self, _translate("MainWindow", "Number needed"), _translate( "MainWindow", "Your maximum download and upload rate must be numbers. Ignoring what you typed.")) + else: + set_rates(BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate"), + BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate")) BMConfigParser().set('bitmessagesettings', 'maxoutboundconnections', str( int(float(self.settingsDialogInstance.ui.lineEditMaxOutboundConnections.text())))) - throttle.SendThrottle().resetLimit() - throttle.ReceiveThrottle().resetLimit() - BMConfigParser().set('bitmessagesettings', 'namecoinrpctype', self.settingsDialogInstance.getNamecoinType()) BMConfigParser().set('bitmessagesettings', 'namecoinrpchost', str( diff --git a/src/helper_generic.py b/src/helper_generic.py index b750e51915..4f7a12996c 100644 --- a/src/helper_generic.py +++ b/src/helper_generic.py @@ -51,7 +51,7 @@ def signal_handler(signal, frame): raise SystemExit if "PoolWorker" in current_process().name: raise SystemExit - if current_thread().name != "PyBitmessage": + if current_thread().name not in ("PyBitmessage", "MainThread"): return logger.error("Got signal %i", signal) if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'): diff --git a/src/network/advanceddispatcher.py b/src/network/advanceddispatcher.py index c50b6a43ad..3e84ed8522 100644 --- a/src/network/advanceddispatcher.py +++ b/src/network/advanceddispatcher.py @@ -75,7 +75,7 @@ def set_state(self, state, length=0, expectBytes=0): def writable(self): self.uploadChunk = AdvancedDispatcher._buf_len if asyncore.maxUploadRate > 0: - self.uploadChunk = asyncore.uploadBucket + self.uploadChunk = int(asyncore.uploadBucket) self.uploadChunk = min(self.uploadChunk, len(self.write_buf)) return asyncore.dispatcher.writable(self) and \ (self.connecting or (self.connected and self.uploadChunk > 0)) @@ -83,7 +83,7 @@ def writable(self): def readable(self): self.downloadChunk = AdvancedDispatcher._buf_len if asyncore.maxDownloadRate > 0: - self.downloadChunk = asyncore.downloadBucket + self.downloadChunk = int(asyncore.downloadBucket) try: if self.expectBytes > 0 and not self.fullyEstablished: self.downloadChunk = min(self.downloadChunk, self.expectBytes - len(self.read_buf)) diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index caa9d650e8..5717ff782d 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -112,6 +112,8 @@ class ExitNow(Exception): sentBytes = 0 def read(obj): + if not can_receive(): + return try: obj.handle_read_event() except _reraised_exceptions: @@ -120,6 +122,8 @@ def read(obj): obj.handle_error() def write(obj): + if not can_send(): + return try: obj.handle_write_event() except _reraised_exceptions: @@ -129,19 +133,25 @@ def write(obj): def set_rates(download, upload): global maxDownloadRate, maxUploadRate, downloadBucket, uploadBucket, downloadTimestamp, uploadTimestamp - maxDownloadRate = float(download) - maxUploadRate = float(upload) + maxDownloadRate = float(download) * 1024 + maxUploadRate = float(upload) * 1024 downloadBucket = maxDownloadRate uploadBucket = maxUploadRate downloadTimestamp = time.time() uploadTimestamp = time.time() +def can_receive(): + return maxDownloadRate == 0 or downloadBucket > 0 + +def can_send(): + return maxUploadRate == 0 or uploadBucket > 0 + def update_received(download=0): global receivedBytes, downloadBucket, downloadTimestamp currentTimestamp = time.time() receivedBytes += download if maxDownloadRate > 0: - bucketIncrease = int(maxDownloadRate * (currentTimestamp - downloadTimestamp)) + bucketIncrease = maxDownloadRate * (currentTimestamp - downloadTimestamp) downloadBucket += bucketIncrease if downloadBucket > maxDownloadRate: downloadBucket = int(maxDownloadRate) @@ -153,7 +163,7 @@ def update_sent(upload=0): currentTimestamp = time.time() sentBytes += upload if maxUploadRate > 0: - bucketIncrease = int(maxUploadRate * (currentTimestamp - uploadTimestamp)) + bucketIncrease = maxUploadRate * (currentTimestamp - uploadTimestamp) uploadBucket += bucketIncrease if uploadBucket > maxUploadRate: uploadBucket = int(maxUploadRate) @@ -170,9 +180,9 @@ def _exception(obj): def readwrite(obj, flags): try: - if flags & select.POLLIN: + if flags & select.POLLIN and can_receive(): obj.handle_read_event() - if flags & select.POLLOUT: + if flags & select.POLLOUT and can_send(): obj.handle_write_event() if flags & select.POLLPRI: obj.handle_expt_event() diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 249ec2ab33..267cac5873 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -8,23 +8,28 @@ import protocol import state -class BMObjectInsufficientPOWError(Exception): pass +class BMObjectInsufficientPOWError(Exception): + errorCodes = ("Insufficient proof of work") -class BMObjectInvalidDataError(Exception): pass +class BMObjectInvalidDataError(Exception): + errorCodes = ("Data invalid") -class BMObjectExpiredError(Exception): pass +class BMObjectExpiredError(Exception): + errorCodes = ("Object expired") -class BMObjectUnwantedStreamError(Exception): pass +class BMObjectUnwantedStreamError(Exception): + errorCodes = ("Object in unwanted stream") -class BMObjectInvalidError(Exception): pass +class BMObjectInvalidError(Exception): + errorCodes = ("Invalid object") class BMObjectAlreadyHaveError(Exception): - pass + errorCodes = ("Already have this object") class BMObject(object): diff --git a/src/network/bmproto.py b/src/network/bmproto.py index 17b2c76144..21ec692cfc 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -24,13 +24,16 @@ import state import protocol -class BMProtoError(ProxyError): pass +class BMProtoError(ProxyError): + errorCodes = ("Protocol error") -class BMProtoInsufficientDataError(BMProtoError): pass +class BMProtoInsufficientDataError(BMProtoError): + errorCodes = ("Insufficient data") -class BMProtoExcessiveDataError(BMProtoError): pass +class BMProtoExcessiveDataError(BMProtoError): + errorCodes = ("Too much data") class BMProto(AdvancedDispatcher, ObjectTracker): @@ -494,6 +497,20 @@ def peerValidityChecks(self): return False except: pass + if not self.isOutbound: + # incoming from a peer we're connected to as outbound, or server full + # report the same error to counter deanonymisation + if state.Peer(self.destination.host, self.peerNode.port) in \ + network.connectionpool.BMConnectionPool().inboundConnections or \ + len(network.connectionpool.BMConnectionPool().inboundConnections) + \ + len(network.connectionpool.BMConnectionPool().outboundConnections) > \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): + self.append_write_buf(protocol.assembleErrorMessage(fatal=2, + errorText="Server full, please try again later.")) + logger.debug ("Closed connection to %s due to server full or duplicate inbound/outbound.", + str(self.destination)) + return False if network.connectionpool.BMConnectionPool().isAlreadyConnected(self.nonce): self.append_write_buf(protocol.assembleErrorMessage(fatal=2, errorText="I'm connected to myself. Closing connection.")) diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 2f937a1558..44534a7668 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -22,8 +22,8 @@ class BMConnectionPool(object): def __init__(self): asyncore.set_rates( - BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate") * 1024, - BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate") * 1024) + BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate"), + BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate")) self.outboundConnections = {} self.inboundConnections = {} self.listeningSockets = {} @@ -65,12 +65,18 @@ def connectToStream(self, streamNumber): def getConnectionByAddr(self, addr): if addr in self.inboundConnections: return self.inboundConnections[addr] - if addr.host in self.inboundConnections: - return self.inboundConnections[addr.host] + try: + if addr.host in self.inboundConnections: + return self.inboundConnections[addr.host] + except AttributeError: + pass if addr in self.outboundConnections: return self.outboundConnections[addr] - if addr.host in self.udpSockets: - return self.udpSockets[addr.host] + try: + if addr.host in self.udpSockets: + return self.udpSockets[addr.host] + except AttributeError: + pass raise KeyError def isAlreadyConnected(self, nodeid): diff --git a/src/network/downloadthread.py b/src/network/downloadthread.py index 5fe1ee254f..35616f1bfb 100644 --- a/src/network/downloadthread.py +++ b/src/network/downloadthread.py @@ -43,7 +43,7 @@ def run(self): connections = BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values() random.shuffle(connections) try: - requestChunk = max(int(DownloadThread.maxRequestChunk / len(connections)), 1) + requestChunk = max(int(min(DownloadThread.maxRequestChunk, len(missingObjects)) / len(connections)), 1) except ZeroDivisionError: requestChunk = 1 for i in connections: @@ -63,15 +63,14 @@ def run(self): except KeyError: continue random.shuffle(request) + if len(request) > requestChunk - downloadPending: + request = request[:max(1, requestChunk - downloadPending)] if not request: continue - if len(request) > requestChunk - downloadPending: - request = request[:requestChunk - downloadPending] # mark them as pending for k in request: i.objectsNewToMe[k] = False missingObjects[k] = now - payload = bytearray() payload.extend(addresses.encodeVarint(len(request))) for chunk in request: diff --git a/src/network/objectracker.py b/src/network/objectracker.py index f846e7d5d7..62f01e4f1c 100644 --- a/src/network/objectracker.py +++ b/src/network/objectracker.py @@ -29,6 +29,7 @@ class ObjectTracker(object): invInitialCapacity = 50000 invErrorRate = 0.03 trackingExpires = 3600 + initialTimeOffset = 60 def __init__(self): self.objectsNewToMe = {} @@ -87,7 +88,7 @@ def handleReceivedInventory(self, hashId): if hashId in Dandelion().hashMap: Dandelion().fluffTrigger(hashId) if hashId not in missingObjects: - missingObjects[hashId] = time.time() + missingObjects[hashId] = time.time() - ObjectTracker.initialTimeOffset with self.objectsNewToMeLock: self.objectsNewToMe[hashId] = True diff --git a/src/network/proxy.py b/src/network/proxy.py index 7d46cd86fa..96930c189a 100644 --- a/src/network/proxy.py +++ b/src/network/proxy.py @@ -10,7 +10,7 @@ class ProxyError(Exception): errorCodes = ("UnknownError") - def __init__(self, code): + def __init__(self, code=-1): self.code = code try: self.message = self.__class__.errorCodes[self.code] diff --git a/src/network/tcp.py b/src/network/tcp.py index 70e22e0887..5a27aca365 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -180,7 +180,7 @@ def sendChunk(): payload += hash objectCount += 1 if objectCount >= BMProto.maxObjectCount: - self.sendChunk() + sendChunk() payload = b'' objectCount = 0 @@ -292,7 +292,10 @@ def handle_accept(self): if len(network.connectionpool.BMConnectionPool().inboundConnections) + \ len(network.connectionpool.BMConnectionPool().outboundConnections) > \ BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ - BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): + BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections") + 10: + # 10 is a sort of buffer, in between it will go through the version handshake + # and return an error to the peer + logger.warning("Server full, dropping connection") sock.close() return try: diff --git a/src/shutdown.py b/src/shutdown.py index 278759e571..49c2fb9b55 100644 --- a/src/shutdown.py +++ b/src/shutdown.py @@ -9,14 +9,12 @@ from helper_threading import StoppableThread from knownnodes import saveKnownNodes from inventory import Inventory -import protocol from queues import addressGeneratorQueue, objectProcessorQueue, UISignalQueue, workerQueue import shared import state def doCleanShutdown(): state.shutdown = 1 #Used to tell proof of work worker threads and the objectProcessorThread to exit. - protocol.broadcastToSendDataQueues((0, 'shutdown', 'no data')) objectProcessorQueue.put(('checkShutdownVariable', 'no data')) for thread in threading.enumerate(): if thread.isAlive() and isinstance(thread, StoppableThread): diff --git a/src/singleinstance.py b/src/singleinstance.py index 7a02594514..ed1048bae8 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -77,6 +77,15 @@ def cleanup(self): return if self.daemon and self.lockPid == os.getpid(): # these are the two initial forks while daemonizing + try: + if sys.platform == 'win32': + if hasattr(self, 'fd'): + os.close(self.fd) + else: + fcntl.lockf(self.fp, fcntl.LOCK_UN) + except Exception, e: + pass + return print "Cleaning up lockfile" try: diff --git a/src/version.py b/src/version.py index bd06ca9b44..694f71e419 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1,2 @@ +softwareName = 'PyBitmessage' softwareVersion = '0.6.2'