diff --git a/.travis.yml b/.travis.yml index 6fdf02ca..1737188e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,10 @@ install: before_script: - openssl version -a script: - - python -m pytest src/Test --cov src --cov-config src/Test/coverage.ini + - python -m pytest src/Test --cov --cov-config src/Test/coverage.ini before_install: + - pip install -U pytest mock pytest-cov - pip install codecov - - pip install pytest-cov - pip install coveralls after_success: - codecov diff --git a/src/Config.py b/src/Config.py index d5fd4c78..8fb8d133 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.2" - self.rev = 431 + self.rev = 445 self.argv = argv self.action = None self.createParser() diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 39e6a53d..8c7063be 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -12,7 +12,7 @@ from Crypt import CryptConnection class Connection(object): __slots__ = ( - "sock", "sock_wrapped", "ip", "port", "peer_id", "id", "protocol", "type", "server", "unpacker", "req_id", + "sock", "sock_wrapped", "ip", "port", "id", "protocol", "type", "server", "unpacker", "req_id", "handshake", "crypt", "connected", "event_connected", "closed", "start_time", "last_recv_time", "last_message_time", "last_send_time", "last_sent_time", "incomplete_buff_recv", "bytes_recv", "bytes_sent", "last_ping_delay", "last_req_time", "last_cmd", "name", "updateName", "waiting_requests", "waiting_streams" @@ -22,7 +22,6 @@ class Connection(object): self.sock = sock self.ip = ip self.port = port - self.peer_id = None # Bittorrent style peer id (not used yet) self.id = server.last_connection_id server.last_connection_id += 1 self.protocol = "?" @@ -161,6 +160,7 @@ class Connection(object): self.port = 0 else: self.port = handshake["fileserver_port"] # Set peer fileserver port + # Check if we can encrypt the connection if handshake.get("crypt_supported") and handshake["peer_id"] not in self.server.broken_ssl_peer_ids: if handshake.get("crypt"): # Recommended crypt by server diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index e15da4c7..72c53c83 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -28,7 +28,6 @@ class ConnectionServer: self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood self.broken_ssl_peer_ids = {} # Peerids of broken ssl connections self.ips = {} # Connection by ip - self.peer_ids = {} # Connections by peer_ids self.running = True self.thread_checker = gevent.spawn(self.checkConnections) @@ -55,10 +54,9 @@ class ConnectionServer: if request_handler: self.handleRequest = request_handler - CryptConnection.manager.loadCerts() - def start(self): self.running = True + CryptConnection.manager.loadCerts() self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % ( self.ip, self.port, ".".join(map(str, msgpack.version)), CryptConnection.manager.crypt_supported) @@ -84,7 +82,7 @@ class ConnectionServer: sock.close() return False else: - self.ip_incoming[ip] = 0 + self.ip_incoming[ip] = 1 connection = Connection(self, ip, port, sock) self.connections.append(connection) @@ -92,24 +90,21 @@ class ConnectionServer: connection.handleIncomingConnection(sock) def getConnection(self, ip=None, port=None, peer_id=None, create=True): - if peer_id and peer_id in self.peer_ids: # Find connection by peer id - connection = self.peer_ids.get(peer_id) - if not connection.connected and create: - succ = connection.event_connected.get() # Wait for connection - if not succ: - raise Exception("Connection event return error") - return connection # Find connection by ip if ip in self.ips: connection = self.ips[ip] - if not connection.connected and create: - succ = connection.event_connected.get() # Wait for connection - if not succ: - raise Exception("Connection event return error") - return connection + if not peer_id or connection.handshake.get("peer_id") == peer_id: # Filter by peer_id + if not connection.connected and create: + succ = connection.event_connected.get() # Wait for connection + if not succ: + raise Exception("Connection event return error") + return connection + # Recover from connection pool for connection in self.connections: if connection.ip == ip: + if peer_id and connection.handshake.get("peer_id") != peer_id: # Does not match + continue if not connection.connected and create: succ = connection.event_connected.get() # Wait for connection if not succ: @@ -141,8 +136,6 @@ class ConnectionServer: self.log.debug("Removing %s..." % connection) if self.ips.get(connection.ip) == connection: # Delete if same as in registry del self.ips[connection.ip] - if connection.peer_id and self.peer_ids.get(connection.peer_id) == connection: # Delete if same as in registry - del self.peer_ids[connection.peer_id] if connection in self.connections: self.connections.remove(connection) diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py index af805387..5608b740 100644 --- a/src/Crypt/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -53,12 +53,12 @@ class CryptConnectionManager: if config.disable_encryption: return False - if self.loadSslRsaCert(): + if self.createSslRsaCert(): self.crypt_supported.append("tls-rsa") # Try to create RSA server cert + sign for connection encryption # Return: True on success - def loadSslRsaCert(self): + def createSslRsaCert(self): import subprocess if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir): @@ -80,11 +80,11 @@ class CryptConnectionManager: if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir): return True else: - logging.error("RSA ECC SSL cert generation failed, cert or key files not exits.") + logging.error("RSA ECC SSL cert generation failed, cert or key files not exist.") return False # Not used yet: Missing on some platform - def createSslEccCert(self): + """def createSslEccCert(self): return False import subprocess @@ -116,6 +116,6 @@ class CryptConnectionManager: else: self.logging.error("ECC SSL cert generation failed, cert or key files not exits.") return False - + """ manager = CryptConnectionManager() \ No newline at end of file diff --git a/src/Db/Db.py b/src/Db/Db.py index 4c9bce02..4a5b043c 100644 --- a/src/Db/Db.py +++ b/src/Db/Db.py @@ -68,9 +68,9 @@ class Db: return self.cur.execute(query, params) def close(self): + self.log.debug("Closing, opened: %s" % opened_dbs) if self in opened_dbs: opened_dbs.remove(self) - self.log.debug("Closing") if self.cur: self.cur.close() if self.conn: diff --git a/src/File/FileServer.py b/src/File/FileServer.py index f5a57729..473224c9 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -15,14 +15,14 @@ from util import UpnpPunch class FileServer(ConnectionServer): - def __init__(self): - ConnectionServer.__init__(self, config.fileserver_ip, config.fileserver_port, self.handleRequest) + def __init__(self, ip=config.fileserver_ip, port=config.fileserver_port): + ConnectionServer.__init__(self, ip, port, self.handleRequest) if config.ip_external: # Ip external definied in arguments self.port_opened = True SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist else: self.port_opened = None # Is file server opened on router - self.sites = SiteManager.site_manager.list() + self.sites = {} # Handle request to fileserver def handleRequest(self, connection, message): @@ -228,6 +228,7 @@ class FileServer(ConnectionServer): # Bind and start serving sites def start(self, check_sites=True): + self.sites = SiteManager.site_manager.list() self.log = logging.getLogger("FileServer") if config.debug: diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index b408ced0..09f4702e 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -15,7 +15,7 @@ if config.use_tempfiles: # Communicate remote peers class Peer(object): __slots__ = ( - "ip", "port", "site", "key", "connection_server", "connection", "last_found", "last_response", + "ip", "port", "site", "key", "connection", "last_found", "last_response", "last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time" ) @@ -24,7 +24,6 @@ class Peer(object): self.port = port self.site = site self.key = "%s:%s" % (ip, port) - self.connection_server = sys.modules["main"].file_server self.connection = None self.last_found = time.time() # Time of last found in the torrent tracker @@ -57,7 +56,7 @@ class Peer(object): self.connection = None try: - self.connection = self.connection_server.getConnection(self.ip, self.port) + self.connection = self.site.connection_server.getConnection(self.ip, self.port) except Exception, err: self.onConnectionError() self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % @@ -69,7 +68,7 @@ class Peer(object): if self.connection and self.connection.connected: # We have connection to peer return self.connection else: # Try to find from other sites connections - self.connection = self.connection_server.getConnection(self.ip, self.port, create=False) + self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False) return self.connection def __str__(self): diff --git a/src/Site/Site.py b/src/Site/Site.py index 0a5a5537..7d6f0fe5 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -4,7 +4,6 @@ import logging import hashlib import re import time -import string import random import sys import binascii @@ -50,7 +49,11 @@ class Site: self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files self.loadSettings() # Load settings from sites.json self.content_manager = ContentManager(self) # Load contents - + self.connection_server = None + if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible + self.connection_server = sys.modules["main"].file_server + else: + self.connection_server = None if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed) self.settings["auth_key"] = CryptHash.random() self.log.debug("New auth key: %s" % self.settings["auth_key"]) @@ -430,6 +433,7 @@ class Site: # Rebuild DB if new_site.storage.isFile("dbschema.json"): + new_site.storage.closeDb() new_site.storage.rebuildDb() return new_site diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 23516d90..02f58f5f 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -17,7 +17,6 @@ class SiteStorage: def __init__(self, site, allow_create=True): self.site = site - self.fs_encoding = sys.getfilesystemencoding() self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory self.allowed_dir = os.path.abspath(self.directory) # Only serve/modify file within this dir self.log = site.log @@ -39,7 +38,10 @@ class SiteStorage: if check: if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: # Not exist or null self.rebuildDb() - self.db = Db(schema, db_path) + + if not self.db: + self.db = Db(schema, db_path) + if check and not self.db_checked: changed_tables = self.db.checkTables() if changed_tables: diff --git a/src/Test/TestConnection.py b/src/Test/TestConnection.py deleted file mode 100644 index bdce6d09..00000000 --- a/src/Test/TestConnection.py +++ /dev/null @@ -1,31 +0,0 @@ -import time - -from Crypt import CryptConnection - -class TestConnection: - def testSslConnection(self, connection_server): - server = connection_server - assert server.running - - # Connect to myself - connection = server.getConnection("127.0.0.1", 1544) - assert connection.handshake - assert connection.crypt - - # Close connection - connection.close() - time.sleep(0.01) - assert len(server.connections) == 0 - - def testRawConnection(self, connection_server): - server = connection_server - crypt_supported_bk = CryptConnection.manager.crypt_supported - CryptConnection.manager.crypt_supported = [] - - connection = server.getConnection("127.0.0.1", 1544) - assert not connection.crypt - - # Close connection - connection.close() - time.sleep(0.01) - assert len(server.connections) == 0 diff --git a/src/Test/TestConnectionServer.py b/src/Test/TestConnectionServer.py new file mode 100644 index 00000000..94175ffb --- /dev/null +++ b/src/Test/TestConnectionServer.py @@ -0,0 +1,100 @@ +import time +import gevent + +import pytest + +from Crypt import CryptConnection +from Connection import ConnectionServer + + +@pytest.mark.usefixtures("resetSettings") +class TestConnection: + def testSslConnection(self, file_server): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer("127.0.0.1", 1545) + assert file_server != client + + # Connect to myself + connection = client.getConnection("127.0.0.1", 1544) + assert len(file_server.connections) == 1 + assert len(file_server.ips) == 1 + assert connection.handshake + assert connection.crypt + + # Close connection + connection.close() + client.stop() + time.sleep(0.01) + assert len(file_server.connections) == 0 + assert len(file_server.ips) == 0 + + def testRawConnection(self, file_server): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer("127.0.0.1", 1545) + assert file_server != client + + # Remove all supported crypto + crypt_supported_bk = CryptConnection.manager.crypt_supported + CryptConnection.manager.crypt_supported = [] + + connection = client.getConnection("127.0.0.1", 1544) + assert len(file_server.connections) == 1 + assert not connection.crypt + + # Close connection + connection.close() + client.stop() + time.sleep(0.01) + assert len(file_server.connections) == 0 + + # Reset supported crypts + CryptConnection.manager.crypt_supported = crypt_supported_bk + + def testPing(self, file_server, site): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer("127.0.0.1", 1545) + connection = client.getConnection("127.0.0.1", 1544) + + assert connection.ping() + + connection.close() + client.stop() + + def testGetConnection(self, file_server): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer("127.0.0.1", 1545) + connection = client.getConnection("127.0.0.1", 1544) + + # Get connection by ip/port + connection2 = client.getConnection("127.0.0.1", 1544) + assert connection == connection2 + + # Get connection by peerid + assert not client.getConnection("127.0.0.1", 1544, peer_id="notexists", create=False) + connection2 = client.getConnection("127.0.0.1", 1544, peer_id=connection.handshake["peer_id"], create=False) + assert connection2 == connection + + connection.close() + client.stop() + + def testFloodProtection(self, file_server): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer("127.0.0.1", 1545) + + # Only allow 3 connection in 1 minute + connection = client.getConnection("127.0.0.1", 1544) + assert connection.handshake + connection.close() + + connection = client.getConnection("127.0.0.1", 1544) + assert connection.handshake + connection.close() + + connection = client.getConnection("127.0.0.1", 1544) + assert connection.handshake + connection.close() + + # The 4. one will timeout + with pytest.raises(gevent.Timeout): + with gevent.Timeout(0.1): + connection = client.getConnection("127.0.0.1", 1544) diff --git a/src/Test/TestContent.py b/src/Test/TestContent.py index 5067b414..c7130a5b 100644 --- a/src/Test/TestContent.py +++ b/src/Test/TestContent.py @@ -1,8 +1,11 @@ import json +import time from cStringIO import StringIO import pytest +from Crypt import CryptBitcoin + @pytest.mark.usefixtures("resetSettings") class TestContent: @@ -31,7 +34,8 @@ class TestContent: # Valid signers for root content.json assert site.content_manager.getValidSigners("content.json") == ["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] - def testSizelimit(self, site): + def testLimits(self, site): + privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # Data validation data_dict = { "files": { @@ -40,29 +44,35 @@ class TestContent: "size": 505 } }, - "modified": 1431451896.656619, - "signs": { - "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo": - "G2QC+ZIozPQQ/XiOEOMzfekOP8ipi+rKaTy/R/3MnDf98mLIhSSA8927FW6D/ZyP7HHuII2y9d0zbAk+rr8ksQM=" - } + "modified": time.time() } - data = StringIO(json.dumps(data_dict)) # Normal data + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + # Reset + del data_dict["signs"] # Too large data_dict["files"]["data.json"]["size"] = 200000 # Emulate 2MB sized data.json + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) - data_dict["files"]["data.json"]["size"] = 505 # Reset + # Reset + data_dict["files"]["data.json"]["size"] = 505 + del data_dict["signs"] # Not allowed file data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"] + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) - del data_dict["files"]["notallowed.exe"] # Reset + # Reset + del data_dict["files"]["notallowed.exe"] + del data_dict["signs"] # Should work again + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) diff --git a/src/Test/TestUserContent.py b/src/Test/TestContentUser.py similarity index 100% rename from src/Test/TestUserContent.py rename to src/Test/TestContentUser.py diff --git a/src/Test/TestCryptConnection.py b/src/Test/TestCryptConnection.py index 4027f047..46d2affc 100644 --- a/src/Test/TestCryptConnection.py +++ b/src/Test/TestCryptConnection.py @@ -21,7 +21,3 @@ class TestCryptConnection: # Check openssl cert generation assert os.path.isfile("%s/cert-rsa.pem" % config.data_dir) assert os.path.isfile("%s/key-rsa.pem" % config.data_dir) - - # Remove created files - os.unlink("%s/cert-rsa.pem" % config.data_dir) - os.unlink("%s/key-rsa.pem" % config.data_dir) diff --git a/src/Test/TestEvent.py b/src/Test/TestEvent.py index b05bca85..8bdafaaa 100644 --- a/src/Test/TestEvent.py +++ b/src/Test/TestEvent.py @@ -1,7 +1,7 @@ import util -class TestClass(object): +class ExampleClass(object): def __init__(self): self.called = [] self.onChanged = util.Event() @@ -12,7 +12,7 @@ class TestClass(object): class TestEvent: def testEvent(self): - test_obj = TestClass() + test_obj = ExampleClass() test_obj.onChanged.append(lambda: test_obj.increment("Called #1")) test_obj.onChanged.append(lambda: test_obj.increment("Called #2")) test_obj.onChanged.once(lambda: test_obj.increment("Once")) @@ -25,7 +25,7 @@ class TestEvent: assert test_obj.called == ["Called #1", "Called #2", "Once", "Called #1", "Called #2", "Called #1", "Called #2"] def testOnce(self): - test_obj = TestClass() + test_obj = ExampleClass() test_obj.onChanged.once(lambda: test_obj.increment("Once test #1")) # It should be called only once @@ -37,7 +37,7 @@ class TestEvent: assert test_obj.called == ["Once test #1"] def testOnceMultiple(self): - test_obj = TestClass() + test_obj = ExampleClass() # Allow queue more than once test_obj.onChanged.once(lambda: test_obj.increment("Once test #1")) test_obj.onChanged.once(lambda: test_obj.increment("Once test #2")) @@ -51,7 +51,7 @@ class TestEvent: assert test_obj.called == ["Once test #1", "Once test #2", "Once test #3"] def testOnceNamed(self): - test_obj = TestClass() + test_obj = ExampleClass() # Dont store more that one from same type test_obj.onChanged.once(lambda: test_obj.increment("Once test #1/1"), "type 1") test_obj.onChanged.once(lambda: test_obj.increment("Once test #1/2"), "type 1") diff --git a/src/Test/TestFileRequest.py b/src/Test/TestFileRequest.py new file mode 100644 index 00000000..c7908c87 --- /dev/null +++ b/src/Test/TestFileRequest.py @@ -0,0 +1,35 @@ +import cStringIO as StringIO + +import pytest + +from Connection import ConnectionServer + + +@pytest.mark.usefixtures("resetSettings") +class TestFileRequest: + def testGetFile(self, file_server, site): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer("127.0.0.1", 1545) + + connection = client.getConnection("127.0.0.1", 1544) + file_server.sites[site.address] = site + + response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0}) + assert "sign" in response["body"] + + connection.close() + client.stop() + + def testStreamFile(self, file_server, site): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer("127.0.0.1", 1545) + connection = client.getConnection("127.0.0.1", 1544) + file_server.sites[site.address] = site + + buff = StringIO.StringIO() + response = connection.request("streamFile", {"site": site.address, "inner_path": "content.json", "location": 0}, buff) + assert "stream_bytes" in response + assert "sign" in buff.getvalue() + + connection.close() + client.stop() diff --git a/src/Test/TestNoparallel.py b/src/Test/TestNoparallel.py index 71fcc3bd..abc4c767 100644 --- a/src/Test/TestNoparallel.py +++ b/src/Test/TestNoparallel.py @@ -6,7 +6,7 @@ monkey.patch_all() import util -class TestClass(object): +class ExampleClass(object): def __init__(self): self.counted = 0 @@ -27,8 +27,8 @@ class TestClass(object): class TestNoparallel: def testBlocking(self): - obj1 = TestClass() - obj2 = TestClass() + obj1 = ExampleClass() + obj2 = ExampleClass() # Dont allow to call again until its running and wait until its running threads = [ @@ -46,8 +46,8 @@ class TestNoparallel: assert obj2.counted == 10 def testNoblocking(self): - obj1 = TestClass() - obj2 = TestClass() + obj1 = ExampleClass() + obj2 = ExampleClass() thread1 = obj1.countNoblocking() thread2 = obj1.countNoblocking() # Ignored diff --git a/src/Test/TestRateLimit.py b/src/Test/TestRateLimit.py index 3ff779ad..a823d88b 100644 --- a/src/Test/TestRateLimit.py +++ b/src/Test/TestRateLimit.py @@ -7,12 +7,12 @@ monkey.patch_all() from util import RateLimit -# Time is around limit +/- 0.01 sec +# Time is around limit +/- 0.05 sec def around(t, limit): - return t >= limit - 0.01 and t <= limit + 0.01 + return t >= limit - 0.05 and t <= limit + 0.05 -class TestClass(object): +class ExampleClass(object): def __init__(self): self.counted = 0 self.last_called = None @@ -25,8 +25,8 @@ class TestClass(object): class TestRateLimit: def testCall(self): - obj1 = TestClass() - obj2 = TestClass() + obj1 = ExampleClass() + obj2 = ExampleClass() s = time.time() assert RateLimit.call("counting", allowed_again=0.1, func=obj1.count) == "counted" @@ -61,8 +61,8 @@ class TestRateLimit: assert obj2.counted == 4 def testCallAsync(self): - obj1 = TestClass() - obj2 = TestClass() + obj1 = ExampleClass() + obj2 = ExampleClass() s = time.time() RateLimit.callAsync("counting async", allowed_again=0.1, func=obj1.count, back="call #1").join() diff --git a/src/Test/TestWorker.py b/src/Test/TestWorker.py new file mode 100644 index 00000000..36a329ef --- /dev/null +++ b/src/Test/TestWorker.py @@ -0,0 +1,35 @@ +import time +import os + +import gevent +import pytest +import mock + +from Crypt import CryptConnection +from Connection import ConnectionServer +from Config import config +from Site import Site + +@pytest.mark.usefixtures("resetTempSettings") +@pytest.mark.usefixtures("resetSettings") +class TestWorker: + def testDownload(self, file_server, site, site_temp): + client = ConnectionServer("127.0.0.1", 1545) + assert site.storage.directory == config.data_dir+"/"+site.address + assert site_temp.storage.directory == config.data_dir+"-temp/"+site.address + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + site_temp.connection_server = client + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + # Download to client from source + site_temp.addPeer("127.0.0.1", 1544) + site_temp.download().join(timeout=5) + + assert not site_temp.bad_files + + site_temp.storage.deleteFiles() diff --git a/src/Test/conftest.py b/src/Test/conftest.py index 5d80f9a7..6bbaf397 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -2,8 +2,10 @@ import os import sys import urllib import time +import logging import pytest +import mock # Config if sys.platform == "win32": @@ -14,24 +16,27 @@ SITE_URL = "http://127.0.0.1:43110" # Imports relative to src dir sys.path.append( - os.path.abspath(os.path.dirname(__file__)+"/..") + os.path.abspath(os.path.dirname(__file__) + "/..") ) from Config import config config.argv = ["none"] # Dont pass any argv to config parser config.parse() config.data_dir = "src/Test/testdata" # Use test data for unittests +logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) from Site import Site from User import UserManager +from File import FileServer from Connection import ConnectionServer from Crypt import CryptConnection import gevent from gevent import monkey monkey.patch_all(thread=False) + @pytest.fixture(scope="session") def resetSettings(request): - os.chdir(os.path.abspath(os.path.dirname(__file__)+"/../..")) # Set working dir + os.chdir(os.path.abspath(os.path.dirname(__file__) + "/../..")) # Set working dir open("%s/sites.json" % config.data_dir, "w").write("{}") open("%s/users.json" % config.data_dir, "w").write(""" { @@ -42,23 +47,58 @@ def resetSettings(request): } } """) + def cleanup(): os.unlink("%s/sites.json" % config.data_dir) os.unlink("%s/users.json" % config.data_dir) request.addfinalizer(cleanup) + +@pytest.fixture(scope="session") +def resetTempSettings(request): + data_dir_temp = config.data_dir + "-temp" + if not os.path.isdir(data_dir_temp): + os.mkdir(data_dir_temp) + open("%s/sites.json" % data_dir_temp, "w").write("{}") + open("%s/users.json" % data_dir_temp, "w").write(""" + { + "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc": { + "certs": {}, + "master_seed": "024bceac1105483d66585d8a60eaf20aa8c3254b0f266e0d626ddb6114e2949a", + "sites": {} + } + } + """) + + def cleanup(): + os.unlink("%s/sites.json" % data_dir_temp) + os.unlink("%s/users.json" % data_dir_temp) + request.addfinalizer(cleanup) + + @pytest.fixture(scope="session") def site(): site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") return site +@pytest.fixture() +def site_temp(request): + with mock.patch("Config.config.data_dir", config.data_dir+"-temp"): + site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") + def cleanup(): + site_temp.storage.deleteFiles() + request.addfinalizer(cleanup) + return site_temp + + @pytest.fixture(scope="session") def user(): user = UserManager.user_manager.get() user.sites = {} # Reset user data return user + @pytest.fixture(scope="session") def browser(): try: @@ -69,6 +109,7 @@ def browser(): raise pytest.skip("Test requires selenium + phantomjs: %s" % err) return browser + @pytest.fixture(scope="session") def site_url(): try: @@ -77,9 +118,14 @@ def site_url(): raise pytest.skip("Test requires zeronet client running: %s" % err) return SITE_URL + @pytest.fixture(scope="session") -def connection_server(): - connection_server = ConnectionServer("127.0.0.1", 1544) - gevent.spawn(connection_server.start) - time.sleep(0) # Port opening - return connection_server +def file_server(request): + CryptConnection.manager.loadCerts() # Load and create certs + request.addfinalizer(CryptConnection.manager.removeCerts) # Remove cert files after end + file_server = FileServer("127.0.0.1", 1544) + gevent.spawn(lambda: ConnectionServer.start(file_server)) + time.sleep(0) # Port opening + assert file_server.running + return file_server + diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json index f0c25a5e..3474221b 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json @@ -1,129 +1,133 @@ { - "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", - "background-color": "white", - "description": "Blogging platform Demo", - "domain": "Blog.ZeroNetwork.bit", - "files": { - "css/all.css": { - "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", - "size": 112710 - }, - "data-default/data.json": { - "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", - "size": 196 - }, - "data-default/users/content-default.json": { - "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", - "size": 679 - }, - "data/data.json": { - "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", - "size": 31126 - }, - "data/img/autoupdate.png": { - "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", - "size": 24460 - }, - "data/img/direct_domains.png": { - "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", - "size": 16185 - }, - "data/img/domain.png": { - "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", - "size": 11881 - }, - "data/img/memory.png": { - "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", - "size": 12775 - }, - "data/img/multiuser.png": { - "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", - "size": 29480 - }, - "data/img/progressbar.png": { - "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", - "size": 13294 - }, - "data/img/slides.png": { - "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", - "size": 14439 - }, - "data/img/slots_memory.png": { - "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", - "size": 9488 - }, - "data/img/trayicon.png": { - "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", - "size": 19040 - }, - "data/img/zeroblog-comments.png": { - "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", - "size": 24001 - }, - "data/img/zeroid.png": { - "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", - "size": 18875 - }, - "data/img/zeroname.png": { - "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", - "size": 36031 - }, - "data/img/zerotalk-mark.png": { - "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", - "size": 44862 - }, - "data/img/zerotalk-upvote.png": { - "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", - "size": 41092 - }, - "data/img/zerotalk.png": { - "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", - "size": 26606 - }, - "dbschema.json": { - "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", - "size": 1529 - }, - "img/loading.gif": { - "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", - "size": 723 - }, - "index.html": { - "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", - "size": 4667 - }, - "js/all.js": { - "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", - "size": 201178 - } + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "background-color": "white", + "description": "Blogging platform Demo", + "domain": "Blog.ZeroNetwork.bit", + "files": { + "css/all.css": { + "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", + "size": 112710 }, - "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", - "includes": { - "data/users/content.json": { - "signers": ["1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f"], - "signers_required": 1 - }, - "data/test_include/content.json": { - "added": 1424976057, - "files_allowed": "data.json", - "includes_allowed": false, - "max_size": 20000, - "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], - "signers_required": 1, - "user_id": 47, - "user_name": "test" - } + "data-default/data.json": { + "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", + "size": 196 }, - "modified": 1434801613.8, - "sign": [ - 107584248894581661953399064048991976924739126704981340547735906786807630121376, - 14052922268999375798453683972186312380248481029778104103759595432459320456230 - ], - "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", - "signs": { - "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HO9esgmZhPJqOG+fEpStwK+u5P/+4Kx5VGApNnbsyA0lBfEV6aWviIP6FBlP3sZSH/EMuoEw42lToRmLppbzmuM=" + "data-default/users/content-default.json": { + "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", + "size": 679 }, - "signs_required": 1, - "title": "ZeroBlog", - "zeronet_version": "0.3.1" + "data/data.json": { + "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", + "size": 31126 + }, + "data/img/autoupdate.png": { + "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", + "size": 24460 + }, + "data/img/direct_domains.png": { + "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", + "size": 16185 + }, + "data/img/domain.png": { + "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", + "size": 11881 + }, + "data/img/memory.png": { + "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", + "size": 12775 + }, + "data/img/multiuser.png": { + "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", + "size": 29480 + }, + "data/img/progressbar.png": { + "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", + "size": 13294 + }, + "data/img/slides.png": { + "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", + "size": 14439 + }, + "data/img/slots_memory.png": { + "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", + "size": 9488 + }, + "data/img/trayicon.png": { + "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", + "size": 19040 + }, + "data/img/zeroblog-comments.png": { + "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", + "size": 24001 + }, + "data/img/zeroid.png": { + "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", + "size": 18875 + }, + "data/img/zeroname.png": { + "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", + "size": 36031 + }, + "data/img/zerotalk-mark.png": { + "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", + "size": 44862 + }, + "data/img/zerotalk-upvote.png": { + "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", + "size": 41092 + }, + "data/img/zerotalk.png": { + "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", + "size": 26606 + }, + "data/test_include/data.json": { + "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", + "size": 505 + }, + "dbschema.json": { + "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", + "size": 1529 + }, + "img/loading.gif": { + "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", + "size": 723 + }, + "index.html": { + "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", + "size": 4667 + }, + "js/all.js": { + "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", + "size": 201178 + } + }, + "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", + "includes": { + "data/test_include/content.json": { + "added": 1424976057, + "files_allowed": "data.json", + "includes_allowed": false, + "max_size": 20000, + "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], + "signers_required": 1, + "user_id": 47, + "user_name": "test" + }, + "data/users/content.json": { + "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], + "signers_required": 1 + } + }, + "modified": 1443088239.123, + "sign": [ + 37796247323133993908968541760020085519225012317332056166386012116450888757672, + 8182016604193300184892407269063757269964429504791487428802219119125679030316 + ], + "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HJ+SuvmYh1DIyvqypUobaspZ3heUfYWoN34S4c2la5NgcBmpZ/YN4Xzi6wtP20W8DePXdsYMC0Azr+L8ZF7FAk4=" + }, + "signs_required": 1, + "title": "ZeroBlog", + "zeronet_version": "0.3.2" } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/test_include/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/test_include/content.json index 807f5ea2..b0bd92e8 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/test_include/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/test_include/content.json @@ -1,12 +1,12 @@ { - "files": { - "data.json": { - "sha512": "f6ea25af270ba6a67c90a053c34da8ba94e6cf2177d4ee7979fd517a31ca6479", - "size": 74 - } - }, - "modified": 1424976057.772182, - "signs": { - "1TaLk3zM7ZRskJvrh3ZNCDVGXvkJusPKQ": "G1Jy36d3LLu+Lh8ikGqOozyiYZ+NvF8QF1OdC6PfDt26bflPPQ0gwWw8AQdFmc/S3BMBDNt0tJshiiJcRK46j/c=" + "files": { + "data.json": { + "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", + "size": 505 } + }, + "modified": 1443088412.024, + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HPpRa/7ic/03aJ6vfz3zt3ezsnkDeaet85HGS3Rm9vCXWGsdOXboMynb/sZcTfPMC1bQ3zLRdUNMqmifKw/gnNg=" + } } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json index 11f9a29a..dabbe5f8 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json @@ -1,26 +1,26 @@ { - "files": {}, - "ignore": ".*", - "modified": 1432466966.003, - "signs": { - "1BLogC9LN4oPDcruNz3qo1ysa133E9AGg8": "HChU28lG4MCnAiui6wDAaVCD4QUrgSy4zZ67+MMHidcUJRkLGnO3j4Eb1N0AWQ86nhSBwoOQf08Rha7gRyTDlAk=" + "files": {}, + "ignore": ".*", + "modified": 1443088330.941, + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G/YCfchtojDA7EjXk5Xa6af5EaEME14LDAvVE9P8PCDb2ncWN79ZTMsczAx7N3HYyM9Vdqn+8or4hh28z4ITKqU=" + }, + "user_contents": { + "cert_signers": { + "zeroid.bit": [ "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] }, - "user_contents": { - "cert_signers": { - "zeroid.bit": [ "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] - }, - "permission_rules": { - ".*": { - "files_allowed": "data.json", - "max_size": 10000, - "signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ] - }, - "bitid/.*@zeroid.bit": { "max_size": 40000 }, - "bitmsg/.*@zeroid.bit": { "max_size": 15000 } - }, - "permissions": { - "bad@zeroid.bit": false, - "nofish@zeroid.bit": { "max_size": 100000 } - } + "permission_rules": { + ".*": { + "files_allowed": "data.json", + "max_size": 10000, + "signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ] + }, + "bitid/.*@zeroid.bit": { "max_size": 40000 }, + "bitmsg/.*@zeroid.bit": { "max_size": 15000 } + }, + "permissions": { + "bad@zeroid.bit": false, + "nofish@zeroid.bit": { "max_size": 100000 } } + } } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/zeroblog.db b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/zeroblog.db new file mode 100644 index 00000000..a22146e7 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/zeroblog.db differ diff --git a/src/main.py b/src/main.py index e1b22e49..5eb112f3 100644 --- a/src/main.py +++ b/src/main.py @@ -96,6 +96,8 @@ class Actions(object): global ui_server, file_server from File import FileServer from Ui import UiServer + logging.info("Creating FileServer....") + file_server = FileServer() logging.info("Creating UiServer....") ui_server = UiServer() @@ -103,9 +105,6 @@ class Actions(object): from Crypt import CryptConnection CryptConnection.manager.removeCerts() - logging.info("Creating FileServer....") - file_server = FileServer() - logging.info("Starting servers....") gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)])