2015-06-25 20:09:41 +02:00
|
|
|
import json
|
|
|
|
import time
|
|
|
|
import re
|
|
|
|
import os
|
|
|
|
import copy
|
|
|
|
|
|
|
|
import gevent
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
from Debug import Debug
|
|
|
|
from Crypt import CryptHash
|
|
|
|
from Config import config
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
from util import helper
|
2015-10-11 02:22:53 +02:00
|
|
|
from Peer import PeerHashfield
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2015-06-18 00:08:45 +02:00
|
|
|
class ContentManager(object):
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2015-06-17 23:49:47 +02:00
|
|
|
def __init__(self, site):
|
|
|
|
self.site = site
|
|
|
|
self.log = self.site.log
|
2015-06-25 20:09:41 +02:00
|
|
|
self.contents = {} # Known content.json (without files and includes)
|
2015-10-11 02:22:53 +02:00
|
|
|
self.hashfield = PeerHashfield()
|
|
|
|
self.site.onFileDone.append(lambda inner_path: self.addOptionalFile(inner_path))
|
2015-09-17 02:20:43 +02:00
|
|
|
self.loadContent(add_bad_files=False, delete_removed_files=False)
|
2015-06-17 23:49:47 +02:00
|
|
|
self.site.settings["size"] = self.getTotalSize()
|
|
|
|
|
|
|
|
# Load content.json to self.content
|
2015-09-16 00:01:23 +02:00
|
|
|
# Return: Changed files ["index.html", "data/messages.json"], Deleted files ["old.jpg"]
|
Version 0.3.4, Rev656, CryptMessage plugin for AES and ECIES encryption, Added pyelliptic lib for OpenSSSL based encryption methods, Test CryptMessage plugin, Force reload content.json before signing and after write, Escaped Sql IN queries support, Test Sql parameter escaping, ui_websocket Test fixture, Plugin testing support, Always return websocket errors as dict, Wait for file on weboscket fileGet command if its already in bad_files queue, PushState and ReplaceState url manipulation support in wrapper API, Per auth-address localstorage, Longer timeout for udp tracker query
2015-12-10 21:36:20 +01:00
|
|
|
def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True, force=False):
|
2015-06-25 20:09:41 +02:00
|
|
|
content_inner_path = content_inner_path.strip("/") # Remove / from begning
|
2015-06-17 23:49:47 +02:00
|
|
|
old_content = self.contents.get(content_inner_path)
|
|
|
|
content_path = self.site.storage.getPath(content_inner_path)
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path))
|
|
|
|
content_inner_dir = helper.getDirname(content_inner_path)
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
if os.path.isfile(content_path):
|
|
|
|
try:
|
2015-11-15 11:13:57 +01:00
|
|
|
# Check if file is newer than what we have
|
Version 0.3.4, Rev656, CryptMessage plugin for AES and ECIES encryption, Added pyelliptic lib for OpenSSSL based encryption methods, Test CryptMessage plugin, Force reload content.json before signing and after write, Escaped Sql IN queries support, Test Sql parameter escaping, ui_websocket Test fixture, Plugin testing support, Always return websocket errors as dict, Wait for file on weboscket fileGet command if its already in bad_files queue, PushState and ReplaceState url manipulation support in wrapper API, Per auth-address localstorage, Longer timeout for udp tracker query
2015-12-10 21:36:20 +01:00
|
|
|
if not force and old_content and not self.site.settings.get("own"):
|
2015-11-15 11:13:57 +01:00
|
|
|
for line in open(content_path):
|
Version 0.3.4, Rev656, CryptMessage plugin for AES and ECIES encryption, Added pyelliptic lib for OpenSSSL based encryption methods, Test CryptMessage plugin, Force reload content.json before signing and after write, Escaped Sql IN queries support, Test Sql parameter escaping, ui_websocket Test fixture, Plugin testing support, Always return websocket errors as dict, Wait for file on weboscket fileGet command if its already in bad_files queue, PushState and ReplaceState url manipulation support in wrapper API, Per auth-address localstorage, Longer timeout for udp tracker query
2015-12-10 21:36:20 +01:00
|
|
|
if '"modified"' not in line:
|
|
|
|
continue
|
|
|
|
match = re.search("([0-9\.]+),$", line.strip(" \r\n"))
|
|
|
|
if match and float(match.group(1)) <= old_content.get("modified", 0):
|
|
|
|
self.log.debug("loadContent same json file, skipping")
|
|
|
|
return [], []
|
2015-11-15 11:13:57 +01:00
|
|
|
|
2015-06-17 23:49:47 +02:00
|
|
|
new_content = json.load(open(content_path))
|
|
|
|
except Exception, err:
|
|
|
|
self.log.error("%s load error: %s" % (content_path, Debug.formatException(err)))
|
2015-09-16 00:01:23 +02:00
|
|
|
return [], []
|
2015-06-17 23:49:47 +02:00
|
|
|
else:
|
|
|
|
self.log.error("Content.json not exist: %s" % content_path)
|
2015-09-16 00:01:23 +02:00
|
|
|
return [], [] # Content.json not exist
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
try:
|
|
|
|
# Get the files where the sha512 changed
|
|
|
|
changed = []
|
2015-09-16 00:01:23 +02:00
|
|
|
deleted = []
|
|
|
|
# Check changed
|
2015-11-15 11:13:57 +01:00
|
|
|
for relative_path, info in new_content.get("files", {}).iteritems():
|
2015-06-17 23:49:47 +02:00
|
|
|
if "sha512" in info:
|
|
|
|
hash_type = "sha512"
|
2015-06-25 20:09:41 +02:00
|
|
|
else: # Backward compatiblity
|
2015-06-17 23:49:47 +02:00
|
|
|
hash_type = "sha1"
|
|
|
|
|
|
|
|
new_hash = info[hash_type]
|
2015-06-25 20:09:41 +02:00
|
|
|
if old_content and old_content["files"].get(relative_path): # We have the file in the old content
|
2015-06-17 23:49:47 +02:00
|
|
|
old_hash = old_content["files"][relative_path].get(hash_type)
|
2015-06-25 20:09:41 +02:00
|
|
|
else: # The file is not in the old content
|
2015-06-17 23:49:47 +02:00
|
|
|
old_hash = None
|
2015-06-25 20:09:41 +02:00
|
|
|
if old_hash != new_hash:
|
2015-09-16 00:01:23 +02:00
|
|
|
changed.append(content_inner_dir + relative_path)
|
|
|
|
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
# Check changed optional files
|
2015-11-15 11:13:57 +01:00
|
|
|
for relative_path, info in new_content.get("files_optional", {}).iteritems():
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
file_inner_path = content_inner_dir + relative_path
|
|
|
|
new_hash = info["sha512"]
|
|
|
|
if old_content and old_content.get("files_optional", {}).get(relative_path): # We have the file in the old content
|
|
|
|
old_hash = old_content["files_optional"][relative_path].get("sha512")
|
|
|
|
if old_hash != new_hash and self.site.settings.get("autodownloadoptional"):
|
|
|
|
changed.append(content_inner_dir + relative_path) # Download new file
|
|
|
|
elif old_hash != new_hash and not self.site.settings.get("own"):
|
|
|
|
try:
|
|
|
|
self.site.storage.delete(file_inner_path)
|
|
|
|
self.log.debug("Deleted changed optional file: %s" % file_inner_path)
|
|
|
|
except Exception, err:
|
|
|
|
self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
|
|
|
|
else: # The file is not in the old content
|
|
|
|
if self.site.settings.get("autodownloadoptional"):
|
|
|
|
changed.append(content_inner_dir + relative_path) # Download new file
|
|
|
|
|
2015-09-16 00:01:23 +02:00
|
|
|
# Check deleted
|
|
|
|
if old_content:
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
old_files = dict(
|
|
|
|
old_content.get("files", {}),
|
|
|
|
**old_content.get("files_optional", {})
|
|
|
|
)
|
|
|
|
|
|
|
|
new_files = dict(
|
|
|
|
new_content.get("files", {}),
|
|
|
|
**new_content.get("files_optional", {})
|
|
|
|
)
|
|
|
|
|
|
|
|
deleted = [content_inner_dir + key for key in old_files if key not in new_files]
|
|
|
|
if deleted and not self.site.settings.get("own"):
|
2015-09-16 00:01:23 +02:00
|
|
|
# Deleting files that no longer in content.json
|
|
|
|
for file_inner_path in deleted:
|
2015-09-17 02:20:43 +02:00
|
|
|
try:
|
|
|
|
self.site.storage.delete(file_inner_path)
|
|
|
|
self.log.debug("Deleted file: %s" % file_inner_path)
|
|
|
|
except Exception, err:
|
|
|
|
self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Load includes
|
|
|
|
if load_includes and "includes" in new_content:
|
|
|
|
for relative_path, info in new_content["includes"].items():
|
2015-09-16 00:01:23 +02:00
|
|
|
include_inner_path = content_inner_dir + relative_path
|
2015-06-25 20:09:41 +02:00
|
|
|
if self.site.storage.isFile(include_inner_path): # Content.json exists, load it
|
2015-09-16 00:01:23 +02:00
|
|
|
include_changed, include_deleted = self.loadContent(
|
|
|
|
include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files
|
|
|
|
)
|
|
|
|
if include_changed:
|
|
|
|
changed += include_changed # Add changed files
|
|
|
|
if include_deleted:
|
|
|
|
deleted += include_deleted # Add changed files
|
2015-06-25 20:09:41 +02:00
|
|
|
else: # Content.json not exist, add to changed files
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.debug("Missing include: %s" % include_inner_path)
|
|
|
|
changed += [include_inner_path]
|
|
|
|
|
|
|
|
# Load blind user includes (all subdir)
|
|
|
|
if load_includes and "user_contents" in new_content:
|
2015-09-16 00:01:23 +02:00
|
|
|
for relative_dir in os.listdir(content_dir):
|
|
|
|
include_inner_path = content_inner_dir + relative_dir + "/content.json"
|
2015-06-25 20:09:41 +02:00
|
|
|
if not self.site.storage.isFile(include_inner_path):
|
|
|
|
continue # Content.json not exist
|
2015-09-16 00:01:23 +02:00
|
|
|
include_changed, include_deleted = self.loadContent(
|
|
|
|
include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files,
|
|
|
|
load_includes=False
|
|
|
|
)
|
|
|
|
if include_changed:
|
|
|
|
changed += include_changed # Add changed files
|
|
|
|
if include_deleted:
|
|
|
|
deleted += include_deleted # Add changed files
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Update the content
|
|
|
|
self.contents[content_inner_path] = new_content
|
|
|
|
except Exception, err:
|
|
|
|
self.log.error("Content.json parse error: %s" % Debug.formatException(err))
|
2015-09-16 00:01:23 +02:00
|
|
|
return [], [] # Content.json parse error
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Add changed files to bad files
|
|
|
|
if add_bad_files:
|
|
|
|
for inner_path in changed:
|
2015-09-16 00:01:23 +02:00
|
|
|
self.site.bad_files[inner_path] = self.site.bad_files.get(inner_path, 0) + 1
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
if new_content["modified"] > self.site.settings.get("modified", 0):
|
2015-06-25 20:09:41 +02:00
|
|
|
# Dont store modifications in the far future (more than 10 minute)
|
|
|
|
self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"])
|
2015-06-17 23:49:47 +02:00
|
|
|
|
2015-09-16 00:01:23 +02:00
|
|
|
return changed, deleted
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Get total size of site
|
|
|
|
# Return: 32819 (size of files in kb)
|
|
|
|
def getTotalSize(self, ignore=None):
|
|
|
|
total_size = 0
|
|
|
|
for inner_path, content in self.contents.iteritems():
|
2015-06-25 20:09:41 +02:00
|
|
|
if inner_path == ignore:
|
|
|
|
continue
|
|
|
|
total_size += self.site.storage.getSize(inner_path) # Size of content.json
|
2015-06-17 23:49:47 +02:00
|
|
|
for file, info in content.get("files", {}).iteritems():
|
|
|
|
total_size += info["size"]
|
|
|
|
return total_size
|
|
|
|
|
|
|
|
# Find the file info line from self.contents
|
2015-07-12 20:36:46 +02:00
|
|
|
# Return: { "sha512": "c29d73d...21f518", "size": 41 , "content_inner_path": "content.json"}
|
2015-06-17 23:49:47 +02:00
|
|
|
def getFileInfo(self, inner_path):
|
2015-06-25 20:09:41 +02:00
|
|
|
dirs = inner_path.split("/") # Parent dirs of content.json
|
|
|
|
inner_path_parts = [dirs.pop()] # Filename relative to content.json
|
2015-06-17 23:49:47 +02:00
|
|
|
while True:
|
|
|
|
content_inner_path = "%s/content.json" % "/".join(dirs)
|
|
|
|
content = self.contents.get(content_inner_path.strip("/"))
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
|
|
|
# Check in files
|
|
|
|
if content and "files" in content:
|
2015-06-17 23:49:47 +02:00
|
|
|
back = content["files"].get("/".join(inner_path_parts))
|
|
|
|
if back:
|
|
|
|
back["content_inner_path"] = content_inner_path
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
back["optional"] = False
|
|
|
|
return back
|
|
|
|
|
|
|
|
# Check in optional files
|
|
|
|
if content and "files_optional" in content: # Check if file in this content.json
|
|
|
|
back = content["files_optional"].get("/".join(inner_path_parts))
|
|
|
|
if back:
|
|
|
|
back["content_inner_path"] = content_inner_path
|
|
|
|
back["optional"] = True
|
2015-06-17 23:49:47 +02:00
|
|
|
return back
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
# Return the rules if user dir
|
|
|
|
if content and "user_contents" in content:
|
2015-06-17 23:49:47 +02:00
|
|
|
back = content["user_contents"]
|
|
|
|
# Content.json is in the users dir
|
|
|
|
back["content_inner_path"] = re.sub("(.*)/.*?$", "\\1/content.json", inner_path)
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
back["optional"] = None
|
2015-06-17 23:49:47 +02:00
|
|
|
return back
|
|
|
|
|
|
|
|
# No inner path in this dir, lets try the parent dir
|
|
|
|
if dirs:
|
|
|
|
inner_path_parts.insert(0, dirs.pop())
|
|
|
|
else: # No more parent dirs
|
|
|
|
break
|
|
|
|
|
|
|
|
# Not found
|
|
|
|
return False
|
|
|
|
|
|
|
|
# Get rules for the file
|
|
|
|
# Return: The rules for the file or False if not allowed
|
|
|
|
def getRules(self, inner_path, content=None):
|
2015-06-25 20:09:41 +02:00
|
|
|
if not inner_path.endswith("content.json"): # Find the files content.json first
|
2015-06-17 23:49:47 +02:00
|
|
|
file_info = self.getFileInfo(inner_path)
|
2015-06-25 20:09:41 +02:00
|
|
|
if not file_info:
|
|
|
|
return False # File not found
|
2015-06-17 23:49:47 +02:00
|
|
|
inner_path = file_info["content_inner_path"]
|
2015-06-25 20:09:41 +02:00
|
|
|
dirs = inner_path.split("/") # Parent dirs of content.json
|
|
|
|
inner_path_parts = [dirs.pop()] # Filename relative to content.json
|
|
|
|
inner_path_parts.insert(0, dirs.pop()) # Dont check in self dir
|
2015-06-17 23:49:47 +02:00
|
|
|
while True:
|
|
|
|
content_inner_path = "%s/content.json" % "/".join(dirs)
|
|
|
|
parent_content = self.contents.get(content_inner_path.strip("/"))
|
|
|
|
if parent_content and "includes" in parent_content:
|
|
|
|
return parent_content["includes"].get("/".join(inner_path_parts))
|
|
|
|
elif parent_content and "user_contents" in parent_content:
|
|
|
|
return self.getUserContentRules(parent_content, inner_path, content)
|
2015-06-25 20:09:41 +02:00
|
|
|
else: # No inner path in this dir, lets try the parent dir
|
2015-06-17 23:49:47 +02:00
|
|
|
if dirs:
|
|
|
|
inner_path_parts.insert(0, dirs.pop())
|
2015-06-25 20:09:41 +02:00
|
|
|
else: # No more parent dirs
|
2015-06-17 23:49:47 +02:00
|
|
|
break
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
# Get rules for a user file
|
|
|
|
# Return: The rules of the file or False if not allowed
|
|
|
|
def getUserContentRules(self, parent_content, inner_path, content):
|
|
|
|
user_contents = parent_content["user_contents"]
|
2015-06-25 20:09:41 +02:00
|
|
|
user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) # Delivered for directory
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
try:
|
2015-06-25 20:09:41 +02:00
|
|
|
if not content:
|
|
|
|
content = self.site.storage.loadJson(inner_path) # Read the file if no content specified
|
2015-06-17 23:49:47 +02:00
|
|
|
except (Exception, ): # Content.json not exist
|
|
|
|
return {"signers": [user_address], "user_address": user_address} # Return information that we know for sure
|
|
|
|
|
|
|
|
"""if not "cert_user_name" in content: # New file, unknown user
|
|
|
|
content["cert_auth_type"] = "unknown"
|
|
|
|
content["cert_user_name"] = "unknown@unknown"
|
|
|
|
"""
|
2015-06-25 20:09:41 +02:00
|
|
|
user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit
|
2015-06-17 23:49:47 +02:00
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
rules = copy.copy(user_contents["permissions"].get(content["cert_user_id"], {})) # Default rules by username
|
|
|
|
if rules is False:
|
2015-06-17 23:49:47 +02:00
|
|
|
return False # User banned
|
|
|
|
if "signers" in rules:
|
|
|
|
rules["signers"] = rules["signers"][:] # Make copy of the signers
|
2015-06-25 20:09:41 +02:00
|
|
|
for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules
|
|
|
|
if not re.match(permission_pattern, user_urn):
|
|
|
|
continue # Rule is not valid for user
|
2015-06-17 23:49:47 +02:00
|
|
|
# Update rules if its better than current recorded ones
|
|
|
|
for key, val in permission_rules.iteritems():
|
|
|
|
if key not in rules:
|
|
|
|
if type(val) is list:
|
|
|
|
rules[key] = val[:] # Make copy
|
|
|
|
else:
|
|
|
|
rules[key] = val
|
|
|
|
elif type(val) is int: # Int, update if larger
|
|
|
|
if val > rules[key]:
|
|
|
|
rules[key] = val
|
|
|
|
elif hasattr(val, "startswith"): # String, update if longer
|
2015-06-25 20:09:41 +02:00
|
|
|
if len(val) > len(rules[key]):
|
|
|
|
rules[key] = val
|
2015-06-17 23:49:47 +02:00
|
|
|
elif type(val) is list: # List, append
|
|
|
|
rules[key] += val
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers
|
|
|
|
if "signers" not in rules:
|
|
|
|
rules["signers"] = []
|
|
|
|
rules["signers"].append(user_address) # Add user as valid signer
|
2015-06-17 23:49:47 +02:00
|
|
|
rules["user_address"] = user_address
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
rules["includes_allowed"] = False
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
return rules
|
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
# Hash files in directory
|
|
|
|
def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None):
|
|
|
|
files_node = {}
|
|
|
|
files_optional_node = {}
|
2015-10-24 13:57:46 +02:00
|
|
|
if not re.match("^[a-zA-Z0-9_\.\+-/]*$", dir_inner_path):
|
|
|
|
ignored = True
|
|
|
|
self.log.error("- [ERROR] Only ascii encoded directories allowed: %s" % dir_inner_path)
|
2015-09-28 22:07:26 +02:00
|
|
|
|
|
|
|
for file_relative_path in self.site.storage.list(dir_inner_path):
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
file_name = helper.getFilename(file_relative_path)
|
2015-09-28 22:07:26 +02:00
|
|
|
|
|
|
|
ignored = optional = False
|
|
|
|
if file_name == "content.json":
|
|
|
|
ignored = True
|
|
|
|
elif ignore_pattern and re.match(ignore_pattern, file_relative_path):
|
|
|
|
ignored = True
|
|
|
|
elif file_name.startswith("."):
|
|
|
|
ignored = True
|
2015-10-24 13:57:46 +02:00
|
|
|
elif not re.match("^[a-zA-Z0-9_\.\+\-/]+$", file_relative_path):
|
|
|
|
ignored = True
|
|
|
|
self.log.error("- [ERROR] Only ascii encodes filenames allowed: %s" % file_relative_path)
|
2015-09-28 22:07:26 +02:00
|
|
|
elif optional_pattern and re.match(optional_pattern, file_relative_path):
|
|
|
|
optional = True
|
|
|
|
|
|
|
|
if ignored: # Ignore content.json, definied regexp and files starting with .
|
|
|
|
self.log.info("- [SKIPPED] %s" % file_relative_path)
|
|
|
|
else:
|
|
|
|
file_path = self.site.storage.getPath(dir_inner_path + "/" + file_relative_path)
|
|
|
|
sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
|
|
|
|
if optional:
|
|
|
|
self.log.info("- [OPTIONAL] %s (SHA512: %s)" % (file_relative_path, sha512sum))
|
|
|
|
files_optional_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
|
2015-11-02 22:39:19 +01:00
|
|
|
self.hashfield.appendHash(sha512sum)
|
2015-09-28 22:07:26 +02:00
|
|
|
else:
|
|
|
|
self.log.info("- %s (SHA512: %s)" % (file_relative_path, sha512sum))
|
|
|
|
files_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
|
|
|
|
return files_node, files_optional_node
|
|
|
|
|
2015-06-17 23:49:47 +02:00
|
|
|
# Create and sign a content.json
|
|
|
|
# Return: The new content if filewrite = False
|
|
|
|
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None):
|
|
|
|
content = self.contents.get(inner_path)
|
2015-06-25 20:09:41 +02:00
|
|
|
if not content: # Content not exist yet, load default one
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.info("File %s not exist yet, loading default values..." % inner_path)
|
|
|
|
content = {"files": {}, "signs": {}} # Default content.json
|
|
|
|
if inner_path == "content.json": # It's the root content.json, add some more fields
|
|
|
|
content["title"] = "%s - ZeroNet_" % self.site.address
|
|
|
|
content["description"] = ""
|
|
|
|
content["signs_required"] = 1
|
|
|
|
content["ignore"] = ""
|
2015-06-25 20:09:41 +02:00
|
|
|
if extend:
|
|
|
|
content.update(extend) # Add custom fields
|
2015-06-17 23:49:47 +02:00
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
directory = helper.getDirname(self.site.storage.getPath(inner_path))
|
|
|
|
inner_directory = helper.getDirname(inner_path)
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.info("Opening site data directory: %s..." % directory)
|
|
|
|
|
|
|
|
changed_files = [inner_path]
|
2015-10-11 02:22:53 +02:00
|
|
|
files_node, files_optional_node = self.hashFiles(
|
|
|
|
helper.getDirname(inner_path), content.get("ignore"), content.get("optional")
|
|
|
|
)
|
2015-06-17 23:49:47 +02:00
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
# Find changed files
|
|
|
|
files_merged = files_node.copy()
|
|
|
|
files_merged.update(files_optional_node)
|
|
|
|
for file_relative_path, file_details in files_merged.iteritems():
|
|
|
|
old_hash = content["files"].get(file_relative_path, {}).get("sha512")
|
|
|
|
new_hash = files_merged[file_relative_path]["sha512"]
|
|
|
|
if old_hash != new_hash:
|
|
|
|
changed_files.append(inner_directory + file_relative_path)
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
self.log.debug("Changed files: %s" % changed_files)
|
|
|
|
if update_changed_files:
|
|
|
|
for file_path in changed_files:
|
|
|
|
self.site.storage.onUpdated(file_path)
|
|
|
|
|
|
|
|
# Generate new content.json
|
|
|
|
self.log.info("Adding timestamp and sha512sums to new content.json...")
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
new_content = content.copy() # Create a copy of current content.json
|
2015-09-28 22:07:26 +02:00
|
|
|
new_content["files"] = files_node # Add files sha512 hash
|
|
|
|
if files_optional_node:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
new_content["files_optional"] = files_optional_node
|
|
|
|
elif "files_optional" in new_content:
|
|
|
|
del new_content["files_optional"]
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
new_content["modified"] = time.time() # Add timestamp
|
2015-06-17 23:49:47 +02:00
|
|
|
if inner_path == "content.json":
|
|
|
|
new_content["address"] = self.site.address
|
|
|
|
new_content["zeronet_version"] = config.version
|
|
|
|
new_content["signs_required"] = content.get("signs_required", 1)
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
# Verify private key
|
2015-06-17 23:49:47 +02:00
|
|
|
from Crypt import CryptBitcoin
|
|
|
|
self.log.info("Verifying private key...")
|
|
|
|
privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey)
|
|
|
|
valid_signers = self.getValidSigners(inner_path, new_content)
|
|
|
|
if privatekey_address not in valid_signers:
|
2015-07-12 20:36:46 +02:00
|
|
|
return self.log.error(
|
|
|
|
"Private key invalid! Valid signers: %s, Private key address: %s" %
|
|
|
|
(valid_signers, privatekey_address)
|
|
|
|
)
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers))
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if inner_path == "content.json" and privatekey_address == self.site.address:
|
|
|
|
# If signing using the root key, then sign the valid signers
|
|
|
|
new_content["signers_sign"] = CryptBitcoin.sign(
|
|
|
|
"%s:%s" % (new_content["signs_required"], ",".join(valid_signers)), privatekey
|
|
|
|
)
|
2015-06-25 20:09:41 +02:00
|
|
|
if not new_content["signers_sign"]:
|
|
|
|
self.log.info("Old style address, signers_sign is none")
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
self.log.info("Signing %s..." % inner_path)
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
if "signs" in new_content:
|
|
|
|
del(new_content["signs"]) # Delete old signs
|
|
|
|
if "sign" in new_content:
|
|
|
|
del(new_content["sign"]) # Delete old sign (backward compatibility)
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
sign_content = json.dumps(new_content, sort_keys=True)
|
|
|
|
sign = CryptBitcoin.sign(sign_content, privatekey)
|
2015-06-25 20:09:41 +02:00
|
|
|
# new_content["signs"] = content.get("signs", {}) # TODO: Multisig
|
|
|
|
if sign: # If signing is successful (not an old address)
|
2015-06-17 23:49:47 +02:00
|
|
|
new_content["signs"] = {}
|
|
|
|
new_content["signs"][privatekey_address] = sign
|
|
|
|
|
|
|
|
if inner_path == "content.json": # To root content.json add old format sign for backward compatibility
|
|
|
|
oldsign_content = json.dumps(new_content, sort_keys=True)
|
|
|
|
new_content["sign"] = CryptBitcoin.signOld(oldsign_content, privatekey)
|
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
if not self.verifyContent(inner_path, new_content):
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.error("Sign failed: Invalid content")
|
|
|
|
return False
|
|
|
|
|
|
|
|
if filewrite:
|
|
|
|
self.log.info("Saving to %s..." % inner_path)
|
|
|
|
self.site.storage.writeJson(inner_path, new_content)
|
Version 0.3.4, Rev656, CryptMessage plugin for AES and ECIES encryption, Added pyelliptic lib for OpenSSSL based encryption methods, Test CryptMessage plugin, Force reload content.json before signing and after write, Escaped Sql IN queries support, Test Sql parameter escaping, ui_websocket Test fixture, Plugin testing support, Always return websocket errors as dict, Wait for file on weboscket fileGet command if its already in bad_files queue, PushState and ReplaceState url manipulation support in wrapper API, Per auth-address localstorage, Longer timeout for udp tracker query
2015-12-10 21:36:20 +01:00
|
|
|
self.contents[inner_path] = new_content
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
self.log.info("File %s signed!" % inner_path)
|
|
|
|
|
|
|
|
if filewrite: # Written to file
|
|
|
|
return True
|
|
|
|
else: # Return the new content
|
|
|
|
return new_content
|
|
|
|
|
|
|
|
# The valid signers of content.json file
|
|
|
|
# Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"]
|
|
|
|
def getValidSigners(self, inner_path, content=None):
|
|
|
|
valid_signers = []
|
2015-06-25 20:09:41 +02:00
|
|
|
if inner_path == "content.json": # Root content.json
|
2015-06-17 23:49:47 +02:00
|
|
|
if "content.json" in self.contents and "signers" in self.contents["content.json"]:
|
|
|
|
valid_signers += self.contents["content.json"]["signers"].keys()
|
|
|
|
else:
|
|
|
|
rules = self.getRules(inner_path, content)
|
|
|
|
if rules and "signers" in rules:
|
|
|
|
valid_signers += rules["signers"]
|
|
|
|
|
|
|
|
if self.site.address not in valid_signers:
|
|
|
|
valid_signers.append(self.site.address) # Site address always valid
|
|
|
|
return valid_signers
|
|
|
|
|
|
|
|
# Return: The required number of valid signs for the content.json
|
|
|
|
def getSignsRequired(self, inner_path, content=None):
|
|
|
|
return 1 # Todo: Multisig
|
|
|
|
|
|
|
|
def verifyCert(self, inner_path, content):
|
|
|
|
from Crypt import CryptBitcoin
|
|
|
|
|
|
|
|
rules = self.getRules(inner_path, content)
|
2015-06-25 20:09:41 +02:00
|
|
|
if not rules.get("cert_signers"):
|
|
|
|
return True # Does not need cert
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
name, domain = content["cert_user_id"].split("@")
|
|
|
|
cert_address = rules["cert_signers"].get(domain)
|
2015-06-25 20:09:41 +02:00
|
|
|
if not cert_address: # Cert signer not allowed
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.error("Invalid cert signer: %s" % domain)
|
|
|
|
return False
|
2015-07-12 20:36:46 +02:00
|
|
|
return CryptBitcoin.verify(
|
|
|
|
"%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"]
|
|
|
|
)
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Checks if the content.json content is valid
|
|
|
|
# Return: True or False
|
2015-09-28 22:07:26 +02:00
|
|
|
def verifyContent(self, inner_path, content):
|
2015-06-25 20:09:41 +02:00
|
|
|
content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
content_size_optional = sum([file["size"] for file in content.get("files_optional", {}).values()])
|
2015-06-25 20:09:41 +02:00
|
|
|
site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content
|
|
|
|
if site_size > self.site.settings.get("size", 0):
|
|
|
|
self.site.settings["size"] = site_size # Save to settings if larger
|
2015-06-17 23:49:47 +02:00
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
site_size_limit = self.site.getSizeLimit() * 1024 * 1024
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Check total site size limit
|
|
|
|
if site_size > site_size_limit:
|
|
|
|
self.log.error("%s: Site too large %s > %s, aborting task..." % (inner_path, site_size, site_size_limit))
|
|
|
|
task = self.site.worker_manager.findTask(inner_path)
|
2015-06-25 20:09:41 +02:00
|
|
|
if task: # Dont try to download from other peers
|
2015-06-17 23:49:47 +02:00
|
|
|
self.site.worker_manager.failTask(task)
|
|
|
|
return False
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
if inner_path == "content.json":
|
|
|
|
return True # Root content.json is passed
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Load include details
|
|
|
|
rules = self.getRules(inner_path, content)
|
|
|
|
if not rules:
|
|
|
|
self.log.error("%s: No rules" % inner_path)
|
|
|
|
return False
|
|
|
|
|
|
|
|
# Check include size limit
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
if rules.get("max_size") is not None: # Include size limit
|
2015-06-17 23:49:47 +02:00
|
|
|
if content_size > rules["max_size"]:
|
|
|
|
self.log.error("%s: Include too large %s > %s" % (inner_path, content_size, rules["max_size"]))
|
|
|
|
return False
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
if rules.get("max_size_optional") is not None: # Include optional files limit
|
|
|
|
if content_size_optional > rules["max_size_optional"]:
|
2015-10-11 02:22:53 +02:00
|
|
|
self.log.error("%s: Include optional files too large %s > %s" % (
|
|
|
|
inner_path, content_size_optional, rules["max_size_optional"])
|
|
|
|
)
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return False
|
2015-06-17 23:49:47 +02:00
|
|
|
|
|
|
|
# Filename limit
|
|
|
|
if rules.get("files_allowed"):
|
|
|
|
for file_inner_path in content["files"].keys():
|
|
|
|
if not re.match("^%s$" % rules["files_allowed"], file_inner_path):
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
self.log.error("%s %s: File not allowed" % (inner_path, file_inner_path))
|
|
|
|
return False
|
|
|
|
|
|
|
|
if rules.get("files_allowed_optional"):
|
|
|
|
for file_inner_path in content.get("files_optional", {}).keys():
|
|
|
|
if not re.match("^%s$" % rules["files_allowed_optional"], file_inner_path):
|
|
|
|
self.log.error("%s %s: Optional file not allowed" % (inner_path, file_inner_path))
|
2015-06-17 23:49:47 +02:00
|
|
|
return False
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
# Check if content includes allowed
|
|
|
|
if rules.get("includes_allowed") is False and content.get("includes"):
|
|
|
|
self.log.error("%s: Includes not allowed" % inner_path)
|
|
|
|
return False # Includes not allowed
|
|
|
|
|
2015-06-17 23:49:47 +02:00
|
|
|
return True # All good
|
|
|
|
|
|
|
|
# Verify file validity
|
|
|
|
# Return: None = Same as before, False = Invalid, True = Valid
|
2015-06-25 20:09:41 +02:00
|
|
|
def verifyFile(self, inner_path, file, ignore_same=True):
|
|
|
|
if inner_path.endswith("content.json"): # content.json: Check using sign
|
2015-06-17 23:49:47 +02:00
|
|
|
from Crypt import CryptBitcoin
|
|
|
|
try:
|
|
|
|
new_content = json.load(file)
|
|
|
|
if inner_path in self.contents:
|
|
|
|
old_content = self.contents.get(inner_path)
|
|
|
|
# Checks if its newer the ours
|
2015-06-25 20:09:41 +02:00
|
|
|
if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
|
2015-06-17 23:49:47 +02:00
|
|
|
return None
|
2015-06-25 20:09:41 +02:00
|
|
|
elif old_content["modified"] > new_content["modified"]: # We have newer
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug(
|
|
|
|
"We have newer %s (Our: %s, Sent: %s)" %
|
|
|
|
(inner_path, old_content["modified"], new_content["modified"])
|
|
|
|
)
|
2015-06-25 20:09:41 +02:00
|
|
|
gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers
|
2015-06-17 23:49:47 +02:00
|
|
|
return False
|
2015-07-12 20:36:46 +02:00
|
|
|
if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+)
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.error("%s modify is in the future!" % inner_path)
|
|
|
|
return False
|
|
|
|
# Check sign
|
|
|
|
sign = new_content.get("sign")
|
|
|
|
signs = new_content.get("signs", {})
|
2015-06-25 20:09:41 +02:00
|
|
|
if "sign" in new_content:
|
|
|
|
del(new_content["sign"]) # The file signed without the sign
|
|
|
|
if "signs" in new_content:
|
|
|
|
del(new_content["signs"]) # The file signed without the signs
|
|
|
|
sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
|
2015-06-17 23:49:47 +02:00
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
if not self.verifyContent(inner_path, new_content):
|
2015-06-25 20:09:41 +02:00
|
|
|
return False # Content not valid (files too large, invalid files)
|
2015-06-17 23:49:47 +02:00
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
if signs: # New style signing
|
2015-06-17 23:49:47 +02:00
|
|
|
valid_signers = self.getValidSigners(inner_path, new_content)
|
|
|
|
signs_required = self.getSignsRequired(inner_path, new_content)
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
|
2015-07-12 20:36:46 +02:00
|
|
|
if not CryptBitcoin.verify(
|
|
|
|
"%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]
|
|
|
|
):
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.error("%s invalid signers_sign!" % inner_path)
|
|
|
|
return False
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.error("%s invalid cert!" % inner_path)
|
|
|
|
return False
|
|
|
|
|
|
|
|
valid_signs = 0
|
|
|
|
for address in valid_signers:
|
2015-06-25 20:09:41 +02:00
|
|
|
if address in signs:
|
|
|
|
valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
|
|
|
|
if valid_signs >= signs_required:
|
|
|
|
break # Break if we has enough signs
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
self.log.debug("%s: Valid signs: %s/%s" % (inner_path, valid_signs, signs_required))
|
2015-06-17 23:49:47 +02:00
|
|
|
return valid_signs >= signs_required
|
2015-06-25 20:09:41 +02:00
|
|
|
else: # Old style signing
|
2015-06-17 23:49:47 +02:00
|
|
|
return CryptBitcoin.verify(sign_content, self.site.address, sign)
|
|
|
|
|
|
|
|
except Exception, err:
|
|
|
|
self.log.error("Verify sign error: %s" % Debug.formatException(err))
|
|
|
|
return False
|
|
|
|
|
|
|
|
else: # Check using sha512 hash
|
|
|
|
file_info = self.getFileInfo(inner_path)
|
|
|
|
if file_info:
|
|
|
|
if "sha512" in file_info:
|
|
|
|
hash_valid = CryptHash.sha512sum(file) == file_info["sha512"]
|
2015-06-25 20:09:41 +02:00
|
|
|
elif "sha1" in file_info: # Backward compatibility
|
2015-06-17 23:49:47 +02:00
|
|
|
hash_valid = CryptHash.sha1sum(file) == file_info["sha1"]
|
|
|
|
else:
|
|
|
|
hash_valid = False
|
2015-09-13 12:52:11 +02:00
|
|
|
if file_info.get("size", 0) != file.tell():
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.error(
|
|
|
|
"%s file size does not match %s <> %s, Hash: %s" %
|
2015-09-13 12:52:11 +02:00
|
|
|
(inner_path, file.tell(), file_info.get("size", 0), hash_valid)
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
2015-06-17 23:49:47 +02:00
|
|
|
return False
|
|
|
|
return hash_valid
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
else: # File not in content.json
|
2015-06-17 23:49:47 +02:00
|
|
|
self.log.error("File not in content.json: %s" % inner_path)
|
|
|
|
return False
|
|
|
|
|
2015-10-11 02:22:53 +02:00
|
|
|
def addOptionalFile(self, inner_path):
|
|
|
|
info = self.getFileInfo(inner_path)
|
|
|
|
if info and info["optional"]:
|
|
|
|
self.log.debug("Downloaded optional file, adding to hashfield: %s" % inner_path)
|
|
|
|
self.hashfield.appendHash(info["sha512"])
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
if __name__ == "__main__":
|
|
|
|
def testSign():
|
|
|
|
global config
|
|
|
|
from Site import Site
|
|
|
|
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
|
|
|
content_manager = ContentManager(site)
|
|
|
|
content_manager.sign(
|
|
|
|
"data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/content.json", "5JCGE6UUruhfmAfcZ2GYjvrswkaiq7uLo6Gmtf2ep2Jh2jtNzWR"
|
|
|
|
)
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
def testVerify():
|
|
|
|
from Site import Site
|
|
|
|
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
content_manager = ContentManager(site)
|
|
|
|
print "Loaded contents:", content_manager.contents.keys()
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
file = open(site.storage.getPath("data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/content.json"))
|
|
|
|
print "content.json valid:", content_manager.verifyFile(
|
|
|
|
"data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/content.json", file, ignore_same=False
|
|
|
|
)
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
file = open(site.storage.getPath("data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/messages.json"))
|
|
|
|
print "messages.json valid:", content_manager.verifyFile(
|
|
|
|
"data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/messages.json", file, ignore_same=False
|
|
|
|
)
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
def testInfo():
|
|
|
|
from Site import Site
|
|
|
|
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
content_manager = ContentManager(site)
|
|
|
|
print content_manager.contents.keys()
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
print content_manager.getFileInfo("index.html")
|
|
|
|
print content_manager.getIncludeInfo("data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/content.json")
|
|
|
|
print content_manager.getValidSigners("data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/content.json")
|
|
|
|
print content_manager.getValidSigners("data/users/content.json")
|
|
|
|
print content_manager.getValidSigners("content.json")
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
import sys
|
|
|
|
import logging
|
2015-06-17 23:49:47 +02:00
|
|
|
os.chdir("../..")
|
|
|
|
sys.path.insert(0, os.path.abspath("."))
|
|
|
|
sys.path.insert(0, os.path.abspath("src"))
|
|
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
# testSign()
|
2015-06-17 23:49:47 +02:00
|
|
|
testVerify()
|
2015-06-25 20:09:41 +02:00
|
|
|
# testInfo()
|