HelloZeroNet/ZeroNet
1
0
Fork 0
mirror of https://github.com/HelloZeroNet/ZeroNet.git synced 2023-12-14 04:33:03 +01:00
Code Issues Projects Releases Wiki Activity

VerifyFiles call returns more detailed statistics

Browse source
This commit is contained in:
shortcutme 2018-03-29 02:49:06 +02:00
parent 6daf583b4b
commit d61cd96d8f
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
4 changed files with 17 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
plugins/Bigfile/Test/TestBigfile.py
View file

@ -136,7 +136,7 @@ class TestBigfile:
# Download site
site_temp.download(blind_includes=True).join(timeout=5)
bad_files = site_temp.storage.verifyFiles(quick_check=True)
bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"]
assert not bad_files
# client_piecefield = peer_client.piecefields[file_info["sha512"]].tostring()

15
src/Site/SiteStorage.py
View file

@ -4,6 +4,7 @@ import shutil
import json
import time
import sys
from collections import defaultdict
import sqlite3
import gevent.event
@ -376,25 +377,32 @@ class SiteStorage(object):
# Verify all files sha512sum using content.json
def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
bad_files = []
back = defaultdict(int)
back["bad_files"] = bad_files
i = 0
self.log.debug("Verifing files...")
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
self.log.debug("VerifyFile content.json not exists")
self.site.needFile("content.json", update=True) # Force update to fix corrupt file
self.site.content_manager.loadContent() # Reload content.json
for content_inner_path, content in self.site.content_manager.contents.items():
back["num_content"] += 1
i += 1
if i % 50 == 0:
time.sleep(0.0001) # Context switch to avoid gevent hangs
if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file
back["num_content_missing"] += 1
self.log.debug("[MISSING] %s" % content_inner_path)
bad_files.append(content_inner_path)
for file_relative_path in content.get("files", {}).keys():
back["num_file"] += 1
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
file_inner_path = file_inner_path.strip("/") # Strip leading /
file_path = self.getPath(file_inner_path)
if not os.path.isfile(file_path):
back["num_file_missing"] += 1
self.log.debug("[MISSING] %s" % file_inner_path)
bad_files.append(file_inner_path)
continue
@ -410,6 +418,7 @@ class SiteStorage(object):
ok = False
if not ok:
back["num_file_invalid"] += 1
self.log.debug("[INVALID] %s: %s" % (file_inner_path, err))
if add_changed or content.get("cert_user_id"): # If updating own site only add changed user files
bad_files.append(file_inner_path)
@ -418,6 +427,7 @@ class SiteStorage(object):
optional_added = 0
optional_removed = 0
for file_relative_path in content.get("files_optional", {}).keys():
back["num_optional"] += 1
file_node = content["files_optional"][file_relative_path]
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
file_inner_path = file_inner_path.strip("/") # Strip leading /
@ -455,16 +465,17 @@ class SiteStorage(object):
)
time.sleep(0.0001) # Context switch to avoid gevent hangs
return bad_files
return back
# Check and try to fix site files integrity
def updateBadFiles(self, quick_check=True):
s = time.time()
bad_files = self.verifyFiles(
res = self.verifyFiles(
quick_check,
add_optional=self.site.isDownloadable(""),
add_changed=not self.site.settings.get("own") # Don't overwrite changed files if site owned
)
bad_files = res["bad_files"]
self.site.bad_files = {}
if bad_files:
for bad_file in bad_files:

2
src/Test/TestSite.py
View file

@ -26,7 +26,7 @@ class TestSite:
assert new_site.storage.isFile("index.html")
assert new_site.storage.isFile("data/users/content.json")
assert new_site.storage.isFile("data/zeroblog.db")
assert new_site.storage.verifyFiles() == [] # No bad files allowed
assert new_site.storage.verifyFiles()["bad_files"] == [] # No bad files allowed
assert new_site.storage.query("SELECT * FROM keyvalue WHERE key = 'title'").fetchone()["value"] == "MyZeroBlog"
# Test re-cloning (updating)

4
src/Test/TestSiteDownload.py
View file

@ -48,7 +48,7 @@ class TestSiteDownload:
assert "-default" in file_requests[-1] # Put default files for cloning to the end
# Check files
bad_files = site_temp.storage.verifyFiles(quick_check=True)
bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"]
# -1 because data/users/1J6... user has invalid cert
assert len(site_temp.content_manager.contents) == len(site.content_manager.contents) - 1
@ -72,7 +72,7 @@ class TestSiteDownload:
# Download normally
site_temp.addPeer("127.0.0.1", 1544)
site_temp.download(blind_includes=True).join(timeout=5)
bad_files = site_temp.storage.verifyFiles(quick_check=True)
bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"]
assert not bad_files
assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" in site_temp.content_manager.contents