diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9c6f6f3b..b69f79cf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,7 @@ jobs: - name: Install run: | python3 -m pip install --upgrade -r requirements.txt + for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do python3 -m pip install --upgrade -r ${PLUGIN}; done python3 -m pip list - name: Prepare for tests diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3e1ed29..42290627 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ stages: - pip install --upgrade requests>=2.22.0 - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium - pip install --upgrade -r requirements.txt + - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done script: - pip list - openssl version -a diff --git a/.travis.yml b/.travis.yml index bdaafa22..feae5231 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ before_install: # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet install: - pip install --upgrade -r requirements.txt + - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done - pip list before_script: - openssl version -a diff --git a/Dockerfile b/Dockerfile index 7839cfa0..9ded0ad5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,27 @@ FROM alpine:3.11 -#Base settings +# Base settings ENV HOME /root +WORKDIR /root +# Install and configure dependencies COPY requirements.txt /root/requirements.txt - -#Install ZeroNet RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \ - && pip3 install -r /root/requirements.txt \ + && pip3 install -r requirements.txt \ + && for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip3 install -r ${PLUGIN}; done \ && apk del python3-dev gcc libffi-dev musl-dev make \ && echo "ControlPort 9051" >> /etc/tor/torrc \ && echo "CookieAuthentication 1" >> /etc/tor/torrc - -RUN python3 -V \ - && python3 -m pip list \ - && tor --version \ - && openssl version -#Add Zeronet source +# Add ZeroNet source COPY . /root VOLUME /root/data -#Control if Tor proxy is started +# Control if Tor proxy is started ENV ENABLE_TOR false -WORKDIR /root - -#Set upstart command +# Set upstart command CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 -#Expose ports +# Expose ports EXPOSE 43110 26552 diff --git a/Vagrantfile b/Vagrantfile index 24fe0c45..eb8ff3dd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,41 +5,43 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - #Set box - config.vm.box = "ubuntu/trusty64" + # Set box + config.vm.box = "ubuntu/bionic64" - #Do not check fo updates + # Do not check fo updates config.vm.box_check_update = false - #Add private network + # Add private network config.vm.network "private_network", type: "dhcp" - #Redirect ports + # Redirect ports config.vm.network "forwarded_port", guest: 43110, host: 43110 config.vm.network "forwarded_port", guest: 15441, host: 15441 - #Sync folder using NFS if not windows + # Sync folder using NFS if not windows config.vm.synced_folder ".", "/vagrant", :nfs => !Vagrant::Util::Platform.windows? - #Virtal Box settings + # Virtal Box settings config.vm.provider "virtualbox" do |vb| # Don't boot with headless mode - #vb.gui = true + vb.gui = false # Set VM settings vb.customize ["modifyvm", :id, "--memory", "512"] vb.customize ["modifyvm", :id, "--cpus", 1] end - #Update system + # Update system config.vm.provision "shell", inline: "sudo apt-get update -y && sudo apt-get upgrade -y" - #Install deps + # Install dependencies config.vm.provision "shell", - inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y" + inline: "sudo apt-get install python3 python3-pip python3-dev gcc libffi-dev musl-dev make -y" config.vm.provision "shell", - inline: "sudo pip install msgpack --upgrade" + inline: "sudo pip3 install -r /vagrant/requirements.txt" + config.vm.provision "shell", + inline: "for PLUGIN in $(ls /vagrant/plugins/[^disabled-]*/requirements.txt); do sudo pip3 install -r ${PLUGIN}; done" end diff --git a/plugins/DNS/ConfigPlugin.py b/plugins/DNS/ConfigPlugin.py new file mode 100644 index 00000000..477480b2 --- /dev/null +++ b/plugins/DNS/ConfigPlugin.py @@ -0,0 +1,19 @@ +from Plugin import PluginManager + +@PluginManager.registerTo('ConfigPlugin') +class ConfigPlugin: + def createArguments(self): + nameservers = [ + 'https://doh.libredns.gr/dns-query', + + 'https://doh-de.blahdns.com/dns-query', + 'https://doh-jp.blahdns.com/dns-query', + 'https://doh-ch.blahdns.com/dns-query' + ] + + group = self.parser.add_argument_group('DNS plugin') + + group.add_argument('--dns_nameservers', help='Nameservers for DNS plugin', default=nameservers, metavar='address', nargs='*') + group.add_argument('--dns_configure', help='Configure resolver with system config for DNS plugin', action='store_true', default=False) + + return super(ConfigPlugin, self).createArguments() diff --git a/plugins/DNS/DNSResolver.py b/plugins/DNS/DNSResolver.py new file mode 100644 index 00000000..cff85528 --- /dev/null +++ b/plugins/DNS/DNSResolver.py @@ -0,0 +1,104 @@ +from Config import config + +import logging +import time +import json +import re +import os + +log = logging.getLogger('DNSPlugin') + +class DNSResolver: + loaded = False + cache = {} + + def __init__(self, site_manager, nameservers, configure): + self.site_manager = site_manager + self.nameservers = nameservers + self.configure = configure + + def load(self): + if not self.loaded: + self.loadModule() + self.loadCache() + + self.resolver = dns.resolver.Resolver(configure=self.configure) + + if not self.configure: + self.resolver.nameservers = self.nameservers + + self.loaded = True + + def loadModule(self): + global dns, dnslink + import dns.resolver + import dnslink + + if config.tor == 'always': + class Response: + flags = dns.flags.TC + + query = lambda *x, **y: Response() + dns.query.udp = query + + def loadCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')): + if os.path.isfile(path): + try: + self.cache = json.load(open(path)) + except json.decoder.JSONDecodeError: + pass + + def saveCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')): + with open(path, 'w') as file: + json.dump(self.cache, file, indent=2) + + def isDomain(self, address): + return re.match(r'(.*?)([A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$', address) + + def resolveDomain(self, domain): + if not self.loaded: + self.load() + + domain = domain.lower() + + cache_entry = self.cache.get(domain) + if cache_entry and time.time() < cache_entry['timeout']: + log.info('cache: %s -> %s', domain, cache_entry['address']) + return cache_entry['address'] + + try: + resolver_record = dnslink.resolve(domain, protocol='zeronet', resolver=self.resolver)[0] + resolver_entry = {'domain': domain, 'address': resolver_record.split('/', 2)[2]} + except IndexError: + resolver_entry = None + + resolver_error = None + try: + self.resolver.query(domain, 'TXT') + except dns.resolver.NXDOMAIN: + pass + except dns.resolver.NoAnswer: + pass + except dns.exception.DNSException as err: + resolver_error = err + + if resolver_entry and not resolver_error: + log.info('resolver: %s -> %s', domain, resolver_entry['address']) + self.saveInCache(resolver_entry) + return resolver_entry['address'] + + if cache_entry and resolver_error: + log.info('fallback: %s -> %s', domain, cache_entry['address']) + self.extendInCache(cache_entry) + return cache_entry['address'] + + def saveInCache(self, entry): + entry['timeout'] = time.time() + 60 * 60 + self.cache[entry['domain']] = entry + + self.saveCache() + + def extendInCache(self, entry): + self.cache[entry['domain']]['timeout'] = time.time() + 60 * 15 + + self.saveCache() diff --git a/plugins/DNS/SiteManagerPlugin.py b/plugins/DNS/SiteManagerPlugin.py new file mode 100644 index 00000000..f6005690 --- /dev/null +++ b/plugins/DNS/SiteManagerPlugin.py @@ -0,0 +1,34 @@ +from Config import config +from Plugin import PluginManager + +from .DNSResolver import DNSResolver + +allow_reload = False + +@PluginManager.registerTo('SiteManager') +class SiteManagerPlugin: + _dns_resolver = None + + @property + def dns_resolver(self): + if not self._dns_resolver: + nameservers = config.dns_nameservers + configure = config.dns_configure + + self._dns_resolver = DNSResolver( + site_manager=self, + nameservers=nameservers, + configure=configure + ) + + return self._dns_resolver + + def load(self, *args, **kwargs): + super(SiteManagerPlugin, self).load(*args, **kwargs) + self.dns_resolver.load() + + def isDomain(self, address): + return self.dns_resolver.isDomain(address) or super(SiteManagerPlugin, self).isDomain(address) + + def resolveDomain(self, domain): + return self.dns_resolver.resolveDomain(domain) or super(SiteManagerPlugin, self).resolveDomain(domain) diff --git a/plugins/DNS/__init__.py b/plugins/DNS/__init__.py new file mode 100644 index 00000000..e453f59b --- /dev/null +++ b/plugins/DNS/__init__.py @@ -0,0 +1,16 @@ +from pkg_resources import DistributionNotFound, VersionConflict +import pkg_resources +import sys +import os + + +directory = os.path.dirname(__file__) + +try: + pkg_resources.require(open(directory + '/requirements.txt')) +except (DistributionNotFound, VersionConflict): + sys.path.append(directory + '/requirements.zip') + + +from . import ConfigPlugin +from . import SiteManagerPlugin diff --git a/plugins/DNS/plugin_info.json b/plugins/DNS/plugin_info.json new file mode 100644 index 00000000..01f507f4 --- /dev/null +++ b/plugins/DNS/plugin_info.json @@ -0,0 +1,5 @@ +{ + "name": "DNS", + "description": "Support classic DNS as domain system.", + "default": "enabled" +} diff --git a/plugins/DNS/requirements.txt b/plugins/DNS/requirements.txt new file mode 100644 index 00000000..39e42521 --- /dev/null +++ b/plugins/DNS/requirements.txt @@ -0,0 +1,2 @@ +dnspython @ git+https://github.com/rthalley/dnspython@ecd040a5b94dd824b658508fb7b429cde7bfe361 +dnslink>=1.0.2,<2.0.0 diff --git a/plugins/DNS/requirements.zip b/plugins/DNS/requirements.zip new file mode 100644 index 00000000..22fc8aa9 Binary files /dev/null and b/plugins/DNS/requirements.zip differ diff --git a/plugins/Zeroname/plugin_info.json b/plugins/Zeroname/plugin_info.json new file mode 100644 index 00000000..67ffd018 --- /dev/null +++ b/plugins/Zeroname/plugin_info.json @@ -0,0 +1,5 @@ +{ + "name": "Zeroname", + "description": "Support Namecoin as domain system.", + "default": "enabled" +} diff --git a/src/Config.py b/src/Config.py index 7095975b..4d397e8e 100644 --- a/src/Config.py +++ b/src/Config.py @@ -241,7 +241,8 @@ class Config(object): self.parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port') self.parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*') self.parser.add_argument('--ui_host', help='Allow access using this hosts', metavar='host', nargs='*') - self.parser.add_argument('--ui_trans_proxy', help='Allow access using a transparent proxy', action='store_true') + + self.parser.add_argument('--ws_server_url', help='Custom WS server URL for proxy requests', metavar='url') self.parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name') diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 8f00efcb..42253a3e 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -85,11 +85,8 @@ class UiRequest(object): self.learnHost(host) return True - if self.isProxyRequest(): # Support for chrome extension proxy - if self.isDomain(host): - return True - else: - return False + if self.isProxyRequest(): # Support for chrome extension proxy and DNS + return bool(self.isDomain(host.rsplit(":", 1)[0])) return False @@ -125,8 +122,9 @@ class UiRequest(object): return iter([ret_error, ret_body]) # Prepend .bit host for transparent proxy - if self.isDomain(self.env.get("HTTP_HOST")): - path = re.sub("^/", "/" + self.env.get("HTTP_HOST") + "/", path) + hostname = self.env.get("HTTP_HOST").split(":")[0] + if self.isDomain(hostname) and not helper.isIp(hostname) and not self.isUiHostRequest(): + path = re.sub("^/", "/" + hostname + "/", path) path = re.sub("^http://zero[/]+", "/", path) # Remove begining http://zero/ for chrome extension path = re.sub("^http://", "/", path) # Remove begining http for chrome extension .bit access @@ -202,7 +200,17 @@ class UiRequest(object): # The request is proxied by chrome extension or a transparent proxy def isProxyRequest(self): - return self.env["PATH_INFO"].startswith("http://") or (self.server.allow_trans_proxy and self.isDomain(self.env.get("HTTP_HOST"))) + hostname = self.env.get("HTTP_HOST").rsplit(":", 1)[0] + if helper.isIp(hostname) or self.isUiHostRequest(): + return False + return self.env["PATH_INFO"].startswith("http://") or self.isDomain(hostname) + + def isUiHostRequest(self): + if not config.ui_host: + return False + + hostname = self.env.get("HTTP_HOST").rsplit(":", 1)[0] + return self.env.get("HTTP_HOST") in config.ui_host or hostname in config.ui_host def isWebSocketRequest(self): return self.env.get("HTTP_UPGRADE") == "websocket" @@ -433,21 +441,17 @@ class UiRequest(object): else: # Bad url return False - def getSiteUrl(self, address): - if self.isProxyRequest(): - return "http://zero/" + address - else: - return "/" + address - def getWsServerUrl(self): - if self.isProxyRequest(): - if self.env["REMOTE_ADDR"] == "127.0.0.1": # Local client, the server address also should be 127.0.0.1 - server_url = "http://127.0.0.1:%s" % self.env["SERVER_PORT"] - else: # Remote client, use SERVER_NAME as server's real address - server_url = "http://%s:%s" % (self.env["SERVER_NAME"], self.env["SERVER_PORT"]) - else: - server_url = "" - return server_url + if not self.isProxyRequest(): # Not a proxy request, use current server's URL + return "" + + if self.env["REMOTE_ADDR"] == "127.0.0.1": # Local client, the server address also should be 127.0.0.1 + return "http://127.0.0.1:%s" % self.env["SERVER_PORT"] + + if config.ws_server_url: # Custom WS server URL set by user, use it + return config.ws_server_url + + return "http://%s:%s" % (self.env["SERVER_NAME"], self.env["SERVER_PORT"]) # Remote client, use SERVER_NAME to guess server's real address def processQueryString(self, site, query_string): match = re.search("zeronet_peers=(.*?)(&|$)", query_string) @@ -475,15 +479,14 @@ class UiRequest(object): address = re.sub("/.*", "", path.lstrip("/")) if self.isProxyRequest() and (not path or "/" in path[1:]): if self.env["HTTP_HOST"] == "zero": - root_url = "/" + address + "/" - file_url = "/" + address + "/" + inner_path + file_url = "/%s/%s" % (address, inner_path) + root_url = "/%s/" % (address,) else: - file_url = "/" + inner_path + file_url = "/%s" % (inner_path,) root_url = "/" - else: - file_url = "/" + address + "/" + inner_path - root_url = "/" + address + "/" + file_url = "/%s/%s" % (address, inner_path) + root_url = "/%s/" % (address,) if self.isProxyRequest(): self.server.allowed_ws_origins.add(self.env["HTTP_HOST"]) diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 9d93ccfd..6bb4cba2 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -74,7 +74,6 @@ class UiServer: else: self.allowed_hosts = set([]) self.allowed_ws_origins = set() - self.allow_trans_proxy = config.ui_trans_proxy self.wrapper_nonces = [] self.add_nonces = [] diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee index 1b98855e..da784be3 100644 --- a/src/Ui/media/Wrapper.coffee +++ b/src/Ui/media/Wrapper.coffee @@ -1,5 +1,5 @@ class Wrapper - constructor: (ws_url) -> + constructor: (ws_url, server_url) -> @log "Created!" @loading = new Loading(@) @@ -21,6 +21,8 @@ class Wrapper @ws.connect() @ws_error = null # Ws error message + @server_url = server_url + @next_cmd_message_id = -1 @site_info = null # Hold latest site info @@ -201,6 +203,8 @@ class Wrapper @actionWebNotification(message) else if cmd == "wrapperCloseWebNotification" @actionCloseWebNotification(message) + else if cmd == "wrapperInfo" + @actionWrapperInfo(message) else # Send to websocket if message.id < 1000000 if message.cmd == "fileWrite" and not @modified_panel_updater_timer and site_info?.settings?.own @@ -450,6 +454,12 @@ class Wrapper @sendInner {"cmd": "response", "to": message.id, "result": back} + actionWrapperInfo: (message) -> + info = { + "server_url": @server_url + } + @sendInner {"cmd": "response", "to": message.id, "result": info} + # EOF actions @@ -710,5 +720,5 @@ else ws_url = proto.ws + ":" + origin.replace(proto.http+":", "") + "/ZeroNet-Internal/Websocket?wrapper_key=" + window.wrapper_key -window.wrapper = new Wrapper(ws_url) +window.wrapper = new Wrapper(ws_url, origin) diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index f5ad947c..4985e527 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -931,7 +931,7 @@ $.extend( $.easing, slice = [].slice; Wrapper = (function() { - function Wrapper(ws_url) { + function Wrapper(ws_url, server_url) { this.reloadIframe = bind(this.reloadIframe, this); this.setSizeLimit = bind(this.setSizeLimit, this); this.updateModifiedPanel = bind(this.updateModifiedPanel, this); @@ -971,6 +971,7 @@ $.extend( $.easing, this.ws.onMessage = this.onMessageWebsocket; this.ws.connect(); this.ws_error = null; + this.server_url = server_url; this.next_cmd_message_id = -1; this.site_info = null; this.server_info = null; @@ -1197,6 +1198,8 @@ $.extend( $.easing, return this.actionWebNotification(message); } else if (cmd === "wrapperCloseWebNotification") { return this.actionCloseWebNotification(message); + } else if (cmd === "wrapperInfo") { + return this.actionWrapperInfo(message); } else { if (message.id < 1000000) { if (message.cmd === "fileWrite" && !this.modified_panel_updater_timer && (typeof site_info !== "undefined" && site_info !== null ? (ref = site_info.settings) != null ? ref.own : void 0 : void 0)) { @@ -1432,9 +1435,11 @@ $.extend( $.easing, Wrapper.prototype.displayPrompt = function(message, type, caption, placeholder, cb) { var body, button, input; body = $("").html(message); - if (placeholder == null) { + if (placeholder != null) { + placeholder; + } else { placeholder = ""; - } + }; input = $("", { type: type, "class": "input button-" + type, @@ -1624,6 +1629,18 @@ $.extend( $.easing, })(this)); }; + Wrapper.prototype.actionWrapperInfo = function(message) { + var info; + info = { + "server_url": this.server_url + }; + return this.sendInner({ + "cmd": "response", + "to": message.id, + "result": info + }); + }; + Wrapper.prototype.onOpenWebsocket = function(e) { if (window.show_loadingscreen) { this.ws.cmd("channelJoin", { @@ -2007,7 +2024,7 @@ $.extend( $.easing, ws_url = proto.ws + ":" + origin.replace(proto.http + ":", "") + "/ZeroNet-Internal/Websocket?wrapper_key=" + window.wrapper_key; - window.wrapper = new Wrapper(ws_url); + window.wrapper = new Wrapper(ws_url, origin); }).call(this); @@ -2122,4 +2139,4 @@ $.extend( $.easing, } }); -}).call(this); \ No newline at end of file +}).call(this);