Merge 2078a48a27 into 454c0b2e7e

.github/workflows/tests.yml

@@ -28,6 +28,7 @@ jobs:
     - name: Install
       run: |
         python3 -m pip install --upgrade -r requirements.txt
+        for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do python3 -m pip install --upgrade -r ${PLUGIN}; done
         python3 -m pip list
 
     - name: Prepare for tests

.gitlab-ci.yml

@@ -9,6 +9,7 @@ stages:
     - pip install --upgrade requests>=2.22.0
     - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
     - pip install --upgrade -r requirements.txt
+    - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done
   script:
     - pip list
     - openssl version -a

.travis.yml

@@ -15,6 +15,7 @@ before_install:
   # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet
 install:
  - pip install --upgrade -r requirements.txt
+ - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done
  - pip list
 before_script:
   - openssl version -a

Dockerfile

@@ -1,33 +1,27 @@
 FROM alpine:3.11 
 
-#Base settings
+# Base settings
 ENV HOME /root
+WORKDIR /root
 
+# Install and configure dependencies
 COPY requirements.txt /root/requirements.txt
-
-#Install ZeroNet
 RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \
- && pip3 install -r /root/requirements.txt \
+ && pip3 install -r requirements.txt \
+ && for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip3 install -r ${PLUGIN}; done \
  && apk del python3-dev gcc libffi-dev musl-dev make \
  && echo "ControlPort 9051" >> /etc/tor/torrc \
  && echo "CookieAuthentication 1" >> /etc/tor/torrc
- 
-RUN python3 -V \
- && python3 -m pip list \
- && tor --version \
- && openssl version
 
-#Add Zeronet source
+# Add ZeroNet source
 COPY . /root
 VOLUME /root/data
 
-#Control if Tor proxy is started
+# Control if Tor proxy is started
 ENV ENABLE_TOR false
 
-WORKDIR /root
-
-#Set upstart command
+# Set upstart command
 CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
 
-#Expose ports
+# Expose ports
 EXPOSE 43110 26552

Vagrantfile

@@ -5,41 +5,43 @@ VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  #Set box
-  config.vm.box = "ubuntu/trusty64"
+  # Set box
+  config.vm.box = "ubuntu/bionic64"
 
-  #Do not check fo updates
+  # Do not check fo updates
   config.vm.box_check_update = false
 
-  #Add private network
+  # Add private network
   config.vm.network "private_network", type: "dhcp"
 
-  #Redirect ports
+  # Redirect ports
   config.vm.network "forwarded_port", guest: 43110, host: 43110
   config.vm.network "forwarded_port", guest: 15441, host: 15441
 
-  #Sync folder using NFS if not windows
+  # Sync folder using NFS if not windows
   config.vm.synced_folder ".", "/vagrant",
       :nfs => !Vagrant::Util::Platform.windows?
 
-  #Virtal Box settings
+  # Virtal Box settings
   config.vm.provider "virtualbox" do |vb|
     # Don't boot with headless mode
-    #vb.gui = true
+    vb.gui = false
 
     # Set VM settings
     vb.customize ["modifyvm", :id, "--memory", "512"]
     vb.customize ["modifyvm", :id, "--cpus", 1]
   end
 
-  #Update system
+  # Update system
   config.vm.provision "shell",
       inline: "sudo apt-get update -y && sudo apt-get upgrade -y"
 
-  #Install deps
+  # Install dependencies
   config.vm.provision "shell",
-      inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y"
+      inline: "sudo apt-get install python3 python3-pip python3-dev gcc libffi-dev musl-dev make -y"
   config.vm.provision "shell",
-      inline: "sudo pip install msgpack --upgrade"
+      inline: "sudo pip3 install -r /vagrant/requirements.txt"
+  config.vm.provision "shell",
+      inline: "for PLUGIN in $(ls /vagrant/plugins/[^disabled-]*/requirements.txt); do sudo pip3 install -r ${PLUGIN}; done"
 
 end

plugins/DNS/ConfigPlugin.py

@@ -0,0 +1,19 @@
+from Plugin import PluginManager
+
+@PluginManager.registerTo('ConfigPlugin')
+class ConfigPlugin:
+    def createArguments(self):
+        nameservers = [
+            'https://doh.libredns.gr/dns-query',
+
+            'https://doh-de.blahdns.com/dns-query',
+            'https://doh-jp.blahdns.com/dns-query',
+            'https://doh-ch.blahdns.com/dns-query'
+        ]
+
+        group = self.parser.add_argument_group('DNS plugin')
+
+        group.add_argument('--dns_nameservers', help='Nameservers for DNS plugin', default=nameservers, metavar='address', nargs='*')
+        group.add_argument('--dns_configure', help='Configure resolver with system config for DNS plugin', action='store_true', default=False)
+
+        return super(ConfigPlugin, self).createArguments()

plugins/DNS/DNSResolver.py

@@ -0,0 +1,104 @@
+from Config import config
+
+import logging
+import time
+import json
+import re
+import os
+
+log = logging.getLogger('DNSPlugin')
+
+class DNSResolver:
+    loaded = False
+    cache = {}
+
+    def __init__(self, site_manager, nameservers, configure):
+        self.site_manager = site_manager
+        self.nameservers = nameservers
+        self.configure = configure
+
+    def load(self):
+        if not self.loaded:
+            self.loadModule()
+            self.loadCache()
+
+            self.resolver = dns.resolver.Resolver(configure=self.configure)
+
+            if not self.configure:
+                self.resolver.nameservers = self.nameservers
+
+            self.loaded = True
+
+    def loadModule(self):
+        global dns, dnslink
+        import dns.resolver
+        import dnslink
+
+        if config.tor == 'always':
+            class Response:
+                flags = dns.flags.TC
+
+            query = lambda *x, **y: Response()
+            dns.query.udp = query
+
+    def loadCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')):
+        if os.path.isfile(path):
+            try:
+                self.cache = json.load(open(path))
+            except json.decoder.JSONDecodeError:
+                pass
+
+    def saveCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')):
+        with open(path, 'w') as file:
+            json.dump(self.cache, file, indent=2)
+
+    def isDomain(self, address):
+        return re.match(r'(.*?)([A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$', address)
+
+    def resolveDomain(self, domain):
+        if not self.loaded:
+            self.load()
+
+        domain = domain.lower()
+
+        cache_entry = self.cache.get(domain)
+        if cache_entry and time.time() < cache_entry['timeout']:
+            log.info('cache: %s -> %s', domain, cache_entry['address'])
+            return cache_entry['address']
+
+        try:
+            resolver_record = dnslink.resolve(domain, protocol='zeronet', resolver=self.resolver)[0]
+            resolver_entry = {'domain': domain, 'address': resolver_record.split('/', 2)[2]}
+        except IndexError:
+            resolver_entry = None
+
+        resolver_error = None
+        try:
+            self.resolver.query(domain, 'TXT')
+        except dns.resolver.NXDOMAIN:
+            pass
+        except dns.resolver.NoAnswer:
+            pass
+        except dns.exception.DNSException as err:
+            resolver_error = err
+
+        if resolver_entry and not resolver_error:
+            log.info('resolver: %s -> %s', domain, resolver_entry['address'])
+            self.saveInCache(resolver_entry)
+            return resolver_entry['address']
+
+        if cache_entry and resolver_error:
+            log.info('fallback: %s -> %s', domain, cache_entry['address'])
+            self.extendInCache(cache_entry)
+            return cache_entry['address']
+
+    def saveInCache(self, entry):
+        entry['timeout'] = time.time() + 60 * 60
+        self.cache[entry['domain']] = entry
+
+        self.saveCache()
+
+    def extendInCache(self, entry):
+        self.cache[entry['domain']]['timeout'] = time.time() + 60 * 15
+
+        self.saveCache()

plugins/DNS/SiteManagerPlugin.py

@@ -0,0 +1,34 @@
+from Config import config
+from Plugin import PluginManager
+
+from .DNSResolver import DNSResolver
+
+allow_reload = False
+
+@PluginManager.registerTo('SiteManager')
+class SiteManagerPlugin:
+    _dns_resolver = None
+
+    @property
+    def dns_resolver(self):
+        if not self._dns_resolver:
+            nameservers = config.dns_nameservers
+            configure = config.dns_configure
+
+            self._dns_resolver = DNSResolver(
+                site_manager=self,
+                nameservers=nameservers,
+                configure=configure
+            )
+
+        return self._dns_resolver
+
+    def load(self, *args, **kwargs):
+        super(SiteManagerPlugin, self).load(*args, **kwargs)
+        self.dns_resolver.load()
+
+    def isDomain(self, address):
+        return self.dns_resolver.isDomain(address) or super(SiteManagerPlugin, self).isDomain(address)
+
+    def resolveDomain(self, domain):
+        return self.dns_resolver.resolveDomain(domain) or super(SiteManagerPlugin, self).resolveDomain(domain)

plugins/DNS/__init__.py

@@ -0,0 +1,16 @@
+from pkg_resources import DistributionNotFound, VersionConflict
+import pkg_resources
+import sys
+import os
+
+
+directory = os.path.dirname(__file__)
+
+try:
+    pkg_resources.require(open(directory + '/requirements.txt'))
+except (DistributionNotFound, VersionConflict):
+    sys.path.append(directory + '/requirements.zip')
+
+
+from . import ConfigPlugin
+from . import SiteManagerPlugin

plugins/DNS/plugin_info.json

@@ -0,0 +1,5 @@
+{
+	"name": "DNS",
+	"description": "Support classic DNS as domain system.",
+	"default": "enabled"
+}

plugins/DNS/requirements.txt

@@ -0,0 +1,2 @@
+dnspython @ git+https://github.com/rthalley/dnspython@ecd040a5b94dd824b658508fb7b429cde7bfe361
+dnslink>=1.0.2,<2.0.0

plugins/Zeroname/plugin_info.json

@@ -0,0 +1,5 @@
+{
+	"name": "Zeroname",
+	"description": "Support Namecoin as domain system.",
+	"default": "enabled"
+}

src/Config.py

@@ -241,7 +241,8 @@ class Config(object):
         self.parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port')
         self.parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*')
         self.parser.add_argument('--ui_host', help='Allow access using this hosts', metavar='host', nargs='*')
-        self.parser.add_argument('--ui_trans_proxy', help='Allow access using a transparent proxy', action='store_true')
+
+        self.parser.add_argument('--ws_server_url', help='Custom WS server URL for proxy requests', metavar='url')
 
         self.parser.add_argument('--open_browser', help='Open homepage in web browser automatically',
                                  nargs='?', const="default_browser", metavar='browser_name')

src/Ui/UiRequest.py

@@ -85,11 +85,8 @@ class UiRequest(object):
             self.learnHost(host)
             return True
 
-        if self.isProxyRequest():  # Support for chrome extension proxy
-            if self.isDomain(host):
-                return True
-            else:
-                return False
+        if self.isProxyRequest():  # Support for chrome extension proxy and DNS
+            return bool(self.isDomain(host.rsplit(":", 1)[0]))
 
         return False
 
@@ -125,8 +122,9 @@ class UiRequest(object):
             return iter([ret_error, ret_body])
 
         # Prepend .bit host for transparent proxy
-        if self.isDomain(self.env.get("HTTP_HOST")):
-            path = re.sub("^/", "/" + self.env.get("HTTP_HOST") + "/", path)
+        hostname = self.env.get("HTTP_HOST").split(":")[0]
+        if self.isDomain(hostname) and not helper.isIp(hostname) and not self.isUiHostRequest():
+            path = re.sub("^/", "/" + hostname + "/", path)
         path = re.sub("^http://zero[/]+", "/", path)  # Remove begining http://zero/ for chrome extension
         path = re.sub("^http://", "/", path)  # Remove begining http for chrome extension .bit access
 
@@ -202,7 +200,17 @@ class UiRequest(object):
 
     # The request is proxied by chrome extension or a transparent proxy
     def isProxyRequest(self):
-        return self.env["PATH_INFO"].startswith("http://") or (self.server.allow_trans_proxy and self.isDomain(self.env.get("HTTP_HOST")))
+        hostname = self.env.get("HTTP_HOST").rsplit(":", 1)[0]
+        if helper.isIp(hostname) or self.isUiHostRequest():
+            return False
+        return self.env["PATH_INFO"].startswith("http://") or self.isDomain(hostname)
+
+    def isUiHostRequest(self):
+        if not config.ui_host:
+            return False
+
+        hostname = self.env.get("HTTP_HOST").rsplit(":", 1)[0]
+        return self.env.get("HTTP_HOST") in config.ui_host or hostname in config.ui_host
 
     def isWebSocketRequest(self):
         return self.env.get("HTTP_UPGRADE") == "websocket"
@@ -433,21 +441,17 @@ class UiRequest(object):
         else:  # Bad url
             return False
 
-    def getSiteUrl(self, address):
-        if self.isProxyRequest():
-            return "http://zero/" + address
-        else:
-            return "/" + address
-
     def getWsServerUrl(self):
-        if self.isProxyRequest():
-            if self.env["REMOTE_ADDR"] == "127.0.0.1":  # Local client, the server address also should be 127.0.0.1
-                server_url = "http://127.0.0.1:%s" % self.env["SERVER_PORT"]
-            else:  # Remote client, use SERVER_NAME as server's real address
-                server_url = "http://%s:%s" % (self.env["SERVER_NAME"], self.env["SERVER_PORT"])
-        else:
-            server_url = ""
-        return server_url
+        if not self.isProxyRequest():  # Not a proxy request, use current server's URL
+            return ""
+
+        if self.env["REMOTE_ADDR"] == "127.0.0.1":  # Local client, the server address also should be 127.0.0.1
+            return "http://127.0.0.1:%s" % self.env["SERVER_PORT"]
+
+        if config.ws_server_url:  # Custom WS server URL set by user, use it
+            return config.ws_server_url
+
+        return "http://%s:%s" % (self.env["SERVER_NAME"], self.env["SERVER_PORT"])  # Remote client, use SERVER_NAME to guess server's real address
 
     def processQueryString(self, site, query_string):
         match = re.search("zeronet_peers=(.*?)(&|$)", query_string)
@@ -475,15 +479,14 @@ class UiRequest(object):
         address = re.sub("/.*", "", path.lstrip("/"))
         if self.isProxyRequest() and (not path or "/" in path[1:]):
             if self.env["HTTP_HOST"] == "zero":
-                root_url = "/" + address + "/"
-                file_url = "/" + address + "/" + inner_path
+                file_url = "/%s/%s" % (address, inner_path)
+                root_url = "/%s/" % (address,)
             else:
-                file_url = "/" + inner_path
+                file_url = "/%s" % (inner_path,)
                 root_url = "/"
-
         else:
-            file_url = "/" + address + "/" + inner_path
-            root_url = "/" + address + "/"
+            file_url = "/%s/%s" % (address, inner_path)
+            root_url = "/%s/" % (address,)
 
         if self.isProxyRequest():
             self.server.allowed_ws_origins.add(self.env["HTTP_HOST"])

src/Ui/UiServer.py

@@ -74,7 +74,6 @@ class UiServer:
         else:
             self.allowed_hosts = set([])
         self.allowed_ws_origins = set()
-        self.allow_trans_proxy = config.ui_trans_proxy
 
         self.wrapper_nonces = []
         self.add_nonces = []

src/Ui/media/Wrapper.coffee

@@ -1,5 +1,5 @@
 class Wrapper
-	constructor: (ws_url) ->
+	constructor: (ws_url, server_url) ->
 		@log "Created!"
 
 		@loading = new Loading(@)
@@ -21,6 +21,8 @@ class Wrapper
 		@ws.connect()
 		@ws_error = null # Ws error message
 
+		@server_url = server_url
+
 		@next_cmd_message_id = -1
 
 		@site_info = null # Hold latest site info
@@ -201,6 +203,8 @@ class Wrapper
 			@actionWebNotification(message)
 		else if cmd == "wrapperCloseWebNotification"
 			@actionCloseWebNotification(message)
+		else if cmd == "wrapperInfo"
+			@actionWrapperInfo(message)
 		else # Send to websocket
 			if message.id < 1000000
 				if message.cmd == "fileWrite" and not @modified_panel_updater_timer and site_info?.settings?.own
@@ -450,6 +454,12 @@ class Wrapper
 			@sendInner {"cmd": "response", "to": message.id, "result": back}
 
 
+	actionWrapperInfo: (message) ->
+		info = {
+			"server_url": @server_url
+		}
+		@sendInner {"cmd": "response", "to": message.id, "result": info}
+
 	# EOF actions
 
 
@@ -710,5 +720,5 @@ else
 
 ws_url = proto.ws + ":" + origin.replace(proto.http+":", "") + "/ZeroNet-Internal/Websocket?wrapper_key=" + window.wrapper_key
 
-window.wrapper = new Wrapper(ws_url)
+window.wrapper = new Wrapper(ws_url, origin)
 

src/Ui/media/all.js

@@ -931,7 +931,7 @@ $.extend( $.easing,
     slice = [].slice;
 
   Wrapper = (function() {
-    function Wrapper(ws_url) {
+    function Wrapper(ws_url, server_url) {
       this.reloadIframe = bind(this.reloadIframe, this);
       this.setSizeLimit = bind(this.setSizeLimit, this);
       this.updateModifiedPanel = bind(this.updateModifiedPanel, this);
@@ -971,6 +971,7 @@ $.extend( $.easing,
       this.ws.onMessage = this.onMessageWebsocket;
       this.ws.connect();
       this.ws_error = null;
+      this.server_url = server_url;
       this.next_cmd_message_id = -1;
       this.site_info = null;
       this.server_info = null;
@@ -1197,6 +1198,8 @@ $.extend( $.easing,
         return this.actionWebNotification(message);
       } else if (cmd === "wrapperCloseWebNotification") {
         return this.actionCloseWebNotification(message);
+      } else if (cmd === "wrapperInfo") {
+        return this.actionWrapperInfo(message);
       } else {
         if (message.id < 1000000) {
           if (message.cmd === "fileWrite" && !this.modified_panel_updater_timer && (typeof site_info !== "undefined" && site_info !== null ? (ref = site_info.settings) != null ? ref.own : void 0 : void 0)) {
@@ -1432,9 +1435,11 @@ $.extend( $.easing,
     Wrapper.prototype.displayPrompt = function(message, type, caption, placeholder, cb) {
       var body, button, input;
       body = $("<span class='message'></span>").html(message);
-      if (placeholder == null) {
+            if (placeholder != null) {
+        placeholder;
+      } else {
         placeholder = "";
-      }
+      };
       input = $("<input/>", {
         type: type,
         "class": "input button-" + type,
@@ -1624,6 +1629,18 @@ $.extend( $.easing,
       })(this));
     };
 
+    Wrapper.prototype.actionWrapperInfo = function(message) {
+      var info;
+      info = {
+        "server_url": this.server_url
+      };
+      return this.sendInner({
+        "cmd": "response",
+        "to": message.id,
+        "result": info
+      });
+    };
+
     Wrapper.prototype.onOpenWebsocket = function(e) {
       if (window.show_loadingscreen) {
         this.ws.cmd("channelJoin", {
@@ -2007,7 +2024,7 @@ $.extend( $.easing,
 
   ws_url = proto.ws + ":" + origin.replace(proto.http + ":", "") + "/ZeroNet-Internal/Websocket?wrapper_key=" + window.wrapper_key;
 
-  window.wrapper = new Wrapper(ws_url);
+  window.wrapper = new Wrapper(ws_url, origin);
 
 }).call(this);
 
@@ -2122,4 +2139,4 @@ $.extend( $.easing,
     }
   });
 
-}).call(this);
+}).call(this);