From ed1cdf04188d1727c3823e22411c41976c6e58d8 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 18 May 2018 16:43:07 +0300 Subject: [PATCH] Custom status bootnodes (#968) This change makes invalidation mechanism more aggressive. With a primary goal to invalidate short living nodes faster. In current setup any node that became known in terms of discovery will stay in this state until it will fail to respond to 5 queries. Removing them earlier from a table allows to reduce latency for finding required nodes. The second change, one adds a version for discovery, separates status dht from ethereum dht. After we rolled out discovery it became obvious that our boot nodes became spammed with irrelevant nodes. And this made discovery process very long, for example with separate dht discovery takes ~2s, with mutual dht - it can take 1m-10m and there is still no guarantee to find a max amount of peers, cause status nodes is a very small part of whole ethereum infra. In my understanding, we don't need to be a part of ethereum dht, and lower latency is way more important for us. Closes: #941 Partially closes: #960 (960 requires futher investigations on devices) --- Makefile | 9 +++ _assets/build/Dockerfile | 2 +- _assets/build/Dockerfile-bootnode | 16 +++++ .../geth/0030-discovery-status-dht.patch | 64 +++++++++++++++++++ cmd/bootnode/main.go | 51 +++++++++++++++ .../ethereum/go-ethereum/p2p/discv5/net.go | 8 ++- .../ethereum/go-ethereum/p2p/discv5/table.go | 7 +- .../ethereum/go-ethereum/p2p/discv5/udp.go | 2 +- 8 files changed, 154 insertions(+), 5 deletions(-) create mode 100644 _assets/build/Dockerfile-bootnode create mode 100644 _assets/patches/geth/0030-discovery-status-dht.patch create mode 100644 cmd/bootnode/main.go diff --git a/Makefile b/Makefile index 0d5c0557a..160a53bbe 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ networkid ?= StatusChain gotest_extraflags = DOCKER_IMAGE_NAME ?= statusteam/status-go +BOOTNODE_IMAGE_NAME ?= statusteam/bootnode DOCKER_TEST_WORKDIR = /go/src/github.com/status-im/status-go/ DOCKER_TEST_IMAGE = golang:1.10 @@ -56,6 +57,10 @@ statusgo: ##@build Build status-go as statusd server @echo "Compilation done." @echo "Run \"build/bin/statusd -h\" to view available commands." +bootnode: ##@build Build discovery v5 bootnode using status-go deps + go build -i -o $(GOBIN)/bootnode -v -tags '$(BUILD_TAGS)' $(BUILD_FLAGS) ./cmd/bootnode/ + @echo "Compilation done." + statusgo-cross: statusgo-android statusgo-ios @echo "Full cross compilation done." @ls -ld $(GOBIN)/statusgo-* @@ -90,6 +95,10 @@ docker-image: ##@docker Build docker image (use DOCKER_IMAGE_NAME to set the ima @echo "Building docker image..." docker build --file _assets/build/Dockerfile --build-arg "build_tags=$(BUILD_TAGS)" . -t $(DOCKER_IMAGE_NAME):latest +bootnode-image: + @echo "Building docker image for bootnode..." + docker build --file _assets/build/Dockerfile-bootnode . -t $(BOOTNODE_IMAGE_NAME):latest + docker-image-tag: ##@docker Tag DOCKER_IMAGE_NAME:latest with a tag following pattern $GIT_SHA[:8]-$BUILD_TAGS @echo "Tagging docker image..." docker tag $(DOCKER_IMAGE_NAME):latest $(DOCKER_IMAGE_NAME):$(shell BUILD_TAGS="$(BUILD_TAGS)" ./_assets/ci/get-docker-image-tag.sh) diff --git a/_assets/build/Dockerfile b/_assets/build/Dockerfile index 6755f2503..8844b7ad3 100644 --- a/_assets/build/Dockerfile +++ b/_assets/build/Dockerfile @@ -14,7 +14,7 @@ FROM alpine:latest RUN apk add --no-cache ca-certificates bash -COPY --from=builder /go/src/github.com/status-im/status-go/build/bin/* /usr/local/bin/ +COPY --from=builder /go/src/github.com/status-im/status-go/build/bin/statusd /usr/local/bin/ RUN mkdir -p /static/keys COPY --from=builder /go/src/github.com/status-im/status-go/static/keys/* /static/keys/ diff --git a/_assets/build/Dockerfile-bootnode b/_assets/build/Dockerfile-bootnode new file mode 100644 index 000000000..1d5bbca00 --- /dev/null +++ b/_assets/build/Dockerfile-bootnode @@ -0,0 +1,16 @@ +FROM golang:1.10-alpine as builder + +ARG build_tags + +RUN apk add --no-cache make gcc musl-dev linux-headers + +RUN mkdir -p /go/src/github.com/status-im/status-go +ADD . /go/src/github.com/status-im/status-go +RUN cd /go/src/github.com/status-im/status-go && make bootnode BUILD_TAGS="$build_tags" + +FROM alpine:latest + +RUN apk add --no-cache ca-certificates bash + +COPY --from=builder /go/src/github.com/status-im/status-go/build/bin/bootnode /usr/local/bin/ +ENTRYPOINT /usr/local/bin/bootnode \ No newline at end of file diff --git a/_assets/patches/geth/0030-discovery-status-dht.patch b/_assets/patches/geth/0030-discovery-status-dht.patch new file mode 100644 index 000000000..0dd4bb8fb --- /dev/null +++ b/_assets/patches/geth/0030-discovery-status-dht.patch @@ -0,0 +1,64 @@ +diff --git c/p2p/discv5/net.go w/p2p/discv5/net.go +index 9b0bd0c80..d0eae28f9 100644 +--- c/p2p/discv5/net.go ++++ w/p2p/discv5/net.go +@@ -40,7 +40,7 @@ var ( + + const ( + autoRefreshInterval = 1 * time.Hour +- bucketRefreshInterval = 1 * time.Minute ++ bucketRefreshInterval = 10 * time.Second + seedCount = 30 + seedMaxAge = 5 * 24 * time.Hour + lowPort = 1024 +@@ -1055,7 +1055,11 @@ func (net *Network) handle(n *Node, ev nodeEvent, pkt *ingressPacket) error { + func (net *Network) checkPacket(n *Node, ev nodeEvent, pkt *ingressPacket) error { + // Replay prevention checks. + switch ev { +- case pingPacket, findnodeHashPacket, neighborsPacket: ++ case pingPacket: ++ if pkt.data.(*ping).Version != Version { ++ return fmt.Errorf("version mismatch") ++ } ++ case findnodeHashPacket, neighborsPacket: + // TODO: check date is > last date seen + // TODO: check ping version + case pongPacket: +diff --git c/p2p/discv5/table.go w/p2p/discv5/table.go +index c8d234b93..42311e1db 100644 +--- c/p2p/discv5/table.go ++++ w/p2p/discv5/table.go +@@ -38,7 +38,7 @@ const ( + hashBits = len(common.Hash{}) * 8 + nBuckets = hashBits + 1 // Number of buckets + +- maxFindnodeFailures = 5 ++ maxFindnodeFailures = 1 + ) + + type Table struct { +@@ -177,6 +177,11 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { + close := &nodesByDistance{target: target} + for _, b := range tab.buckets { + for _, n := range b.entries { ++ // node can be in table only in two states ++ // known and contested ++ if n.state != known { ++ continue ++ } + close.push(n, nresults) + } + } +diff --git c/p2p/discv5/udp.go w/p2p/discv5/udp.go +index 49e1cb811..0ead22753 100644 +--- c/p2p/discv5/udp.go ++++ w/p2p/discv5/udp.go +@@ -32,7 +32,7 @@ import ( + "github.com/ethereum/go-ethereum/rlp" + ) + +-const Version = 4 ++const Version = 55 + + // Errors + var ( diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go new file mode 100644 index 000000000..37726c7d1 --- /dev/null +++ b/cmd/bootnode/main.go @@ -0,0 +1,51 @@ +// bootnode runs a bootstrap node for the Ethereum Discovery Protocol. +package main + +import ( + "flag" + "net" + "os" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discv5" +) + +func main() { + var ( + listenAddr = flag.String("addr", ":30301", "listen address") + nodeKeyFile = flag.String("nodekey", "", "private key filename") + verbosity = flag.Int("verbosity", int(log.LvlInfo), "log verbosity (0-9)") + vmodule = flag.String("vmodule", "", "log verbosity pattern") + ) + flag.Parse() + + glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) + glogger.Verbosity(log.Lvl(*verbosity)) + if err := glogger.Vmodule(*vmodule); err != nil { + log.Crit("Failed to set glog verbosity", "value", *vmodule, "err", err) + } + log.Root().SetHandler(glogger) + + nodeKey, err := crypto.LoadECDSA(*nodeKeyFile) + if err != nil { + log.Crit("Failed to load ecdsa key from", "file", *nodeKeyFile, "error", err) + } + + addr, err := net.ResolveUDPAddr("udp", *listenAddr) + if err != nil { + log.Crit("Unable to resolve UDP", "address", *listenAddr, "error", err) + } + conn, err := net.ListenUDP("udp", addr) + if err != nil { + log.Crit("Unable to listen on udp", "address", addr, "error", err) + } + + realaddr := conn.LocalAddr().(*net.UDPAddr) + tab, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", nil) + if err != nil { + log.Crit("Failed to create discovery v5 table:", "error", err) + } + defer tab.Close() + select {} +} diff --git a/vendor/github.com/ethereum/go-ethereum/p2p/discv5/net.go b/vendor/github.com/ethereum/go-ethereum/p2p/discv5/net.go index 9b0bd0c80..d0eae28f9 100644 --- a/vendor/github.com/ethereum/go-ethereum/p2p/discv5/net.go +++ b/vendor/github.com/ethereum/go-ethereum/p2p/discv5/net.go @@ -40,7 +40,7 @@ var ( const ( autoRefreshInterval = 1 * time.Hour - bucketRefreshInterval = 1 * time.Minute + bucketRefreshInterval = 10 * time.Second seedCount = 30 seedMaxAge = 5 * 24 * time.Hour lowPort = 1024 @@ -1055,7 +1055,11 @@ func (net *Network) handle(n *Node, ev nodeEvent, pkt *ingressPacket) error { func (net *Network) checkPacket(n *Node, ev nodeEvent, pkt *ingressPacket) error { // Replay prevention checks. switch ev { - case pingPacket, findnodeHashPacket, neighborsPacket: + case pingPacket: + if pkt.data.(*ping).Version != Version { + return fmt.Errorf("version mismatch") + } + case findnodeHashPacket, neighborsPacket: // TODO: check date is > last date seen // TODO: check ping version case pongPacket: diff --git a/vendor/github.com/ethereum/go-ethereum/p2p/discv5/table.go b/vendor/github.com/ethereum/go-ethereum/p2p/discv5/table.go index c8d234b93..42311e1db 100644 --- a/vendor/github.com/ethereum/go-ethereum/p2p/discv5/table.go +++ b/vendor/github.com/ethereum/go-ethereum/p2p/discv5/table.go @@ -38,7 +38,7 @@ const ( hashBits = len(common.Hash{}) * 8 nBuckets = hashBits + 1 // Number of buckets - maxFindnodeFailures = 5 + maxFindnodeFailures = 1 ) type Table struct { @@ -177,6 +177,11 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { close := &nodesByDistance{target: target} for _, b := range tab.buckets { for _, n := range b.entries { + // node can be in table only in two states + // known and contested + if n.state != known { + continue + } close.push(n, nresults) } } diff --git a/vendor/github.com/ethereum/go-ethereum/p2p/discv5/udp.go b/vendor/github.com/ethereum/go-ethereum/p2p/discv5/udp.go index 49e1cb811..0ead22753 100644 --- a/vendor/github.com/ethereum/go-ethereum/p2p/discv5/udp.go +++ b/vendor/github.com/ethereum/go-ethereum/p2p/discv5/udp.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -const Version = 4 +const Version = 55 // Errors var (