k8s: Kubernetes - Part 2 of ?? (#401)

k8s/README.md

@@ -54,15 +54,80 @@ cat /tmp/genesis.json | json-minify | base64
 
 ### Deploying Geth Mainchain
 
+First, launch the bootnode so that geth nodes can discover each other.
+
+```bash
+bazel run //k8s/geth:bootnode.deploy.apply
+```
+
+Then launch everything else.
+
 ```bash
 bazel run //k8s:everything.apply
 ```
 
 This creates a few nodes and one miner with CPU restrictions. After ~30 
-minutes, the miner has generated the DAG and begins mining. This seems iterate
-over 2 DAG epochs dispite the flags set for 1 DAG in memory and in disk. 
+minutes, the miner has generated the DAG and begins mining. The miners have a
+stateful volume for their DAGs so that they do not have to regenerate them on
+restart. 
 
-Note: This can be improved by giving the miner more CPU.
+Note: DAG generation time can be improved by giving the miner more CPU in the 
+deployment yaml.
+
+### Bootstrapping the Beacon Chain
+
+TODO: This process is currently manual and needs to be improved!
+
+Using the private key above and the deployVRC tool, deploy the validator
+registration contract.
+
+```bash
+# get the address the node service
+minikube service geth-nodes --url
+```
+
+Example response:
+
+```bash
+http://192.168.99.100:30051
+http://192.168.99.100:31745
+```
+
+Using the first port provided (RPC). Run the deploy VRC tool
+
+```
+bazel run //contracts/validator-registration-contract/deployVRC --\
+  --privKey=783da8ef5343c3019748506305d400bca8c324a5819f3a7f7fbf0c0a0d799b09 \
+  --httpPath=http://192.168.99.100:30051
+```
+
+Example output:
+
+```
+INFO main: New contract deployed address=0x541AfaC5266c534de039B4A1a53519e76ea82846
+```
+
+Set this value for the vrcaddr flag in 
+k8s/beacon-chain/beacon-chain.deploy.yaml.
+
+Ensure that the beacon-chain and client docker images are up to date. 
+
+```bash
+bazel run //beacon-chain:push_image
+bazel run //client:push_image
+```
+
+Start the beacon chain nodes
+
+```bash
+bazel run //k8s/beacon-chain:everything.apply
+```
+
+Start the clients
+
+```bash
+bazel run //k8s/client:everything.apply
+```
 
 ### Accessing Geth Services
 
@@ -89,4 +154,5 @@ So we can use these values locally to connect to our local cluster.
 
 ```bash
 bazel run //beacon-chain -- --web3provider=ws://192.168.99.100:32164
-```
+```
+

k8s/beacon-chain/BUILD.bazel

@@ -0,0 +1,41 @@
+package(default_visibility = ["//k8s:__subpackages__"])
+
+load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects")
+load("@k8s_deploy//:defaults.bzl", "k8s_deploy")
+load("@k8s_service//:defaults.bzl", "k8s_service")
+
+k8s_objects(
+    name = "everything",
+    objects = [
+        ":deployments",
+        ":services",
+    ],
+)
+
+_deployments = [
+    "beacon-chain",
+]
+
+_services = [
+    "beacon-chain",
+]
+
+k8s_objects(
+    name = "deployments",
+    objects = [":" + name + ".deploy" for name in _deployments],
+)
+
+[k8s_deploy(
+    name = name + ".deploy",
+    template = name + ".deploy.yaml",
+) for name in _deployments]
+
+k8s_objects(
+    name = "services",
+    objects = [":" + name + ".service" for name in _services],
+)
+
+[k8s_service(
+    name = name + ".service",
+    template = name + ".service.yaml",
+) for name in _services]

k8s/beacon-chain/beacon-chain.deploy.yaml

@@ -0,0 +1,36 @@
+kind: Deployment
+apiVersion: apps/v1beta1
+metadata:
+  name: beacon-chain
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      component: beacon-chain
+      universe: beacon-chain
+  template:
+    metadata:
+      labels:
+        component: beacon-chain
+        universe: beacon-chain
+    spec:
+      priorityClassName: production-priority
+      containers:
+        - name: beacon-chain
+          image: gcr.io/prysmaticlabs/prysm/beacon-chain:latest
+          args: 
+            - "--web3provider=ws://$(GETH_NODES_SERVICE_HOST):$(GETH_NODES_SERVICE_PORT_WS)"
+            - "--verbosity=debug"
+            - "--vrcaddr=0x541AfaC5266c534de039B4A1a53519e76ea82846"
+            - "--rpc-port=4000"
+            - "--simulator=false"
+          resources:
+            requests:
+              memory: "100Mi"
+              cpu: "50m"
+            limits:
+              memory: "250Mi"
+              cpu: "100m"
+          ports:
+            - containerPort: 4000
+              name: grpc

k8s/beacon-chain/beacon-chain.service.yaml

@@ -0,0 +1,13 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: beacon-chain
+spec:
+  selector:
+    component: beacon-chain
+    universe: beacon-chain
+  ports:
+    - port: 4000
+      targetPort: 4000
+      name: grpc
+  type: LoadBalancer

k8s/client/BUILD.bazel

@@ -0,0 +1,25 @@
+package(default_visibility = ["//k8s:__subpackages__"])
+
+load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects")
+load("@k8s_deploy//:defaults.bzl", "k8s_deploy")
+
+k8s_objects(
+    name = "everything",
+    objects = [
+        ":deployments",
+    ],
+)
+
+_deployments = [
+    "proposer",
+]
+
+k8s_objects(
+    name = "deployments",
+    objects = [":" + name + ".deploy" for name in _deployments],
+)
+
+[k8s_deploy(
+    name = name + ".deploy",
+    template = name + ".deploy.yaml",
+) for name in _deployments]

k8s/client/proposer.deploy.yaml

@@ -0,0 +1,30 @@
+kind: Deployment
+apiVersion: apps/v1beta1
+metadata:
+  name: proposer
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      component: proposer
+      universe: client
+  template:
+    metadata:
+      labels:
+        component: proposer
+        universe: client
+    spec:
+      priorityClassName: batch-priority
+      containers:
+        - name: sharding-client
+          image: gcr.io/prysmaticlabs/prysm/client:latest
+          args: 
+            - "--actor=proposer"
+            - "--beacon-rpc-provider=$(BEACON_CHAIN_SERVICE_HOST):$(BEACON_CHAIN_SERVICE_PORT_GRPC)"
+          resources:
+            requests:
+              memory: "100Mi"
+              cpu: "50m"
+            limits:
+              memory: "250Mi"
+              cpu: "100m"

k8s/geth/bootnode.deploy.yaml

@@ -29,7 +29,7 @@ spec:
         command: ["bootnode"]
         args:
           - "--nodekey=/data/private_key"
-          - "--verbosity=9"
+          - "--verbosity=4"
         volumeMounts:
           - name: secrets
             mountPath: "/data/"

k8s/geth/miners.deploy.yaml

@@ -1,5 +1,5 @@
-kind: Deployment
-apiVersion: apps/v1beta1
+kind: StatefulSet
+apiVersion: apps/v1
 metadata:
   name: miner
   labels:
@@ -7,6 +7,7 @@ metadata:
     component: miner
 spec:
   replicas: 1
+  serviceName: "miners"
   selector: 
     matchLabels:
       universe: geth
@@ -41,11 +42,11 @@ spec:
             - >
               geth
               --networkid=1337
-              --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):30301
-              --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):3000
+              --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):$(GETH_BOOTNODE_SERVICE_PORT_BOOTNODE_UDP)
+              --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):$(GETH_ETHSTATS_SERVICE_PORT)
               --rpc
               --rpcaddr=0.0.0.0
-              --rpccorsdomain=\"*\"
+              --rpccorsdomain="*"
               --ws
               --datadir=/ethereum
               --debug
@@ -55,13 +56,16 @@ spec:
               --etherbase=0x717c3a6e4cbd476c2312612155eb233bf498dd5b
               --extradata=$HOSTNAME
               --ethash.dagsinmem=1
-              --ethash.dagsondisk=1
+              --ethash.dagsondisk=2
+              --ethash.dagdir="/dag"
               --nousb
               --cache=1024
               --gasprice=0
           volumeMounts:
             - name: chaindata
               mountPath: "/ethereum"
+            - name: dag
+              mountPath: "/dag"
           env:
             - name: ETHSTATS_WS_SECRET
               valueFrom:
@@ -78,7 +82,7 @@ spec:
               memory: "2Gi"
               cpu: "500m"
             limits:
-              memory: "2Gi"
+              memory: "2.5Gi"
               cpu: "500m"
       initContainers:
         - name: genesis
@@ -100,3 +104,11 @@ spec:
             items:
               - key: json
                 path: genesis.json
+  volumeClaimTemplates:
+    - metadata:
+        name: dag
+      spec: 
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: 2Gi

k8s/geth/nodes.deploy.yaml

@@ -41,14 +41,14 @@ spec:
             - >
               geth
               --networkid=1337
-              --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):30301
-              --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):3000
+              --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):$(GETH_BOOTNODE_SERVICE_PORT_BOOTNODE_UDP)
+              --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):$(GETH_ETHSTATS_SERVICE_PORT)
               --rpc
               --rpcaddr=0.0.0.0
-              --rpccorsdomain=\"*\"
+              --rpccorsdomain="*"
               --ws
               --wsaddr=0.0.0.0
-              --wsorigins=\"*\"
+              --wsorigins="*"
               --datadir=/ethereum
               --debug
               --verbosity=4

shared/iputils/BUILD.bazel

@@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["external_ip.go"],
+    importpath = "github.com/prysmaticlabs/prysm/shared/iputils",
+    visibility = ["//visibility:public"],
+)

shared/iputils/external_ip.go

@@ -0,0 +1,44 @@
+package iputils
+
+import (
+	"errors"
+	"net"
+)
+
+// ExternalIPv4 returns the first IPv4 available.
+func ExternalIPv4() (string, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return "", err
+	}
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagUp == 0 {
+			continue // interface down
+		}
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue // loopback interface
+		}
+		addrs, err := iface.Addrs()
+		if err != nil {
+			return "", err
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil || ip.IsLoopback() {
+				continue
+			}
+			ip = ip.To4()
+			if ip == nil {
+				continue // not an ipv4 address
+			}
+			return ip.String(), nil
+		}
+	}
+	return "", errors.New("are you connected to the network?")
+}

shared/p2p/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
     deps = [
         "//proto/beacon/p2p/v1:go_default_library",
         "//proto/sharding/p2p/v1:go_default_library",
+        "//shared/iputils:go_default_library",
         "@com_github_ethereum_go_ethereum//event:go_default_library",
         "@com_github_golang_protobuf//proto:go_default_library",
         "@com_github_libp2p_go_floodsub//:go_default_library",

shared/p2p/discovery.go

@@ -52,6 +52,11 @@ type discovery struct {
 
 // HandlePeerFound registers the peer with the host.
 func (d *discovery) HandlePeerFound(pi ps.PeerInfo) {
+	log.WithFields(logrus.Fields{
+		"peer addrs": pi.Addrs,
+		"peer id":    pi.ID,
+	}).Debug("Attempting to connect to a peer")
+
 	d.host.Peerstore().AddAddrs(pi.ID, pi.Addrs, ps.PermanentAddrTTL)
 	if err := d.host.Connect(d.ctx, pi); err != nil {
 		log.Warnf("Failed to connect to peer: %v", err)

shared/p2p/options.go

@@ -8,6 +8,7 @@ import (
 	libp2p "github.com/libp2p/go-libp2p"
 	crypto "github.com/libp2p/go-libp2p-crypto"
 	ma "github.com/multiformats/go-multiaddr"
+	"github.com/prysmaticlabs/prysm/shared/iputils"
 )
 
 var port int32 = 9000
@@ -24,7 +25,12 @@ func buildOptions() []libp2p.Option {
 		log.Errorf("Failed to generate crypto key pair: %v", err)
 	}
 
-	listen, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", port+(rand.Int31n(portRange))))
+	ip, err := iputils.ExternalIPv4()
+	if err != nil {
+		log.Errorf("Could not get IPv4 address: %v", err)
+	}
+
+	listen, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%d", ip, port+(rand.Int31n(portRange))))
 	if err != nil {
 		log.Errorf("Failed to p2p listen: %v", err)
 	}