Kubernetes - Part 1 of ?? (#348)

* Add basic k8s configuration for geth deployments * set genesis difficulty to 0x0 * gazelle, comments * More comments on priority class and adjust priority class for ethstats * Adjustments to the miner limits * set minimum gas price to 0
2025-01-03 00:27:38 +00:00 · 2018-07-30 10:04:47 -04:00 · 2018-07-30 10:04:47 -04:00 · 26fec66da4
commit 26fec66da4
parent f9d23da2dd
19 changed files with 684 additions and 4 deletions
--- a/40
+++ b/40
@ -15,13 +15,45 @@ http_archive(
    urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.12.0/bazel-gazelle-0.12.0.tar.gz"],
 )

-http_archive(
+git_repository(
    name = "io_bazel_rules_docker",
-    sha256 = "6dede2c65ce86289969b907f343a1382d33c14fbce5e30dd17bb59bb55bb6593",
-    strip_prefix = "rules_docker-0.4.0",
-    urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.4.0.tar.gz"],
+    commit = "7401cb256222615c497c0dee5a4de5724a4f4cc7",  # 2018-06-22
+    remote = "https://github.com/bazelbuild/rules_docker.git",
 )

+load("@io_bazel_rules_docker//docker:docker.bzl", "docker_repositories")
+
+docker_repositories()
+
+# This requires rules_docker to be fully instantiated before it is pulled in.
+git_repository(
+    name = "io_bazel_rules_k8s",
+    commit = "2054f7bf4d51f9e439313c56d7a208960a8a179f",  # 2018-07-29
+    remote = "https://github.com/bazelbuild/rules_k8s.git",
+)
+
+load("@io_bazel_rules_k8s//k8s:k8s.bzl", "k8s_repositories", "k8s_defaults")
+
+k8s_repositories()
+
+_CLUSTER = "minikube"
+
+_NAMESPACE = "default"
+
+[k8s_defaults(
+    name = "k8s_" + kind,
+    cluster = _CLUSTER,
+    #context = _CONTEXT,
+    kind = kind,
+    namespace = _NAMESPACE,
+) for kind in [
+    "deploy",
+    "service",
+    "secret",
+    "priority_class",
+    "pod",
+]]
+
 load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains")

 go_rules_dependencies()
--- a/beacon-chain/BUILD.bazel
+++ b/beacon-chain/BUILD.bazel
@ -1,5 +1,6 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_push")

 go_library(
    name = "go_default_library",
@ -24,6 +25,7 @@ go_image(
    goos = "linux",
    importpath = "github.com/prysmaticlabs/prysm/beacon-chain",
    static = "on",
+    visibility = ["//visibility:private"],
    deps = [
        "//beacon-chain/node:go_default_library",
        "//beacon-chain/utils:go_default_library",
@ -35,6 +37,16 @@ go_image(
    ],
 )

+container_push(
+    name = "push_image",
+    format = "Docker",
+    image = ":image",
+    registry = "gcr.io",
+    repository = "prysmaticlabs/prysm/beacon-chain",
+    tag = "latest",
+    visibility = ["//visibility:private"],
+)
+
 go_binary(
    name = "beacon-chain",
    embed = [":go_default_library"],
--- a/client/BUILD.bazel
+++ b/client/BUILD.bazel
@ -1,5 +1,6 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_push")

 go_library(
    name = "go_default_library",
@ -24,6 +25,7 @@ go_image(
    goos = "linux",
    importpath = "github.com/prysmaticlabs/prysm/client",
    static = "on",
+    visibility = ["//visibility:private"],
    deps = [
        "//client/node:go_default_library",
        "//client/utils:go_default_library",
@ -35,6 +37,15 @@ go_image(
    ],
 )

+container_push(
+    name = "push_image",
+    format = "Docker",
+    image = ":image",
+    registry = "gcr.io",
+    repository = "prysmaticlabs/prysm/client",
+    tag = "latest",
+)
+
 go_binary(
    name = "client",
    embed = [":go_default_library"],
--- a/k8s/BUILD.bazel
+++ b/k8s/BUILD.bazel
@ -0,0 +1,15 @@
+load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects")
+load("@k8s_priority_class//:defaults.bzl", "k8s_priority_class")
+
+k8s_objects(
+    name = "everything",
+    objects = [
+        "//k8s/geth:everything",
+        ":priority_class",
+    ],
+)
+
+k8s_priority_class(
+    name = "priority_class",
+    template = "priority.yaml",
+)
--- a/k8s/README.md
+++ b/k8s/README.md
@ -0,0 +1,92 @@
+# Kubernetes 
+
+## Requirements
+
+- Kubernetes v1.11+ (for PriorityClass)
+- Minikube (for now)
+
+### Starting minikube with v1.11
+
+As of minikube 0.28.2, the default version of kubernetes is 1.10.0. In order to
+start a local cluster with v1.11.0, run the following:
+
+```
+minikube start --kubernetes-version=v1.11.0 --cpus 4
+```
+
+### Geth's Genesis file
+
+This file is the default provided by geth-genesis secret. 
+
+```json
+{                                                                               
+    "config": {                                                                 
+        "chainId": 1337,                                                        
+        "homesteadBlock": 0,                                                    
+        "eip155Block": 0,                                                       
+        "eip158Block": 0                                                        
+    },                                                                          
+    "difficulty": "0x0",                                                      
+    "gasLimit": "0x2100000",                                                    
+    "alloc": {                                                                  
+       "717c3a6e4cbd476c2312612155eb233bf498dd5b":                              
+         { "balance": "0x1337000000000000000000" }                             
+    }                                                                           
+}
+```
+
+The private key for the allocation above is:
+
+```text
+783da8ef5343c3019748506305d400bca8c324a5819f3a7f7fbf0c0a0d799b09
+```
+
+NOTE: Obviously, do not use this wallet key for anything with real money on it!
+
+To update the genesis secret, change value in geth/genesis.secret.yaml to the
+base64 encoded string for the genesis.json.
+
+Example:
+
+```bash
+cat /tmp/genesis.json | json-minify | base64
+```
+
+### Deploying Geth Mainchain
+
+```bash
+bazel run //k8s:everything.apply
+```
+
+This creates a few nodes and one miner with CPU restrictions. After ~30 
+minutes, the miner has generated the DAG and begins mining. This seems iterate
+over 2 DAG epochs dispite the flags set for 1 DAG in memory and in disk. 
+
+Note: This can be improved by giving the miner more CPU.
+
+### Accessing Geth Services
+
+Check out the ethstats dashboard by querying minikube for the service URL.
+
+```bash
+minikube service geth-ethstats --url
+```
+
+Accessing the geth nodes.
+
+```bash
+minikube service geth-nodes --url
+
+# Example output
+http://192.168.99.100:30451
+http://192.168.99.100:32164
+```
+
+The first URL will be the rpc endpoint and the second URL will be the websocket
+endpoint.
+
+So we can use these values locally to connect to our local cluster.
+
+```bash
+bazel run //beacon-chain -- --web3provider=ws://192.168.99.100:32164
+```
--- a/k8s/geth/BUILD.bazel
+++ b/k8s/geth/BUILD.bazel
@ -0,0 +1,64 @@
+package(default_visibility = ["//k8s:__subpackages__"])
+
+load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects")
+load("@k8s_deploy//:defaults.bzl", "k8s_deploy")
+load("@k8s_service//:defaults.bzl", "k8s_service")
+load("@k8s_secret//:defaults.bzl", "k8s_secret")
+
+k8s_objects(
+    name = "everything",
+    objects = [
+        ":secrets",
+        ":services",
+        ":deployments",
+    ],
+)
+
+_deployments = [
+    "bootnode",
+    "ethstats",
+    "miners",
+    "nodes",
+]
+
+_services = [
+    "bootnode",
+    "ethstats",
+    "nodes",
+]
+
+_secrets = [
+    "bootnode",
+    "ethstats",
+    "genesis",
+]
+
+k8s_objects(
+    name = "deployments",
+    objects = [":" + name + ".deploy" for name in _deployments],
+)
+
+[k8s_deploy(
+    name = name + ".deploy",
+    template = name + ".deploy.yaml",
+) for name in _deployments]
+
+k8s_objects(
+    name = "secrets",
+    objects = [":" + name + ".secret" for name in _secrets],
+)
+
+[k8s_secret(
+    name = name + ".secret",
+    template = name + ".secret.yaml",
+) for name in _secrets]
+
+k8s_objects(
+    name = "services",
+    objects = [":" + name + ".service" for name in _services],
+)
+
+[k8s_service(
+    name = name + ".service",
+    template = name + ".service.yaml",
+) for name in _services]
--- a/k8s/geth/bootnode.deploy.yaml
+++ b/k8s/geth/bootnode.deploy.yaml
@ -0,0 +1,47 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: geth-bootnode
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        component: bootnode
+        universe: geth
+    spec:
+      priorityClassName: production-priority
+      containers:
+      - name: bootnode
+        image: ethereum/client-go:alltools-stable
+        ports: 
+          - containerPort: 8545
+            name: rpc
+          - containerPort: 30303
+            name: discovery-tcp
+            protocol: TCP
+          - containerPort: 30303
+            name: discovery-udp
+            protocol: UDP
+          - containerPort: 30301
+            name: bootnode-udp
+            protocol: UDP
+        command: ["bootnode"]
+        args:
+          - "--nodekey=/data/private_key"
+          - "--verbosity=9"
+        volumeMounts:
+          - name: secrets
+            mountPath: "/data/"
+            readOnly: true
+        resources:
+          requests:
+            memory: "25Mi"
+            cpu: "25m"
+          limits:
+            memory: "100Mi"
+            cpu: "150m"
+      volumes:
+        - name: secrets
+          secret:
+            secretName: geth-bootnode-secret
--- a/k8s/geth/bootnode.secret.yaml
+++ b/k8s/geth/bootnode.secret.yaml
@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name:  geth-bootnode-secret
+data:
+  public_key: MDZmMmI0OGFhODY1OTQ2OTdiZjZjMmI0NjRhMjFhMmYwMWVhNzYyM2MxNGQxOWU5MTE3OGMzZTRkNDNhZDg2M2FjMzdjZmQwODA0OWY3OWIxOTgxN2VmNGZlZjk5NDUxNTYzNjM3N2M1ZjhjN2UyY2MwYWJlY2VmZjkyZTc0MWY=
+  private_key: OGUxMDg1YmQwZThmOGI2MTY0OWRjMWNlYjA2Y2Q1ZTQyNTllY2YwOWRmYTFmZWRlNGNmNDVhMmZiZDE0ODVmNg==
+type: Opaque
--- a/k8s/geth/bootnode.service.yaml
+++ b/k8s/geth/bootnode.service.yaml
@ -0,0 +1,25 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: geth-bootnode
+spec:
+  selector:
+    component: bootnode
+    universe: geth
+  ports:
+    - port: 8545
+      targetPort: 8545
+      name: rpc
+      protocol: TCP
+    - port: 30303
+      targetPort: 30303
+      name: discovery-tcp
+      protocol: TCP
+    - port: 30303
+      targetPort: 30303
+      name: discovery-udp
+      protocol: UDP
+    - port: 30301
+      targetPort: 30301
+      name: bootnode-udp
+      protocol: UDP
--- a/k8s/geth/ethstats.deploy.yaml
+++ b/k8s/geth/ethstats.deploy.yaml
@ -0,0 +1,38 @@
+kind: Deployment
+apiVersion: apps/v1beta1
+metadata:
+  name: geth-ethstats
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: ethstats
+      universe: geth
+  template:
+    metadata:
+      labels:
+        component: ethstats
+        universe: geth
+    spec:
+      priorityClassName: production-priority
+      containers:
+        - name: ethstats
+          image: ethereumex/eth-netstats:latest
+          command: ["npm"]
+          args: ["start"]
+          ports: 
+            - containerPort: 3000
+              name: web
+          env:
+            - name: WS_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: ethstats-secrets
+                  key: ws
+          resources:
+            requests:
+              memory: "100Mi"
+              cpu: "50m"
+            limits:
+              memory: "250Mi"
+              cpu: "100m"
--- a/k8s/geth/ethstats.secret.yaml
+++ b/k8s/geth/ethstats.secret.yaml
@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ethstats-secrets
+type: Opaque
+data:
+  # Secret for websocket connections
+  ws: cHJ5c20= # prysm
--- a/k8s/geth/ethstats.service.yaml
+++ b/k8s/geth/ethstats.service.yaml
@ -0,0 +1,12 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: geth-ethstats
+spec:
+  selector:
+    component: ethstats
+    universe: geth
+  ports:
+    - port: 3000
+      targetPort: 3000
+  type: LoadBalancer
--- a/k8s/geth/genesis.secret.yaml
+++ b/k8s/geth/genesis.secret.yaml
@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: geth-genesis
+data:
+  json: eyJjb25maWciOnsiY2hhaW5JZCI6MTMzNywiaG9tZXN0ZWFkQmxvY2siOjAsImVpcDE1NUJsb2NrIjowLCJlaXAxNThCbG9jayI6MH0sImRpZmZpY3VsdHkiOiIweDAiLCJnYXNMaW1pdCI6IjB4MjEwMDAwMCIsImFsbG9jIjp7IjcxN2MzYTZlNGNiZDQ3NmMyMzEyNjEyMTU1ZWIyMzNiZjQ5OGRkNWIiOnsiYmFsYW5jZSI6IjB4MTMzNzAwMDAwMDAwMDAwMDAwMDAwMCJ9fX0K
+type: Opaque
--- a/k8s/geth/miners.deploy.yaml
+++ b/k8s/geth/miners.deploy.yaml
@ -0,0 +1,102 @@
+kind: Deployment
+apiVersion: apps/v1beta1
+metadata:
+  name: miner
+  labels:
+    universe: geth
+    component: miner
+spec:
+  replicas: 1
+  selector: 
+    matchLabels:
+      universe: geth
+      component: miner
+  template:
+    metadata:
+      labels:
+        universe: geth
+        component: miner
+    spec:
+      priorityClassName: production-priority
+      containers:
+        - name: miner
+          image: ethereum/client-go:alltools-stable
+          ports: 
+           - containerPort: 8545
+             name: rpc
+           - containerPort: 8546
+             name: ws
+           - containerPort: 30303
+             name: discovery-tcp
+             protocol: TCP
+           - containerPort: 30303
+             name: discovery-udp
+             protocol: UDP
+          # Use /bin/sh -c to execute geth so that we have access to HOSTNAME in 
+          # the command arguments.
+          # https://github.com/kubernetes/kubernetes/issues/57726
+          command:
+            - "/bin/sh"
+            - "-c"
+            - >
+              geth
+              --networkid=1337
+              --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):30301
+              --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):3000
+              --rpc
+              --rpcaddr=0.0.0.0
+              --rpccorsdomain=\"*\"
+              --ws
+              --datadir=/ethereum
+              --debug
+              --verbosity=4
+              --mine
+              --minerthreads=1
+              --etherbase=0x717c3a6e4cbd476c2312612155eb233bf498dd5b
+              --extradata=$HOSTNAME
+              --ethash.dagsinmem=1
+              --ethash.dagsondisk=1
+              --nousb
+              --cache=1024
+              --gasprice=0
+          volumeMounts:
+            - name: chaindata
+              mountPath: "/ethereum"
+          env:
+            - name: ETHSTATS_WS_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: ethstats-secrets
+                  key: ws
+            - name: BOOTNODE_PUBKEY
+              valueFrom:
+                secretKeyRef:
+                  name: geth-bootnode-secret
+                  key: public_key
+          resources:
+            requests:
+              memory: "2Gi"
+              cpu: "500m"
+            limits:
+              memory: "2Gi"
+              cpu: "500m"
+      initContainers:
+        - name: genesis
+          image: ethereum/client-go:alltools-stable
+          command: ["geth"]
+          args: ["--datadir=/ethereum", "init", "/data/genesis.json"]
+          volumeMounts:
+            - name: genesis
+              mountPath: "/data"
+              readOnly: true
+            - name: chaindata
+              mountPath: "/ethereum"
+      volumes:
+        - name: chaindata
+          emptyDir: {}
+        - name: genesis
+          secret: 
+            secretName: geth-genesis
+            items:
+              - key: json
+                path: genesis.json
--- a/k8s/geth/nodes.deploy.yaml
+++ b/k8s/geth/nodes.deploy.yaml
@ -0,0 +1,96 @@
+kind: Deployment
+apiVersion: apps/v1beta1
+metadata:
+  name: node
+  labels:
+    universe: geth
+    component: node
+spec:
+  replicas: 3
+  selector: 
+    matchLabels:
+      universe: geth
+      component: node
+  template:
+    metadata:
+      labels:
+        universe: geth
+        component: node
+    spec:
+      priorityClassName: batch-priority
+      containers:
+        - name: node
+          image: ethereum/client-go:alltools-stable
+          ports: 
+           - containerPort: 8545
+             name: rpc
+           - containerPort: 8546
+             name: ws
+           - containerPort: 30303
+             name: discovery-tcp
+             protocol: TCP
+           - containerPort: 30303
+             name: discovery-udp
+             protocol: UDP
+          # Use /bin/sh -c to execute geth so that we have access to HOSTNAME in 
+          # the command arguments.
+          # https://github.com/kubernetes/kubernetes/issues/57726
+          command:
+            - "/bin/sh"
+            - "-c"
+            - >
+              geth
+              --networkid=1337
+              --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):30301
+              --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):3000
+              --rpc
+              --rpcaddr=0.0.0.0
+              --rpccorsdomain=\"*\"
+              --ws
+              --wsaddr=0.0.0.0
+              --wsorigins=\"*\"
+              --datadir=/ethereum
+              --debug
+              --verbosity=4
+              --nousb
+          volumeMounts:
+            - name: chaindata
+              mountPath: "/ethereum"
+          env:
+            - name: ETHSTATS_WS_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: ethstats-secrets
+                  key: ws
+            - name: BOOTNODE_PUBKEY
+              valueFrom:
+                secretKeyRef:
+                  name: geth-bootnode-secret
+                  key: public_key
+          resources:
+            requests:
+              memory: "500Mi"
+              cpu: "100m"
+            limits:
+              memory: "500Mi"
+              cpu: "100m"
+      initContainers:
+        - name: genesis
+          image: ethereum/client-go:alltools-stable
+          command: ["geth"]
+          args: ["--datadir=/ethereum", "init", "/data/genesis.json"]
+          volumeMounts:
+            - name: genesis
+              mountPath: "/data"
+              readOnly: true
+            - name: chaindata
+              mountPath: "/ethereum"
+      volumes:
+        - name: chaindata
+          emptyDir: {}
+        - name: genesis
+          secret: 
+            secretName: geth-genesis
+            items:
+              - key: json
+                path: genesis.json
--- a/k8s/geth/nodes.service.yaml
+++ b/k8s/geth/nodes.service.yaml
@ -0,0 +1,18 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: geth-nodes
+spec:
+  selector:
+    component: node
+    universe: geth
+  ports:
+    - port: 8545
+      targetPort: 8545
+      name: rpc
+      protocol: TCP
+    - port: 8546
+      targetPort: 8546
+      name: ws
+      protocol: TCP
+  type: LoadBalancer
--- a/k8s/priority.yaml
+++ b/k8s/priority.yaml
@ -0,0 +1,55 @@
+# Priority classes
+# 
+# These values allow us to assign importance to pods so that the scheduler can
+# appropriately preempt low priority pods when a higher priority pod is pending
+# scheduling.
+#
+# Example:
+# 1) Running a tool to query production data for one-off analysis: best-effort-priority. 
+# This tool is not mission critical and should not use resources that are 
+# necessary for production applications
+#
+# 2) A long running cron job that encodes data with a large SLO window. This
+# would be ideal for a batch-priority job since would be acceptable for 
+# temporary evictions if high priority jobs need to scale.
+#
+# 3) User facing application services. This type of work falls under the default
+# priority: production-priority. This is suitable for anything that cannot 
+# tolerate eviction or user experience will suffer.
+#
+# 4) A job that alerts when mission critical containers are offline. This type
+# of job should have a very high priority and never be evicted. This is
+# considered the monitoring-priority and should be used sparingly. 
+#  
+# See: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+apiVersion: scheduling.k8s.io/v1beta1
+kind: PriorityClass
+metadata:
+  name: best-effort-priority
+value: 0
+globalDefault: false
+description: "Free quota tier. Unlikely to stay scheduled."
+---
+apiVersion: scheduling.k8s.io/v1beta1
+kind: PriorityClass
+metadata:
+  name: batch-priority
+value: 100
+globalDefault: false
+description: "Suitible for batch jobs that can tolerate occasional downtime."
+---
+apiVersion: scheduling.k8s.io/v1beta1
+kind: PriorityClass
+metadata:
+  name: production-priority
+value: 200
+globalDefault: true
+description: "Default priority for production jobs."
+---
+apiVersion: scheduling.k8s.io/v1beta1
+kind: PriorityClass
+metadata:
+  name: monitoring-priority
+value: 300
+globalDefault: false
+description: "Reserved for monitoring health of production services."
--- a/k8s/tools/BUILD.bazel
+++ b/k8s/tools/BUILD.bazel
@ -0,0 +1,8 @@
+package(default_visibility = ["//k8s:__subpackages__"])
+
+load("@k8s_pod//:defaults.bzl", "k8s_pod")
+
+k8s_pod(
+    name = "busybox.pod",
+    template = "busybox.yaml",
+)
--- a/k8s/tools/busybox.yaml
+++ b/k8s/tools/busybox.yaml
@ -0,0 +1,30 @@
+# Busybox is a general utility container that can be used to run arbirary 
+# commands from within the cluster. 
+# 
+# Examples:
+# Print environment variables
+# kubectl exec -it busybox -- printenv 
+# 
+# Lookup address via kube-dns
+# kubectl exec -it busybox -- nslookup kubernetes.default.svc.cluster.local
+kind: Pod
+apiVersion: v1
+metadata:
+  name: busybox
+  labels:
+    name: busybox
+spec:
+  priorityClassName: best-effort-priority
+  containers:
+  - name: busybox
+    image: busybox
+    command:
+      - sleep
+      - "3600"
+    resources:
+      requests:
+        memory: "25Mi"
+        cpu: "25m"
+      limits:
+        memory: "50Mi"
+        cpu: "50m"