diff --git a/WORKSPACE b/WORKSPACE index e4c6f6d96..4cfd1d4eb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -15,13 +15,45 @@ http_archive( urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.12.0/bazel-gazelle-0.12.0.tar.gz"], ) -http_archive( +git_repository( name = "io_bazel_rules_docker", - sha256 = "6dede2c65ce86289969b907f343a1382d33c14fbce5e30dd17bb59bb55bb6593", - strip_prefix = "rules_docker-0.4.0", - urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.4.0.tar.gz"], + commit = "7401cb256222615c497c0dee5a4de5724a4f4cc7", # 2018-06-22 + remote = "https://github.com/bazelbuild/rules_docker.git", ) +load("@io_bazel_rules_docker//docker:docker.bzl", "docker_repositories") + +docker_repositories() + +# This requires rules_docker to be fully instantiated before it is pulled in. +git_repository( + name = "io_bazel_rules_k8s", + commit = "2054f7bf4d51f9e439313c56d7a208960a8a179f", # 2018-07-29 + remote = "https://github.com/bazelbuild/rules_k8s.git", +) + +load("@io_bazel_rules_k8s//k8s:k8s.bzl", "k8s_repositories", "k8s_defaults") + +k8s_repositories() + +_CLUSTER = "minikube" + +_NAMESPACE = "default" + +[k8s_defaults( + name = "k8s_" + kind, + cluster = _CLUSTER, + #context = _CONTEXT, + kind = kind, + namespace = _NAMESPACE, +) for kind in [ + "deploy", + "service", + "secret", + "priority_class", + "pod", +]] + load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains") go_rules_dependencies() diff --git a/beacon-chain/BUILD.bazel b/beacon-chain/BUILD.bazel index 730039023..10c21a6f3 100644 --- a/beacon-chain/BUILD.bazel +++ b/beacon-chain/BUILD.bazel @@ -1,5 +1,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") load("@io_bazel_rules_docker//go:image.bzl", "go_image") +load("@io_bazel_rules_docker//container:container.bzl", "container_push") go_library( name = "go_default_library", @@ -24,6 +25,7 @@ go_image( goos = "linux", importpath = "github.com/prysmaticlabs/prysm/beacon-chain", static = "on", + visibility = ["//visibility:private"], deps = [ "//beacon-chain/node:go_default_library", "//beacon-chain/utils:go_default_library", @@ -35,6 +37,16 @@ go_image( ], ) +container_push( + name = "push_image", + format = "Docker", + image = ":image", + registry = "gcr.io", + repository = "prysmaticlabs/prysm/beacon-chain", + tag = "latest", + visibility = ["//visibility:private"], +) + go_binary( name = "beacon-chain", embed = [":go_default_library"], diff --git a/client/BUILD.bazel b/client/BUILD.bazel index 3d6c62e30..b582f30ae 100644 --- a/client/BUILD.bazel +++ b/client/BUILD.bazel @@ -1,5 +1,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") load("@io_bazel_rules_docker//go:image.bzl", "go_image") +load("@io_bazel_rules_docker//container:container.bzl", "container_push") go_library( name = "go_default_library", @@ -24,6 +25,7 @@ go_image( goos = "linux", importpath = "github.com/prysmaticlabs/prysm/client", static = "on", + visibility = ["//visibility:private"], deps = [ "//client/node:go_default_library", "//client/utils:go_default_library", @@ -35,6 +37,15 @@ go_image( ], ) +container_push( + name = "push_image", + format = "Docker", + image = ":image", + registry = "gcr.io", + repository = "prysmaticlabs/prysm/client", + tag = "latest", +) + go_binary( name = "client", embed = [":go_default_library"], diff --git a/k8s/BUILD.bazel b/k8s/BUILD.bazel new file mode 100644 index 000000000..b3dc3a0b5 --- /dev/null +++ b/k8s/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") +load("@k8s_priority_class//:defaults.bzl", "k8s_priority_class") + +k8s_objects( + name = "everything", + objects = [ + "//k8s/geth:everything", + ":priority_class", + ], +) + +k8s_priority_class( + name = "priority_class", + template = "priority.yaml", +) diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 000000000..773a0a143 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,92 @@ +# Kubernetes + +## Requirements + +- Kubernetes v1.11+ (for PriorityClass) +- Minikube (for now) + +### Starting minikube with v1.11 + +As of minikube 0.28.2, the default version of kubernetes is 1.10.0. In order to +start a local cluster with v1.11.0, run the following: + +``` +minikube start --kubernetes-version=v1.11.0 --cpus 4 +``` + +### Geth's Genesis file + +This file is the default provided by geth-genesis secret. + +```json +{ + "config": { + "chainId": 1337, + "homesteadBlock": 0, + "eip155Block": 0, + "eip158Block": 0 + }, + "difficulty": "0x0", + "gasLimit": "0x2100000", + "alloc": { + "717c3a6e4cbd476c2312612155eb233bf498dd5b": + { "balance": "0x1337000000000000000000" } + } +} +``` + +The private key for the allocation above is: + +```text +783da8ef5343c3019748506305d400bca8c324a5819f3a7f7fbf0c0a0d799b09 +``` + +NOTE: Obviously, do not use this wallet key for anything with real money on it! + +To update the genesis secret, change value in geth/genesis.secret.yaml to the +base64 encoded string for the genesis.json. + +Example: + +```bash +cat /tmp/genesis.json | json-minify | base64 +``` + +### Deploying Geth Mainchain + +```bash +bazel run //k8s:everything.apply +``` + +This creates a few nodes and one miner with CPU restrictions. After ~30 +minutes, the miner has generated the DAG and begins mining. This seems iterate +over 2 DAG epochs dispite the flags set for 1 DAG in memory and in disk. + +Note: This can be improved by giving the miner more CPU. + +### Accessing Geth Services + +Check out the ethstats dashboard by querying minikube for the service URL. + +```bash +minikube service geth-ethstats --url +``` + +Accessing the geth nodes. + +```bash +minikube service geth-nodes --url + +# Example output +http://192.168.99.100:30451 +http://192.168.99.100:32164 +``` + +The first URL will be the rpc endpoint and the second URL will be the websocket +endpoint. + +So we can use these values locally to connect to our local cluster. + +```bash +bazel run //beacon-chain -- --web3provider=ws://192.168.99.100:32164 +``` \ No newline at end of file diff --git a/k8s/geth/BUILD.bazel b/k8s/geth/BUILD.bazel new file mode 100644 index 000000000..2f180656c --- /dev/null +++ b/k8s/geth/BUILD.bazel @@ -0,0 +1,64 @@ +package(default_visibility = ["//k8s:__subpackages__"]) + +load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") +load("@k8s_deploy//:defaults.bzl", "k8s_deploy") +load("@k8s_service//:defaults.bzl", "k8s_service") +load("@k8s_secret//:defaults.bzl", "k8s_secret") + +k8s_objects( + name = "everything", + objects = [ + ":secrets", + ":services", + ":deployments", + ], +) + +_deployments = [ + "bootnode", + "ethstats", + "miners", + "nodes", +] + +_services = [ + "bootnode", + "ethstats", + "nodes", +] + +_secrets = [ + "bootnode", + "ethstats", + "genesis", +] + +k8s_objects( + name = "deployments", + objects = [":" + name + ".deploy" for name in _deployments], +) + +[k8s_deploy( + name = name + ".deploy", + template = name + ".deploy.yaml", +) for name in _deployments] + +k8s_objects( + name = "secrets", + objects = [":" + name + ".secret" for name in _secrets], +) + +[k8s_secret( + name = name + ".secret", + template = name + ".secret.yaml", +) for name in _secrets] + +k8s_objects( + name = "services", + objects = [":" + name + ".service" for name in _services], +) + +[k8s_service( + name = name + ".service", + template = name + ".service.yaml", +) for name in _services] diff --git a/k8s/geth/bootnode.deploy.yaml b/k8s/geth/bootnode.deploy.yaml new file mode 100644 index 000000000..4cb92f8c9 --- /dev/null +++ b/k8s/geth/bootnode.deploy.yaml @@ -0,0 +1,47 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: geth-bootnode +spec: + replicas: 1 + template: + metadata: + labels: + component: bootnode + universe: geth + spec: + priorityClassName: production-priority + containers: + - name: bootnode + image: ethereum/client-go:alltools-stable + ports: + - containerPort: 8545 + name: rpc + - containerPort: 30303 + name: discovery-tcp + protocol: TCP + - containerPort: 30303 + name: discovery-udp + protocol: UDP + - containerPort: 30301 + name: bootnode-udp + protocol: UDP + command: ["bootnode"] + args: + - "--nodekey=/data/private_key" + - "--verbosity=9" + volumeMounts: + - name: secrets + mountPath: "/data/" + readOnly: true + resources: + requests: + memory: "25Mi" + cpu: "25m" + limits: + memory: "100Mi" + cpu: "150m" + volumes: + - name: secrets + secret: + secretName: geth-bootnode-secret \ No newline at end of file diff --git a/k8s/geth/bootnode.secret.yaml b/k8s/geth/bootnode.secret.yaml new file mode 100644 index 000000000..e624b5b3a --- /dev/null +++ b/k8s/geth/bootnode.secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: geth-bootnode-secret +data: + public_key: MDZmMmI0OGFhODY1OTQ2OTdiZjZjMmI0NjRhMjFhMmYwMWVhNzYyM2MxNGQxOWU5MTE3OGMzZTRkNDNhZDg2M2FjMzdjZmQwODA0OWY3OWIxOTgxN2VmNGZlZjk5NDUxNTYzNjM3N2M1ZjhjN2UyY2MwYWJlY2VmZjkyZTc0MWY= + private_key: OGUxMDg1YmQwZThmOGI2MTY0OWRjMWNlYjA2Y2Q1ZTQyNTllY2YwOWRmYTFmZWRlNGNmNDVhMmZiZDE0ODVmNg== +type: Opaque \ No newline at end of file diff --git a/k8s/geth/bootnode.service.yaml b/k8s/geth/bootnode.service.yaml new file mode 100644 index 000000000..cfe2fbc1f --- /dev/null +++ b/k8s/geth/bootnode.service.yaml @@ -0,0 +1,25 @@ +kind: Service +apiVersion: v1 +metadata: + name: geth-bootnode +spec: + selector: + component: bootnode + universe: geth + ports: + - port: 8545 + targetPort: 8545 + name: rpc + protocol: TCP + - port: 30303 + targetPort: 30303 + name: discovery-tcp + protocol: TCP + - port: 30303 + targetPort: 30303 + name: discovery-udp + protocol: UDP + - port: 30301 + targetPort: 30301 + name: bootnode-udp + protocol: UDP \ No newline at end of file diff --git a/k8s/geth/ethstats.deploy.yaml b/k8s/geth/ethstats.deploy.yaml new file mode 100644 index 000000000..2daa369f4 --- /dev/null +++ b/k8s/geth/ethstats.deploy.yaml @@ -0,0 +1,38 @@ +kind: Deployment +apiVersion: apps/v1beta1 +metadata: + name: geth-ethstats +spec: + replicas: 1 + selector: + matchLabels: + component: ethstats + universe: geth + template: + metadata: + labels: + component: ethstats + universe: geth + spec: + priorityClassName: production-priority + containers: + - name: ethstats + image: ethereumex/eth-netstats:latest + command: ["npm"] + args: ["start"] + ports: + - containerPort: 3000 + name: web + env: + - name: WS_SECRET + valueFrom: + secretKeyRef: + name: ethstats-secrets + key: ws + resources: + requests: + memory: "100Mi" + cpu: "50m" + limits: + memory: "250Mi" + cpu: "100m" diff --git a/k8s/geth/ethstats.secret.yaml b/k8s/geth/ethstats.secret.yaml new file mode 100644 index 000000000..edd76ec68 --- /dev/null +++ b/k8s/geth/ethstats.secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ethstats-secrets +type: Opaque +data: + # Secret for websocket connections + ws: cHJ5c20= # prysm diff --git a/k8s/geth/ethstats.service.yaml b/k8s/geth/ethstats.service.yaml new file mode 100644 index 000000000..025ab7f48 --- /dev/null +++ b/k8s/geth/ethstats.service.yaml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: geth-ethstats +spec: + selector: + component: ethstats + universe: geth + ports: + - port: 3000 + targetPort: 3000 + type: LoadBalancer \ No newline at end of file diff --git a/k8s/geth/genesis.secret.yaml b/k8s/geth/genesis.secret.yaml new file mode 100644 index 000000000..7778b7091 --- /dev/null +++ b/k8s/geth/genesis.secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: geth-genesis +data: + json: eyJjb25maWciOnsiY2hhaW5JZCI6MTMzNywiaG9tZXN0ZWFkQmxvY2siOjAsImVpcDE1NUJsb2NrIjowLCJlaXAxNThCbG9jayI6MH0sImRpZmZpY3VsdHkiOiIweDAiLCJnYXNMaW1pdCI6IjB4MjEwMDAwMCIsImFsbG9jIjp7IjcxN2MzYTZlNGNiZDQ3NmMyMzEyNjEyMTU1ZWIyMzNiZjQ5OGRkNWIiOnsiYmFsYW5jZSI6IjB4MTMzNzAwMDAwMDAwMDAwMDAwMDAwMCJ9fX0K +type: Opaque \ No newline at end of file diff --git a/k8s/geth/miners.deploy.yaml b/k8s/geth/miners.deploy.yaml new file mode 100644 index 000000000..35dd2e5e4 --- /dev/null +++ b/k8s/geth/miners.deploy.yaml @@ -0,0 +1,102 @@ +kind: Deployment +apiVersion: apps/v1beta1 +metadata: + name: miner + labels: + universe: geth + component: miner +spec: + replicas: 1 + selector: + matchLabels: + universe: geth + component: miner + template: + metadata: + labels: + universe: geth + component: miner + spec: + priorityClassName: production-priority + containers: + - name: miner + image: ethereum/client-go:alltools-stable + ports: + - containerPort: 8545 + name: rpc + - containerPort: 8546 + name: ws + - containerPort: 30303 + name: discovery-tcp + protocol: TCP + - containerPort: 30303 + name: discovery-udp + protocol: UDP + # Use /bin/sh -c to execute geth so that we have access to HOSTNAME in + # the command arguments. + # https://github.com/kubernetes/kubernetes/issues/57726 + command: + - "/bin/sh" + - "-c" + - > + geth + --networkid=1337 + --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):30301 + --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):3000 + --rpc + --rpcaddr=0.0.0.0 + --rpccorsdomain=\"*\" + --ws + --datadir=/ethereum + --debug + --verbosity=4 + --mine + --minerthreads=1 + --etherbase=0x717c3a6e4cbd476c2312612155eb233bf498dd5b + --extradata=$HOSTNAME + --ethash.dagsinmem=1 + --ethash.dagsondisk=1 + --nousb + --cache=1024 + --gasprice=0 + volumeMounts: + - name: chaindata + mountPath: "/ethereum" + env: + - name: ETHSTATS_WS_SECRET + valueFrom: + secretKeyRef: + name: ethstats-secrets + key: ws + - name: BOOTNODE_PUBKEY + valueFrom: + secretKeyRef: + name: geth-bootnode-secret + key: public_key + resources: + requests: + memory: "2Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "500m" + initContainers: + - name: genesis + image: ethereum/client-go:alltools-stable + command: ["geth"] + args: ["--datadir=/ethereum", "init", "/data/genesis.json"] + volumeMounts: + - name: genesis + mountPath: "/data" + readOnly: true + - name: chaindata + mountPath: "/ethereum" + volumes: + - name: chaindata + emptyDir: {} + - name: genesis + secret: + secretName: geth-genesis + items: + - key: json + path: genesis.json diff --git a/k8s/geth/nodes.deploy.yaml b/k8s/geth/nodes.deploy.yaml new file mode 100644 index 000000000..a44a45e64 --- /dev/null +++ b/k8s/geth/nodes.deploy.yaml @@ -0,0 +1,96 @@ +kind: Deployment +apiVersion: apps/v1beta1 +metadata: + name: node + labels: + universe: geth + component: node +spec: + replicas: 3 + selector: + matchLabels: + universe: geth + component: node + template: + metadata: + labels: + universe: geth + component: node + spec: + priorityClassName: batch-priority + containers: + - name: node + image: ethereum/client-go:alltools-stable + ports: + - containerPort: 8545 + name: rpc + - containerPort: 8546 + name: ws + - containerPort: 30303 + name: discovery-tcp + protocol: TCP + - containerPort: 30303 + name: discovery-udp + protocol: UDP + # Use /bin/sh -c to execute geth so that we have access to HOSTNAME in + # the command arguments. + # https://github.com/kubernetes/kubernetes/issues/57726 + command: + - "/bin/sh" + - "-c" + - > + geth + --networkid=1337 + --bootnodes=enode://$(BOOTNODE_PUBKEY)@$(GETH_BOOTNODE_SERVICE_HOST):30301 + --ethstats=$HOSTNAME:$(ETHSTATS_WS_SECRET)@$(GETH_ETHSTATS_SERVICE_HOST):3000 + --rpc + --rpcaddr=0.0.0.0 + --rpccorsdomain=\"*\" + --ws + --wsaddr=0.0.0.0 + --wsorigins=\"*\" + --datadir=/ethereum + --debug + --verbosity=4 + --nousb + volumeMounts: + - name: chaindata + mountPath: "/ethereum" + env: + - name: ETHSTATS_WS_SECRET + valueFrom: + secretKeyRef: + name: ethstats-secrets + key: ws + - name: BOOTNODE_PUBKEY + valueFrom: + secretKeyRef: + name: geth-bootnode-secret + key: public_key + resources: + requests: + memory: "500Mi" + cpu: "100m" + limits: + memory: "500Mi" + cpu: "100m" + initContainers: + - name: genesis + image: ethereum/client-go:alltools-stable + command: ["geth"] + args: ["--datadir=/ethereum", "init", "/data/genesis.json"] + volumeMounts: + - name: genesis + mountPath: "/data" + readOnly: true + - name: chaindata + mountPath: "/ethereum" + volumes: + - name: chaindata + emptyDir: {} + - name: genesis + secret: + secretName: geth-genesis + items: + - key: json + path: genesis.json diff --git a/k8s/geth/nodes.service.yaml b/k8s/geth/nodes.service.yaml new file mode 100644 index 000000000..dda774f97 --- /dev/null +++ b/k8s/geth/nodes.service.yaml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: geth-nodes +spec: + selector: + component: node + universe: geth + ports: + - port: 8545 + targetPort: 8545 + name: rpc + protocol: TCP + - port: 8546 + targetPort: 8546 + name: ws + protocol: TCP + type: LoadBalancer \ No newline at end of file diff --git a/k8s/priority.yaml b/k8s/priority.yaml new file mode 100644 index 000000000..74ee2b51f --- /dev/null +++ b/k8s/priority.yaml @@ -0,0 +1,55 @@ +# Priority classes +# +# These values allow us to assign importance to pods so that the scheduler can +# appropriately preempt low priority pods when a higher priority pod is pending +# scheduling. +# +# Example: +# 1) Running a tool to query production data for one-off analysis: best-effort-priority. +# This tool is not mission critical and should not use resources that are +# necessary for production applications +# +# 2) A long running cron job that encodes data with a large SLO window. This +# would be ideal for a batch-priority job since would be acceptable for +# temporary evictions if high priority jobs need to scale. +# +# 3) User facing application services. This type of work falls under the default +# priority: production-priority. This is suitable for anything that cannot +# tolerate eviction or user experience will suffer. +# +# 4) A job that alerts when mission critical containers are offline. This type +# of job should have a very high priority and never be evicted. This is +# considered the monitoring-priority and should be used sparingly. +# +# See: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +apiVersion: scheduling.k8s.io/v1beta1 +kind: PriorityClass +metadata: + name: best-effort-priority +value: 0 +globalDefault: false +description: "Free quota tier. Unlikely to stay scheduled." +--- +apiVersion: scheduling.k8s.io/v1beta1 +kind: PriorityClass +metadata: + name: batch-priority +value: 100 +globalDefault: false +description: "Suitible for batch jobs that can tolerate occasional downtime." +--- +apiVersion: scheduling.k8s.io/v1beta1 +kind: PriorityClass +metadata: + name: production-priority +value: 200 +globalDefault: true +description: "Default priority for production jobs." +--- +apiVersion: scheduling.k8s.io/v1beta1 +kind: PriorityClass +metadata: + name: monitoring-priority +value: 300 +globalDefault: false +description: "Reserved for monitoring health of production services." diff --git a/k8s/tools/BUILD.bazel b/k8s/tools/BUILD.bazel new file mode 100644 index 000000000..26b6d3ba3 --- /dev/null +++ b/k8s/tools/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//k8s:__subpackages__"]) + +load("@k8s_pod//:defaults.bzl", "k8s_pod") + +k8s_pod( + name = "busybox.pod", + template = "busybox.yaml", +) diff --git a/k8s/tools/busybox.yaml b/k8s/tools/busybox.yaml new file mode 100644 index 000000000..ca345c37b --- /dev/null +++ b/k8s/tools/busybox.yaml @@ -0,0 +1,30 @@ +# Busybox is a general utility container that can be used to run arbirary +# commands from within the cluster. +# +# Examples: +# Print environment variables +# kubectl exec -it busybox -- printenv +# +# Lookup address via kube-dns +# kubectl exec -it busybox -- nslookup kubernetes.default.svc.cluster.local +kind: Pod +apiVersion: v1 +metadata: + name: busybox + labels: + name: busybox +spec: + priorityClassName: best-effort-priority + containers: + - name: busybox + image: busybox + command: + - sleep + - "3600" + resources: + requests: + memory: "25Mi" + cpu: "25m" + limits: + memory: "50Mi" + cpu: "50m"