From 0d854fea6e324312a977261e1dcd342fb62d5060 Mon Sep 17 00:00:00 2001 From: Mark Shields <4237425+beejiujitsu@users.noreply.github.com> Date: Thu, 17 Nov 2022 18:37:53 -0500 Subject: [PATCH] feat(k8s): base with google-kubernetes-engine overlay example (#6041) re: https://discord.com/channels/687972960811745322/738982866670714901/1041381487754354689 Introduces a more modular base with a google-kubernetes-engine overlay example. --- k8s/base/kustomization.yaml | 9 +- .../{ => services}/eth66-peering-tcp.yaml | 11 +- .../{ => services}/eth66-peering-udp.yaml | 11 +- k8s/base/{ => services}/http.yaml | 13 +-- k8s/base/services/kustomization.yaml | 8 ++ k8s/base/{ => services}/metrics.yaml | 11 +- k8s/base/{ => services}/snap-sync-tcp.yaml | 11 +- k8s/base/{ => services}/snap-sync-udp.yaml | 11 +- k8s/base/statefulset.yaml | 102 ++++++++---------- .../kustomization.yaml | 12 +++ .../podmonitoring.yaml | 13 +++ .../statefulset-erigon-patch.yaml | 70 ++++++++++++ .../tls/certificate.yaml | 21 ++++ k8s/google-kubernetes-engine/tls/ingress.yaml | 21 ++++ .../tls/kustomization.yaml | 4 + 15 files changed, 231 insertions(+), 97 deletions(-) rename k8s/base/{ => services}/eth66-peering-tcp.yaml (66%) rename k8s/base/{ => services}/eth66-peering-udp.yaml (66%) rename k8s/base/{ => services}/http.yaml (59%) create mode 100644 k8s/base/services/kustomization.yaml rename k8s/base/{ => services}/metrics.yaml (63%) rename k8s/base/{ => services}/snap-sync-tcp.yaml (65%) rename k8s/base/{ => services}/snap-sync-udp.yaml (65%) create mode 100644 k8s/google-kubernetes-engine/kustomization.yaml create mode 100644 k8s/google-kubernetes-engine/podmonitoring.yaml create mode 100644 k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml create mode 100644 k8s/google-kubernetes-engine/tls/certificate.yaml create mode 100644 k8s/google-kubernetes-engine/tls/ingress.yaml create mode 100644 k8s/google-kubernetes-engine/tls/kustomization.yaml diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index 54effbae0..aad92fdf1 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -1,8 +1,3 @@ +--- resources: -- eth66-peering-tcp.yaml -- eth66-peering-udp.yaml -- http.yaml -- metrics.yaml -- snap-sync-tcp.yaml -- snap-sync-udp.yaml -- statefulset.yaml + - statefulset.yaml \ No newline at end of file diff --git a/k8s/base/eth66-peering-tcp.yaml b/k8s/base/services/eth66-peering-tcp.yaml similarity index 66% rename from k8s/base/eth66-peering-tcp.yaml rename to k8s/base/services/eth66-peering-tcp.yaml index e78717488..f81a0c65d 100644 --- a/k8s/base/eth66-peering-tcp.yaml +++ b/k8s/base/services/eth66-peering-tcp.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -6,13 +7,13 @@ metadata: name: eth66-peering-tcp spec: ipFamilies: - - IPv4 + - IPv4 ipFamilyPolicy: SingleStack ports: - - name: eth66-pr-tcp - port: 30303 - protocol: TCP - targetPort: 30303 + - name: eth66-pr-tcp + port: 30303 + protocol: TCP + targetPort: 30303 selector: app: erigon type: LoadBalancer diff --git a/k8s/base/eth66-peering-udp.yaml b/k8s/base/services/eth66-peering-udp.yaml similarity index 66% rename from k8s/base/eth66-peering-udp.yaml rename to k8s/base/services/eth66-peering-udp.yaml index 612141d73..a70e1c490 100644 --- a/k8s/base/eth66-peering-udp.yaml +++ b/k8s/base/services/eth66-peering-udp.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -6,13 +7,13 @@ metadata: name: eth66-peering-udp spec: ipFamilies: - - IPv4 + - IPv4 ipFamilyPolicy: SingleStack ports: - - name: eth66-pr-udp - port: 30303 - protocol: UDP - targetPort: 30303 + - name: eth66-pr-udp + port: 30303 + protocol: UDP + targetPort: 30303 selector: app: erigon type: LoadBalancer diff --git a/k8s/base/http.yaml b/k8s/base/services/http.yaml similarity index 59% rename from k8s/base/http.yaml rename to k8s/base/services/http.yaml index fff0733b6..c40c06892 100644 --- a/k8s/base/http.yaml +++ b/k8s/base/services/http.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -6,13 +7,13 @@ metadata: name: http spec: ipFamilies: - - IPv4 + - IPv4 ipFamilyPolicy: SingleStack ports: - - name: http - port: 8545 - protocol: TCP - targetPort: 8545 + - name: http + port: 8545 + protocol: TCP + targetPort: 8545 selector: app: erigon - type: LoadBalancer + type: LoadBalancer \ No newline at end of file diff --git a/k8s/base/services/kustomization.yaml b/k8s/base/services/kustomization.yaml new file mode 100644 index 000000000..ea7627f24 --- /dev/null +++ b/k8s/base/services/kustomization.yaml @@ -0,0 +1,8 @@ +--- +resources: + - eth66-peering-tcp.yaml + - eth66-peering-udp.yaml + - http.yaml + - metrics.yaml + - snap-sync-tcp.yaml + - snap-sync-udp.yaml \ No newline at end of file diff --git a/k8s/base/metrics.yaml b/k8s/base/services/metrics.yaml similarity index 63% rename from k8s/base/metrics.yaml rename to k8s/base/services/metrics.yaml index 3c910a725..702732973 100644 --- a/k8s/base/metrics.yaml +++ b/k8s/base/services/metrics.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -6,12 +7,12 @@ metadata: name: metrics spec: ipFamilies: - - IPv4 + - IPv4 ipFamilyPolicy: SingleStack ports: - - name: metrics - port: 6060 - protocol: TCP - targetPort: 6060 + - name: metrics + port: 6060 + protocol: TCP + targetPort: 6060 selector: app: erigon diff --git a/k8s/base/snap-sync-tcp.yaml b/k8s/base/services/snap-sync-tcp.yaml similarity index 65% rename from k8s/base/snap-sync-tcp.yaml rename to k8s/base/services/snap-sync-tcp.yaml index 7f800dd01..98c548a06 100644 --- a/k8s/base/snap-sync-tcp.yaml +++ b/k8s/base/services/snap-sync-tcp.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -6,13 +7,13 @@ metadata: name: snap-sync-tcp spec: ipFamilies: - - IPv4 + - IPv4 ipFamilyPolicy: SingleStack ports: - - name: snap-sync-tcp - port: 42069 - protocol: TCP - targetPort: 42069 + - name: snap-sync-tcp + port: 42069 + protocol: TCP + targetPort: 42069 selector: app: erigon type: LoadBalancer diff --git a/k8s/base/snap-sync-udp.yaml b/k8s/base/services/snap-sync-udp.yaml similarity index 65% rename from k8s/base/snap-sync-udp.yaml rename to k8s/base/services/snap-sync-udp.yaml index b1a41e1e9..5f600091f 100644 --- a/k8s/base/snap-sync-udp.yaml +++ b/k8s/base/services/snap-sync-udp.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -6,13 +7,13 @@ metadata: name: snap-sync-udp spec: ipFamilies: - - IPv4 + - IPv4 ipFamilyPolicy: SingleStack ports: - - name: snap-sync-udp - port: 42069 - protocol: UDP - targetPort: 42069 + - name: snap-sync-udp + port: 42069 + protocol: UDP + targetPort: 42069 selector: app: erigon type: LoadBalancer diff --git a/k8s/base/statefulset.yaml b/k8s/base/statefulset.yaml index 1896dfdac..5ff103ab8 100644 --- a/k8s/base/statefulset.yaml +++ b/k8s/base/statefulset.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -17,65 +18,48 @@ spec: app: erigon spec: containers: - - args: - - '--authrpc.vhosts=*' - - '--datadir=/home/erigon/.local/share/erigon' - - '--healthcheck' - - '--log.json' - - '--metrics' - - '--metrics.addr=0.0.0.0' - - '--metrics.port=6060' - - '--nat=none' - - '--private.api.ratelimit=50000' - - '--private.api.addr=0.0.0.0:9090' - - '--torrent.download.rate=3000mb' - - '--torrent.download.slots=200' - - '--verbosity=3' - - —-batchSize=8000M - command: - - erigon - image: erigon-image - livenessProbe: - initialDelaySeconds: 1800 - periodSeconds: 20 - tcpSocket: - port: 9090 - name: erigon - ports: - - containerPort: 9090 - name: private-api - - containerPort: 8551 - name: engine - - containerPort: 30303 - name: eth66-pr-tcp - protocol: TCP - - containerPort: 30303 - name: eth66-pr-udp - protocol: UDP - - containerPort: 6060 - name: metrics - - containerPort: 6070 - name: pprof - - containerPort: 42069 - name: snap-sync-tcp - protocol: TCP - - containerPort: 42069 - name: snap-sync-udp - protocol: UDP - readinessProbe: - initialDelaySeconds: 1800 - periodSeconds: 20 - tcpSocket: - port: 9090 - resources: - requests: - cpu: 2462m - memory: 16Gi - securityContext: - allowPrivilegeEscalation: false - runAsGroup: 1000 - runAsUser: 1000 - volumeMounts: [] + - args: [] + command: + - erigon + env: [] + image: thorax/erigon + name: erigon + ports: + - containerPort: 9090 + name: private-api + - containerPort: 8551 + name: engine + - containerPort: 30303 + name: eth66-pr-tcp + protocol: TCP + - containerPort: 30303 + name: eth66-pr-udp + protocol: UDP + - containerPort: 8545 + name: http + protocol: TCP + - containerPort: 6060 + name: metrics + - containerPort: 6070 + name: pprof + - containerPort: 42069 + name: snap-sync-tcp + protocol: TCP + - containerPort: 42069 + name: snap-sync-udp + protocol: UDP + readinessProbe: + tcpSocket: + port: 8545 + resources: + requests: + cpu: 2462m + memory: 16Gi + securityContext: + allowPrivilegeEscalation: false + runAsGroup: 1000 + runAsUser: 1000 + volumeMounts: [] initContainers: [] volumes: [] volumeClaimTemplates: [] \ No newline at end of file diff --git a/k8s/google-kubernetes-engine/kustomization.yaml b/k8s/google-kubernetes-engine/kustomization.yaml new file mode 100644 index 000000000..c39f19a60 --- /dev/null +++ b/k8s/google-kubernetes-engine/kustomization.yaml @@ -0,0 +1,12 @@ +--- +patchesJson6902: + - path: statefulset-erigon-patch.yaml + target: + group: apps + kind: StatefulSet + name: erigon + version: v1 +resources: + - ../base + - ../base/services + - podmonitoring.yaml \ No newline at end of file diff --git a/k8s/google-kubernetes-engine/podmonitoring.yaml b/k8s/google-kubernetes-engine/podmonitoring.yaml new file mode 100644 index 000000000..48b319703 --- /dev/null +++ b/k8s/google-kubernetes-engine/podmonitoring.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + name: erigon +spec: + endpoints: + - interval: 30s + path: /debug/metrics/prometheus + port: metrics + selector: + matchLabels: + app: erigon diff --git a/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml b/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml new file mode 100644 index 000000000..c8185cf28 --- /dev/null +++ b/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml @@ -0,0 +1,70 @@ +--- +- op: replace + path: /spec/replicas + value: 3 +- op: add + path: /spec/template/spec/containers/0/resources/limits + value: + cpu: '9' + memory: 110Gi +- op: replace + path: /spec/template/spec/containers/0/resources/requests/cpu + value: '9' +- op: replace + path: /spec/template/spec/containers/0/resources/requests/memory + value: 110Gi +- op: replace + path: /spec/template/spec/containers/0/args + value: + - '--chain=mainnet' + - '--datadir=/home/erigon/.local/share/erigon' + - '--db.pagesize=64KB' + - '--healthcheck' + - '--http' + - '--http.addr=0.0.0.0' + - '--http.api=eth,erigon,web3,net,debug,ots,trace,txpool' + - '--http.corsdomain=*' + - '--http.vhosts=*' + - '--log.console.verbosity=1' + - '--log.json' + - '--metrics' + - '--metrics.addr=0.0.0.0' + - '--metrics.port=6060' + - '--nat=none' + - '--torrent.download.rate=3000mb' + - '--torrent.download.slots=200' + - '--ws' + - '--ws.compression' +- op: replace + path: /spec/template/spec/containers/0/volumeMounts + value: + - mountPath: /home/erigon/.local/share/erigon + name: mainnet +- op: add + path: /spec/template/spec/initContainers/- + value: + command: + - sh + - '-c' + - chown 1000 -R /home/erigon/.local/share/erigon + image: busybox + name: chown-datadir + securityContext: + capabilities: + add: + - CHOWN + volumeMounts: + - mountPath: /home/erigon/.local/share/erigon + name: mainnet +- op: add + path: /spec/volumeClaimTemplates/- + value: + metadata: + name: mainnet + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3000Gi + storageClassName: premium-rwo \ No newline at end of file diff --git a/k8s/google-kubernetes-engine/tls/certificate.yaml b/k8s/google-kubernetes-engine/tls/certificate.yaml new file mode 100644 index 000000000..65c092eb2 --- /dev/null +++ b/k8s/google-kubernetes-engine/tls/certificate.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: mainnet +spec: + commonName: "" + dnsNames: + - "" + duration: 24h0m0s + issuerRef: + kind: ClusterIssuer + name: "" + renewBefore: 8h0m0s + secretName: "" + subject: + organizations: + - "" + usages: + - server auth + - client auth \ No newline at end of file diff --git a/k8s/google-kubernetes-engine/tls/ingress.yaml b/k8s/google-kubernetes-engine/tls/ingress.yaml new file mode 100644 index 000000000..574477689 --- /dev/null +++ b/k8s/google-kubernetes-engine/tls/ingress.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: mainnet +spec: + rules: + - host: "" + http: + paths: + - backend: + service: + name: http + port: + number: 8545 + path: / + pathType: Prefix + tls: + - hosts: + - "" + secretName: "" \ No newline at end of file diff --git a/k8s/google-kubernetes-engine/tls/kustomization.yaml b/k8s/google-kubernetes-engine/tls/kustomization.yaml new file mode 100644 index 000000000..942294934 --- /dev/null +++ b/k8s/google-kubernetes-engine/tls/kustomization.yaml @@ -0,0 +1,4 @@ +--- +resources: + - certificate.yaml + - ingress.yaml \ No newline at end of file