feat(k8s): base with google-kubernetes-engine overlay example (#6041)

re:
https://discord.com/channels/687972960811745322/738982866670714901/1041381487754354689

Introduces a more modular base with a google-kubernetes-engine overlay
example.

k8s/base/kustomization.yaml

@@ -1,8 +1,3 @@
+---
 resources:
-- eth66-peering-tcp.yaml
+  - statefulset.yaml
-- eth66-peering-udp.yaml
-- http.yaml
-- metrics.yaml
-- snap-sync-tcp.yaml
-- snap-sync-udp.yaml
-- statefulset.yaml

k8s/base/services/eth66-peering-tcp.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: eth66-peering-tcp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: eth66-pr-tcp
+    - name: eth66-pr-tcp
-    port: 30303
+      port: 30303
-    protocol: TCP
+      protocol: TCP
-    targetPort: 30303
+      targetPort: 30303
   selector:
     app: erigon
   type: LoadBalancer

k8s/base/services/eth66-peering-udp.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: eth66-peering-udp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: eth66-pr-udp
+    - name: eth66-pr-udp
-    port: 30303
+      port: 30303
-    protocol: UDP
+      protocol: UDP
-    targetPort: 30303
+      targetPort: 30303
   selector:
     app: erigon
   type: LoadBalancer

k8s/base/services/http.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: http
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: http
+    - name: http
-    port: 8545
+      port: 8545
-    protocol: TCP
+      protocol: TCP
-    targetPort: 8545
+      targetPort: 8545
   selector:
     app: erigon
   type: LoadBalancer

k8s/base/services/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+resources:
+  - eth66-peering-tcp.yaml
+  - eth66-peering-udp.yaml
+  - http.yaml
+  - metrics.yaml
+  - snap-sync-tcp.yaml
+  - snap-sync-udp.yaml

k8s/base/services/metrics.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,12 +7,12 @@ metadata:
   name: metrics
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: metrics
+    - name: metrics
-    port: 6060
+      port: 6060
-    protocol: TCP
+      protocol: TCP
-    targetPort: 6060
+      targetPort: 6060
   selector:
     app: erigon

k8s/base/services/snap-sync-tcp.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: snap-sync-tcp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: snap-sync-tcp
+    - name: snap-sync-tcp
-    port: 42069
+      port: 42069
-    protocol: TCP
+      protocol: TCP
-    targetPort: 42069
+      targetPort: 42069
   selector:
     app: erigon
   type: LoadBalancer

k8s/base/services/snap-sync-udp.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: snap-sync-udp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: snap-sync-udp
+    - name: snap-sync-udp
-    port: 42069
+      port: 42069
-    protocol: UDP
+      protocol: UDP
-    targetPort: 42069
+      targetPort: 42069
   selector:
     app: erigon
   type: LoadBalancer

k8s/base/statefulset.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -17,65 +18,48 @@ spec:
         app: erigon
     spec:
       containers:
-      - args:
+        - args: []
-        - '--authrpc.vhosts=*'
+          command:
-        - '--datadir=/home/erigon/.local/share/erigon'
+            - erigon
-        - '--healthcheck'
+          env: []
-        - '--log.json'
+          image: thorax/erigon
-        - '--metrics'
+          name: erigon
-        - '--metrics.addr=0.0.0.0'
+          ports:
-        - '--metrics.port=6060'
+            - containerPort: 9090
-        - '--nat=none'
+              name: private-api
-        - '--private.api.ratelimit=50000'
+            - containerPort: 8551
-        - '--private.api.addr=0.0.0.0:9090'
+              name: engine
-        - '--torrent.download.rate=3000mb'
+            - containerPort: 30303
-        - '--torrent.download.slots=200'
+              name: eth66-pr-tcp
-        - '--verbosity=3'
+              protocol: TCP
-        - —-batchSize=8000M
+            - containerPort: 30303
-        command:
+              name: eth66-pr-udp
-        - erigon
+              protocol: UDP
-        image: erigon-image
+            - containerPort: 8545
-        livenessProbe:
+              name: http
-          initialDelaySeconds: 1800
+              protocol: TCP
-          periodSeconds: 20
+            - containerPort: 6060
-          tcpSocket:
+              name: metrics
-            port: 9090
+            - containerPort: 6070
-        name: erigon
+              name: pprof
-        ports:
+            - containerPort: 42069
-        - containerPort: 9090
+              name: snap-sync-tcp
-          name: private-api
+              protocol: TCP
-        - containerPort: 8551
+            - containerPort: 42069
-          name: engine
+              name: snap-sync-udp
-        - containerPort: 30303
+              protocol: UDP
-          name: eth66-pr-tcp
+          readinessProbe:
-          protocol: TCP
+            tcpSocket:
-        - containerPort: 30303
+              port: 8545
-          name: eth66-pr-udp
+          resources:
-          protocol: UDP
+            requests:
-        - containerPort: 6060
+              cpu: 2462m
-          name: metrics
+              memory: 16Gi
-        - containerPort: 6070
+          securityContext:
-          name: pprof
+            allowPrivilegeEscalation: false
-        - containerPort: 42069
+            runAsGroup: 1000
-          name: snap-sync-tcp
+            runAsUser: 1000
-          protocol: TCP
+          volumeMounts: []
-        - containerPort: 42069
-          name: snap-sync-udp
-          protocol: UDP
-        readinessProbe:
-          initialDelaySeconds: 1800
-          periodSeconds: 20
-          tcpSocket:
-            port: 9090
-        resources:
-          requests:
-            cpu: 2462m
-            memory: 16Gi
-        securityContext:
-          allowPrivilegeEscalation: false
-          runAsGroup: 1000
-          runAsUser: 1000
-        volumeMounts: []
       initContainers: []
       volumes: []
   volumeClaimTemplates: []

k8s/google-kubernetes-engine/kustomization.yaml

@@ -0,0 +1,12 @@
+---
+patchesJson6902:
+  - path: statefulset-erigon-patch.yaml
+    target:
+      group: apps
+      kind: StatefulSet
+      name: erigon
+      version: v1
+resources:
+  - ../base
+  - ../base/services
+  - podmonitoring.yaml

k8s/google-kubernetes-engine/podmonitoring.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+  name: erigon
+spec:
+  endpoints:
+    - interval: 30s
+      path: /debug/metrics/prometheus
+      port: metrics
+  selector:
+    matchLabels:
+      app: erigon

k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml

@@ -0,0 +1,70 @@
+---
+- op: replace
+  path: /spec/replicas
+  value: 3
+- op: add
+  path: /spec/template/spec/containers/0/resources/limits
+  value:
+    cpu: '9'
+    memory: 110Gi
+- op: replace
+  path: /spec/template/spec/containers/0/resources/requests/cpu
+  value: '9'
+- op: replace
+  path: /spec/template/spec/containers/0/resources/requests/memory
+  value: 110Gi
+- op: replace
+  path: /spec/template/spec/containers/0/args
+  value:
+    - '--chain=mainnet'
+    - '--datadir=/home/erigon/.local/share/erigon'
+    - '--db.pagesize=64KB'
+    - '--healthcheck'
+    - '--http'
+    - '--http.addr=0.0.0.0'
+    - '--http.api=eth,erigon,web3,net,debug,ots,trace,txpool'
+    - '--http.corsdomain=*'
+    - '--http.vhosts=*'
+    - '--log.console.verbosity=1'
+    - '--log.json'
+    - '--metrics'
+    - '--metrics.addr=0.0.0.0'
+    - '--metrics.port=6060'
+    - '--nat=none'
+    - '--torrent.download.rate=3000mb'
+    - '--torrent.download.slots=200'
+    - '--ws'
+    - '--ws.compression'
+- op: replace
+  path: /spec/template/spec/containers/0/volumeMounts
+  value:
+    - mountPath: /home/erigon/.local/share/erigon
+      name: mainnet
+- op: add
+  path: /spec/template/spec/initContainers/-
+  value:
+    command:
+      - sh
+      - '-c'
+      - chown 1000 -R /home/erigon/.local/share/erigon
+    image: busybox
+    name: chown-datadir
+    securityContext:
+      capabilities:
+        add:
+          - CHOWN
+    volumeMounts:
+      - mountPath: /home/erigon/.local/share/erigon
+        name: mainnet
+- op: add
+  path: /spec/volumeClaimTemplates/-
+  value:
+    metadata:
+      name: mainnet
+    spec:
+      accessModes:
+        - ReadWriteOnce
+      resources:
+        requests:
+          storage: 3000Gi
+      storageClassName: premium-rwo

k8s/google-kubernetes-engine/tls/certificate.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mainnet
+spec:
+  commonName: ""
+  dnsNames:
+    - ""
+  duration: 24h0m0s
+  issuerRef:
+    kind: ClusterIssuer
+    name: ""
+  renewBefore: 8h0m0s
+  secretName: ""
+  subject:
+    organizations:
+      - ""
+  usages:
+    - server auth
+    - client auth

k8s/google-kubernetes-engine/tls/ingress.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: mainnet
+spec:
+  rules:
+    - host: ""
+      http:
+        paths:
+          - backend:
+              service:
+                name: http
+                port:
+                  number: 8545
+            path: /
+            pathType: Prefix
+  tls:
+    - hosts:
+        - ""
+      secretName: ""

k8s/google-kubernetes-engine/tls/kustomization.yaml

@@ -0,0 +1,4 @@
+---
+resources:
+  - certificate.yaml
+  - ingress.yaml