make: refactor erigon-lib make tasks (#8249)

2024-12-22 03:30:37 +00:00 · 2023-09-21 13:50:59 +02:00 · 2023-09-21 13:50:59 +02:00 · a4d29a6412
commit a4d29a6412
parent dedf04caaa
8 changed files with 68 additions and 61 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -54,28 +54,17 @@ jobs:
        if: runner.os == 'Linux'
        uses: golangci/golangci-lint-action@v3
        with:
-          version: v1.54
+          version: v1.54.2
          skip-build-cache: true
          args: --help

-      - name: Lint erigon
+      - name: Install go-licenses
        if: runner.os == 'Linux'
-        run: golangci-lint run --config ./.golangci.yml
+        run: cd erigon-lib && make lint-licenses-deps

-      - name: Lint erigon-lib
+      - name: Lint
        if: runner.os == 'Linux'
-        env:
-          CGO_CXXFLAGS: "-g -O2 -std=c++17"
-        run: cd erigon-lib && golangci-lint run --config ./.golangci.yml
-
-      - name: Lint source code licenses
-        if: runner.os == 'Linux'
-        run: cd erigon-lib && make lint-licenses-deps lint-licenses
-
-      - name: Test erigon-lib
-        env:
-          CGO_CXXFLAGS: "-g -O2 -std=c++17"
-        run: cd erigon-lib && make test
+        run: make lint

      - name: Test
        run: make test
@ -115,8 +104,6 @@ jobs:
        run: .\wmake.ps1 test

      - name: Test erigon-lib
-        env:
-          CGO_CXXFLAGS: "-g -O2 -std=c++17"
        run: cd erigon-lib && make test-no-fuzz

  docker-build-check:
--- a/24
+++ b/24
@ -141,31 +141,35 @@ db-tools:

 ## test:                              run unit tests with a 100s timeout
 test:
+	@cd erigon-lib && $(MAKE) test
 	$(GOTEST) --timeout 100s

 test3:
+	@cd erigon-lib && $(MAKE) test
 	$(GOTEST) --timeout 100s -tags $(BUILD_TAGS),e3

 ## test-integration:                  run integration tests with a 30m timeout
 test-integration:
+	@cd erigon-lib && $(MAKE) test
 	$(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration

 test3-integration:
+	@cd erigon-lib && $(MAKE) test
 	$(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration,e3

-## lint:                              run golangci-lint with .golangci.yml config file
-lint:
-	@./build/bin/golangci-lint run --config ./.golangci.yml
+## lint-deps:                         install lint dependencies
+lint-deps:
+	@cd erigon-lib && $(MAKE) lint-deps

-## lintci:                            run golangci-lint (additionally outputs message before run)
+## lintci:                            run golangci-lint linters
 lintci:
-	@echo "--> Running linter for code"
-	@./build/bin/golangci-lint run --config ./.golangci.yml
+	@cd erigon-lib && $(MAKE) lintci
+	@./erigon-lib/tools/golangci_lint.sh

-## lintci-deps:                       (re)installs golangci-lint to build/bin/golangci-lint
-lintci-deps:
-	rm -f ./build/bin/golangci-lint
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.54.2
+## lint:                              run all linters
+lint:
+	@cd erigon-lib && $(MAKE) lint
+	@./erigon-lib/tools/golangci_lint.sh

 ## clean:                             cleans the go cache, build dir, libmdbx db dir
 clean:
--- a/erigon-lib/.github/workflows/ci.yml
+++ b/erigon-lib/.github/workflows/ci.yml
@ -12,8 +12,7 @@ on:
      - alpha
 env:
  CGO_ENABLED: "1"
-  CGO_CXXFLAGS: "-g -O2 -std=c++17"
-  GO111MODULE: "on"
+  CGO_CXXFLAGS: "-g -O2"
 jobs:
  tests:
    strategy:
--- a/erigon-lib/Makefile
+++ b/erigon-lib/Makefile
@ -1,9 +1,18 @@
 GOBINREL = build/bin
 GOBIN = $(CURDIR)/$(GOBINREL)
+
 BUILD_TAGS = nosqlite,noboltdb,disable_libutp
-GOBUILD = env GO111MODULE=on go build -trimpath -tags $(BUILD_TAGS)
-GOTEST = go test -trimpath -tags $(BUILD_TAGS)
-GOTEST_NOFUZZ = go test -trimpath --tags=$(BUILD_TAGS),nofuzz
+
+CGO_CXXFLAGS ?= $(shell go env CGO_CXXFLAGS 2>/dev/null)
+ifeq ($(CGO_CXXFLAGS),)
+	CGO_CXXFLAGS += -g
+	CGO_CXXFLAGS += -O2
+endif
+
+GOBUILD = CGO_CXXFLAGS="$(CGO_CXXFLAGS)" go build -trimpath -tags $(BUILD_TAGS)
+GOTEST = CGO_CXXFLAGS="$(CGO_CXXFLAGS)" go test -trimpath -tags $(BUILD_TAGS)
+GOTEST_NOFUZZ = CGO_CXXFLAGS="$(CGO_CXXFLAGS)" go test -trimpath --tags=$(BUILD_TAGS),nofuzz
+
 OS = $(shell uname -s)
 ARCH = $(shell uname -m)

@ -71,19 +80,10 @@ mocks: $(GOBINREL)/moq
 	rm -f gointerfaces/sentry/mocks.go
 	PATH="$(GOBIN):$(PATH)" go generate ./...

-lintci: $(GOBINREL)/golangci-lint
-	@"$(GOBIN)/golangci-lint" run --config ./.golangci.yml
-
-# force re-make golangci-lint
-lintci-deps: lintci-deps-clean $(GOBINREL)/golangci-lint
-lintci-deps-clean: golangci-lint-clean
-
-# download and build golangci-lint (https://golangci-lint.run)
-$(GOBINREL)/golangci-lint: | $(GOBINREL)
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(GOBIN)" v1.54.2
-
-golangci-lint-clean:
-	rm -f "$(GOBIN)/golangci-lint"
+lintci-deps:
+	@./tools/golangci_lint.sh --install-deps
+lintci:
+	@CGO_CXXFLAGS="$(CGO_CXXFLAGS)" ./tools/golangci_lint.sh

 lint-licenses-deps:
 	@./tools/licenses_check.sh --install-deps
--- a/erigon-lib/README.md
+++ b/erigon-lib/README.md
@ -1,22 +1,18 @@
 # erigon-lib
-Dependencies of Erigon project, rewritten from scratch and licensed under Apache 2.0

-## Dev workflow
+Parts of Erigon codebase, written from scratch and licensed under Apache 2.0.

-In erigon folder create go.work file (it’s already in .gitignore)
-```
-go 1.20
+## License requirements

-use (
-    .
+erigon-lib dependencies use various open source licenses compatible with Apache 2.0. This is checked on CI using `make lint-licenses`.

-     ./../erigon-lib
-)
-```
+In order to keep license purity it is not allowed to refer to the code in the erigon root module from erigon-lib. This is ensured by the `go.mod` separation.

-Create PR in erigon-lib, don’t merge PR, refer from erigon to non-merged erigon-lib branch (commit) by: 
-go get github.com/ledgerwatch/erigon-lib/kv@<commit_hash> 
+It is not allowed to copy or move code from erigon to erigon-lib unless all original authors agree to relief the code license from GPL to Apache 2.0.

-Create Erigon PR
+## Code migration policy

-When both CI are green - merge 2 PR. That’s it.
+It is encouraged to write new erigon code inside erigon-lib.
+
+It is encouraged to move and relicense parts of the code from erigon to erigon-lib
+that are safe and easy to move. For example, code written from scratch by erigon core contributors that has no significant external contributions could be refactored and moved.
--- a/erigon-lib/pedersen_hash/hash.go
+++ b/erigon-lib/pedersen_hash/hash.go
@ -4,6 +4,7 @@
 package hash

 /*
+#cgo CXXFLAGS: -std=c++17
 #include <stdlib.h>
 #include "hash.h"
 */
--- a/erigon-lib/tools/golangci_lint.sh
+++ b/erigon-lib/tools/golangci_lint.sh
@ -0,0 +1,21 @@
+#!/bin/bash
+
+scriptDir=$(dirname "${BASH_SOURCE[0]}")
+scriptName=$(basename "${BASH_SOURCE[0]}")
+version="v1.54.2"
+
+if [[ "$1" == "--install-deps" ]]
+then
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" "$version"
+	exit
+fi
+
+if ! which golangci-lint > /dev/null
+then
+	echo "golangci-lint tool is not found, install it with:"
+	echo "    $scriptName --install-deps"
+	echo "or follow https://golangci-lint.run/usage/install/"
+	exit
+fi
+
+golangci-lint run --config ./.golangci.yml
--- a/wmake.ps1
+++ b/wmake.ps1
@ -420,7 +420,6 @@ $Erigon.BuildFlags = "-trimpath -tags $($Erigon.BuildTags) -buildvcs=false -v"
 $Erigon.BuildFlags += " -ldflags ""-X $($Erigon.Package)/params.GitCommit=$($Erigon.Commit) -X $($Erigon.Package)/params.GitBranch=$($Erigon.Branch) -X $($Erigon.Package)/params.GitTag=$($Erigon.Tag)"""

 $Erigon.BinPath    = [string](Join-Path $MyContext.StartDir "\build\bin")
-$env:GO111MODULE = "on"
 $env:CGO_CFLAGS = "-g -O2 -D__BLST_PORTABLE__"

 New-Item -Path $Erigon.BinPath -ItemType Directory -Force | Out-Null