From a4d29a6412f25027175155afd2d6a424325c21d2 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Thu, 21 Sep 2023 13:50:59 +0200 Subject: [PATCH] make: refactor erigon-lib make tasks (#8249) --- .github/workflows/ci.yml | 23 +++++---------------- Makefile | 24 +++++++++++++--------- erigon-lib/.github/workflows/ci.yml | 3 +-- erigon-lib/Makefile | 32 ++++++++++++++--------------- erigon-lib/README.md | 24 +++++++++------------- erigon-lib/pedersen_hash/hash.go | 1 + erigon-lib/tools/golangci_lint.sh | 21 +++++++++++++++++++ wmake.ps1 | 1 - 8 files changed, 68 insertions(+), 61 deletions(-) create mode 100755 erigon-lib/tools/golangci_lint.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40336d1ea..fd3248e92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,28 +54,17 @@ jobs: if: runner.os == 'Linux' uses: golangci/golangci-lint-action@v3 with: - version: v1.54 + version: v1.54.2 skip-build-cache: true args: --help - - name: Lint erigon + - name: Install go-licenses if: runner.os == 'Linux' - run: golangci-lint run --config ./.golangci.yml + run: cd erigon-lib && make lint-licenses-deps - - name: Lint erigon-lib + - name: Lint if: runner.os == 'Linux' - env: - CGO_CXXFLAGS: "-g -O2 -std=c++17" - run: cd erigon-lib && golangci-lint run --config ./.golangci.yml - - - name: Lint source code licenses - if: runner.os == 'Linux' - run: cd erigon-lib && make lint-licenses-deps lint-licenses - - - name: Test erigon-lib - env: - CGO_CXXFLAGS: "-g -O2 -std=c++17" - run: cd erigon-lib && make test + run: make lint - name: Test run: make test @@ -115,8 +104,6 @@ jobs: run: .\wmake.ps1 test - name: Test erigon-lib - env: - CGO_CXXFLAGS: "-g -O2 -std=c++17" run: cd erigon-lib && make test-no-fuzz docker-build-check: diff --git a/Makefile b/Makefile index 3408e11d6..a7c60c24f 100644 --- a/Makefile +++ b/Makefile @@ -141,31 +141,35 @@ db-tools: ## test: run unit tests with a 100s timeout test: + @cd erigon-lib && $(MAKE) test $(GOTEST) --timeout 100s test3: + @cd erigon-lib && $(MAKE) test $(GOTEST) --timeout 100s -tags $(BUILD_TAGS),e3 ## test-integration: run integration tests with a 30m timeout test-integration: + @cd erigon-lib && $(MAKE) test $(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration test3-integration: + @cd erigon-lib && $(MAKE) test $(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration,e3 -## lint: run golangci-lint with .golangci.yml config file -lint: - @./build/bin/golangci-lint run --config ./.golangci.yml +## lint-deps: install lint dependencies +lint-deps: + @cd erigon-lib && $(MAKE) lint-deps -## lintci: run golangci-lint (additionally outputs message before run) +## lintci: run golangci-lint linters lintci: - @echo "--> Running linter for code" - @./build/bin/golangci-lint run --config ./.golangci.yml + @cd erigon-lib && $(MAKE) lintci + @./erigon-lib/tools/golangci_lint.sh -## lintci-deps: (re)installs golangci-lint to build/bin/golangci-lint -lintci-deps: - rm -f ./build/bin/golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.54.2 +## lint: run all linters +lint: + @cd erigon-lib && $(MAKE) lint + @./erigon-lib/tools/golangci_lint.sh ## clean: cleans the go cache, build dir, libmdbx db dir clean: diff --git a/erigon-lib/.github/workflows/ci.yml b/erigon-lib/.github/workflows/ci.yml index 485d313d0..b76ed8d3b 100644 --- a/erigon-lib/.github/workflows/ci.yml +++ b/erigon-lib/.github/workflows/ci.yml @@ -12,8 +12,7 @@ on: - alpha env: CGO_ENABLED: "1" - CGO_CXXFLAGS: "-g -O2 -std=c++17" - GO111MODULE: "on" + CGO_CXXFLAGS: "-g -O2" jobs: tests: strategy: diff --git a/erigon-lib/Makefile b/erigon-lib/Makefile index 85dc3aa84..390733c1b 100644 --- a/erigon-lib/Makefile +++ b/erigon-lib/Makefile @@ -1,9 +1,18 @@ GOBINREL = build/bin GOBIN = $(CURDIR)/$(GOBINREL) + BUILD_TAGS = nosqlite,noboltdb,disable_libutp -GOBUILD = env GO111MODULE=on go build -trimpath -tags $(BUILD_TAGS) -GOTEST = go test -trimpath -tags $(BUILD_TAGS) -GOTEST_NOFUZZ = go test -trimpath --tags=$(BUILD_TAGS),nofuzz + +CGO_CXXFLAGS ?= $(shell go env CGO_CXXFLAGS 2>/dev/null) +ifeq ($(CGO_CXXFLAGS),) + CGO_CXXFLAGS += -g + CGO_CXXFLAGS += -O2 +endif + +GOBUILD = CGO_CXXFLAGS="$(CGO_CXXFLAGS)" go build -trimpath -tags $(BUILD_TAGS) +GOTEST = CGO_CXXFLAGS="$(CGO_CXXFLAGS)" go test -trimpath -tags $(BUILD_TAGS) +GOTEST_NOFUZZ = CGO_CXXFLAGS="$(CGO_CXXFLAGS)" go test -trimpath --tags=$(BUILD_TAGS),nofuzz + OS = $(shell uname -s) ARCH = $(shell uname -m) @@ -71,19 +80,10 @@ mocks: $(GOBINREL)/moq rm -f gointerfaces/sentry/mocks.go PATH="$(GOBIN):$(PATH)" go generate ./... -lintci: $(GOBINREL)/golangci-lint - @"$(GOBIN)/golangci-lint" run --config ./.golangci.yml - -# force re-make golangci-lint -lintci-deps: lintci-deps-clean $(GOBINREL)/golangci-lint -lintci-deps-clean: golangci-lint-clean - -# download and build golangci-lint (https://golangci-lint.run) -$(GOBINREL)/golangci-lint: | $(GOBINREL) - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(GOBIN)" v1.54.2 - -golangci-lint-clean: - rm -f "$(GOBIN)/golangci-lint" +lintci-deps: + @./tools/golangci_lint.sh --install-deps +lintci: + @CGO_CXXFLAGS="$(CGO_CXXFLAGS)" ./tools/golangci_lint.sh lint-licenses-deps: @./tools/licenses_check.sh --install-deps diff --git a/erigon-lib/README.md b/erigon-lib/README.md index 4bb31ab94..85866c38e 100644 --- a/erigon-lib/README.md +++ b/erigon-lib/README.md @@ -1,22 +1,18 @@ # erigon-lib -Dependencies of Erigon project, rewritten from scratch and licensed under Apache 2.0 -## Dev workflow +Parts of Erigon codebase, written from scratch and licensed under Apache 2.0. -In erigon folder create go.work file (it’s already in .gitignore) -``` -go 1.20 +## License requirements -use ( - . +erigon-lib dependencies use various open source licenses compatible with Apache 2.0. This is checked on CI using `make lint-licenses`. - ./../erigon-lib -) -``` +In order to keep license purity it is not allowed to refer to the code in the erigon root module from erigon-lib. This is ensured by the `go.mod` separation. -Create PR in erigon-lib, don’t merge PR, refer from erigon to non-merged erigon-lib branch (commit) by: -go get github.com/ledgerwatch/erigon-lib/kv@ +It is not allowed to copy or move code from erigon to erigon-lib unless all original authors agree to relief the code license from GPL to Apache 2.0. -Create Erigon PR +## Code migration policy -When both CI are green - merge 2 PR. That’s it. +It is encouraged to write new erigon code inside erigon-lib. + +It is encouraged to move and relicense parts of the code from erigon to erigon-lib +that are safe and easy to move. For example, code written from scratch by erigon core contributors that has no significant external contributions could be refactored and moved. diff --git a/erigon-lib/pedersen_hash/hash.go b/erigon-lib/pedersen_hash/hash.go index 3714a1a68..983c2a02d 100644 --- a/erigon-lib/pedersen_hash/hash.go +++ b/erigon-lib/pedersen_hash/hash.go @@ -4,6 +4,7 @@ package hash /* +#cgo CXXFLAGS: -std=c++17 #include #include "hash.h" */ diff --git a/erigon-lib/tools/golangci_lint.sh b/erigon-lib/tools/golangci_lint.sh new file mode 100755 index 000000000..5dbdcbd33 --- /dev/null +++ b/erigon-lib/tools/golangci_lint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +scriptDir=$(dirname "${BASH_SOURCE[0]}") +scriptName=$(basename "${BASH_SOURCE[0]}") +version="v1.54.2" + +if [[ "$1" == "--install-deps" ]] +then + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" "$version" + exit +fi + +if ! which golangci-lint > /dev/null +then + echo "golangci-lint tool is not found, install it with:" + echo " $scriptName --install-deps" + echo "or follow https://golangci-lint.run/usage/install/" + exit +fi + +golangci-lint run --config ./.golangci.yml diff --git a/wmake.ps1 b/wmake.ps1 index d1ad3342d..5f8b42d6c 100644 --- a/wmake.ps1 +++ b/wmake.ps1 @@ -420,7 +420,6 @@ $Erigon.BuildFlags = "-trimpath -tags $($Erigon.BuildTags) -buildvcs=false -v" $Erigon.BuildFlags += " -ldflags ""-X $($Erigon.Package)/params.GitCommit=$($Erigon.Commit) -X $($Erigon.Package)/params.GitBranch=$($Erigon.Branch) -X $($Erigon.Package)/params.GitTag=$($Erigon.Tag)""" $Erigon.BinPath = [string](Join-Path $MyContext.StartDir "\build\bin") -$env:GO111MODULE = "on" $env:CGO_CFLAGS = "-g -O2 -D__BLST_PORTABLE__" New-Item -Path $Erigon.BinPath -ItemType Directory -Force | Out-Null