erigon-pulse/cmd/release/main.go

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/hashicorp/go-version"
)

type Binary struct {
	Name     string `json:"name"`
	FileName string `json:"file_name"`
	File     string `json:"file"`
	OS       string `json:"os"`
	Arch     string `json:"arch"`
	SigFile  string `json:"sig"`
	Checksum string `json:"checksum"`
	Commit   string `json:"commit"`
	Version  string `json:"version"`
	Tag      string `json:"tag"`
	Release  string `json:"release"`
}

type Release struct {
	Version  string   `json:"version"`
	Binaries []Binary `json:"binaries"`
}

type Releases struct {
	Releases []Release `json:"releases"`
}

type GithubReleaseAsset struct {
	Url                string `json:"url"`
	Name               string `json:"name"`
	BrowserDownloadUrl string `json:"browser_download_url"`
	Size               int    `json:"size"`
	ContentType        string `json:"content_type"`
}

type GithubRelease struct {
	TagName     string               `json:"tag_name"`
	Name        string               `json:"name"`
	PublishedAt time.Time            `json:"published_at"`
	Assets      []GithubReleaseAsset `json:"assets"`
}

type DockerImage struct {
	Creator int `json:"creator"`
	Id      int `json:"id"`
	Images  []struct {
		Architecture string      `json:"architecture"`
		Features     string      `json:"features"`
		Variant      interface{} `json:"variant"`
		Digest       string      `json:"digest"`
		Os           string      `json:"os"`
		OsFeatures   string      `json:"os_features"`
		OsVersion    interface{} `json:"os_version"`
		Size         int         `json:"size"`
		Status       string      `json:"status"`
		LastPulled   time.Time   `json:"last_pulled"`
		LastPushed   time.Time   `json:"last_pushed"`
	} `json:"images"`
	LastUpdated         time.Time `json:"last_updated"`
	LastUpdater         int       `json:"last_updater"`
	LastUpdaterUsername string    `json:"last_updater_username"`
	Name                string    `json:"name"`
	Repository          int       `json:"repository"`
	FullSize            int       `json:"full_size"`
	V2                  bool      `json:"v2"`
	TagStatus           string    `json:"tag_status"`
	TagLastPulled       time.Time `json:"tag_last_pulled"`
	TagLastPushed       time.Time `json:"tag_last_pushed"`
	MediaType           string    `json:"media_type"`
	ContentType         string    `json:"content_type"`
	Digest              string    `json:"digest"`
}

var githubToken string
var releasesCount int

func init() {
	flag.StringVar(&githubToken, "github-token", "", "Github token")
	flag.IntVar(&releasesCount, "releases", 5, "Number of releases to fetch")

	flag.Parse()

	// check all flags are set
	flag.VisitAll(func(f *flag.Flag) {
		if f.Value.String() == "" {
			fmt.Printf("Flag %s is not set\n", f.Name)
			os.Exit(1)
		}
	})
}

// release is a small program which takes command line flags from a ci release job for an individual binary
// and adds the binary to the releases.json file which is then served by downloads.erigon.ch
func main() {
	// read releases from disk (same file as used to populate downloads.erigon.ch)
	releases, err := readJsonFile()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// get last 5 releases
	ghReleases, err := getReleases(githubToken, "https://api.github.com/repos/ledgerwatch/erigon/releases", releasesCount)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// loop binaries to get stable and rc versions
	var stable []GithubRelease
	var rc []GithubRelease
	for _, ghRelease := range ghReleases {
		if strings.Contains(ghRelease.TagName, "rc") {
			if len(rc) < releasesCount {
				rc = append(rc, ghRelease)
			}
		} else {
			if len(stable) < releasesCount {
				stable = append(stable, ghRelease)
			}
		}
	}

	allReleases := append(stable, rc...)

	for _, ghRelease := range allReleases {
		checksums := map[string]string{}

		// get checksums first
		for _, asset := range ghRelease.Assets {
			if strings.Contains(asset.Name, "checksums") {
				checksums, err = parseChecksums(asset)
				if err != nil {
					fmt.Println(err)
					continue
				}
			}
		}

		for _, asset := range ghRelease.Assets {
			if strings.Contains(asset.Name, "checksums") {
				continue
			}

			name, v, o, arch, err := parseAssetName(asset)
			binary := Binary{
				Name:     name,
				File:     asset.BrowserDownloadUrl,
				FileName: asset.Name,
				OS:       o,
				Arch:     arch,
				SigFile:  "",
				Checksum: checksums[asset.Name],
				Commit:   "",
				Version:  v,
				Tag:      ghRelease.TagName,
				Release:  ghRelease.Name,
			}

			// add current binary to releases (if doesn't already exist)
			releases, err = addCurrentBinary(releases, binary)
			if err != nil {
				fmt.Println(err)
				continue
			}
		}

		v := strings.TrimPrefix(ghRelease.TagName, "v")

		// add docker images
		repositories, err := getDockerImages("https://hub.docker.com/v2/repositories/thorax/erigon/tags", v)
		if err != nil {
			fmt.Println(err)
			continue
		}
		for _, repo := range repositories {
			file := fmt.Sprintf("https://hub.docker.com/r/thorax/erigon/tags?page=1&ordering=last_updated&name=%s", repo.Name)
			binary := Binary{
				Name:     repo.Name,
				File:     file,
				FileName: fmt.Sprintf("%s (%s, %s)", repo.Name, repo.Images[0].Os, repo.Images[0].Architecture),
				OS:       "docker",
				Arch:     repo.Images[0].Architecture,
				SigFile:  "",
				Checksum: repo.Images[0].Digest,
				Commit:   "",
				Version:  v,
				Tag:      ghRelease.TagName,
				Release:  ghRelease.Name,
			}

			releases, err = addCurrentBinary(releases, binary)
			if err != nil {
				fmt.Println(err)
				continue
			}
		}
	}

	// sort releases descending by version
	sort.Slice(releases.Releases, func(i, j int) bool {
		return releases.Releases[i].Version > releases.Releases[j].Version
	})

	// write releases back to disk
	err = writeJsonFile(releases)
}

func getDockerImages(url string, search string) ([]DockerImage, error) {
	var images []DockerImage

	for {
		// Make GET request to DockerHub API
		resp, err := http.Get(url)
		if err != nil {
			return nil, err
		}

		// Decode JSON response
		var data struct {
			Results []DockerImage `json:"results"`
			Next    string        `json:"next"`
		}

		if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
			return nil, err
		}
		resp.Body.Close()

		// Append images with matching prefix to results
		for _, image := range data.Results {
			if strings.Contains(image.Name, search) {
				images = append(images, image)
			}
		}

		// Stop if there are no more pages to retrieve
		if data.Next == "" {
			break
		}

		// Update URL to retrieve next page
		url = data.Next
	}

	return images, nil
}

func getReleases(token string, releasesUrl string, count int) ([]GithubRelease, error) {
	// make request to github api to get release at url using token
	req, err := http.NewRequest("GET", releasesUrl, nil)
	if err != nil {
		return nil, fmt.Errorf("error creating request: %w", err)
	}
	req.Header.Set("Accept", "application/vnd.github+json")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
	req.Header.Set("Per-Page", fmt.Sprintf("%d", count))
	req.Header.Set("Page", "1")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("error making request: %w", err)
	}
	defer resp.Body.Close()

	var releases []GithubRelease
	err = json.NewDecoder(resp.Body).Decode(&releases)
	if err != nil {
		return nil, fmt.Errorf("error parsing json: %w", err)
	}

	return releases, nil

}

func getLatestRelease(token string, releaseUrl string) (GithubRelease, error) {
	// make request to github api to get release at url using token
	req, err := http.NewRequest("GET", releaseUrl, nil)
	if err != nil {
		return GithubRelease{}, fmt.Errorf("error creating request: %w", err)
	}
	req.Header.Set("Accept", "application/vnd.github+json")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("X-GitHub-Api-Version", "2022-11-28")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return GithubRelease{}, fmt.Errorf("error making request: %w", err)
	}
	defer resp.Body.Close()

	var release GithubRelease
	err = json.NewDecoder(resp.Body).Decode(&release)
	if err != nil {
		return GithubRelease{}, fmt.Errorf("error parsing json: %w", err)
	}

	return release, nil
}

func parseAssetName(asset GithubReleaseAsset) (name string, version string, os string, arch string, err error) {
	// parse asset name to get binary name, os and arch
	// example: erigon_2.39.0_darwin_amd64.tar.gz

	fileName := strings.TrimSuffix(asset.Name, ".tar.gz")
	list := strings.Split(fileName, "_")
	if len(list) != 4 {
		if strings.Contains(asset.Name, "checksum") {
			// tbc
			return "checksums", list[1], "", "", nil
		}
		return "", "", "", "", fmt.Errorf("error parsing asset name: %s", asset.Name)
	}

	return list[0], list[1], list[2], list[3], nil
}

func parseChecksums(asset GithubReleaseAsset) (map[string]string, error) {
	// download checksum file
	resp, err := http.Get(asset.BrowserDownloadUrl)
	if err != nil {
		return nil, fmt.Errorf("error downloading checksum file: %w", err)
	}

	// parse checksum file
	checksums := map[string]string{}
	scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
		line := scanner.Text()
		list := strings.Split(line, " ")
		if len(list) != 3 {
			return nil, fmt.Errorf("error parsing checksum file: %s", line)
		}
		checksums[list[2]] = list[0]
	}

	return checksums, nil
}

func readJsonFile() (Releases, error) {
	data, err := os.ReadFile("./releases.json")
	if err != nil {
		if os.IsNotExist(err) {
			return Releases{}, nil
		}
		return Releases{}, fmt.Errorf("error reading file: %w", err)
	}

	var releases Releases
	err = json.Unmarshal(data, &releases)
	if err != nil {
		return Releases{}, fmt.Errorf("error parsing json: %w", err)
	}
	return releases, nil
}

func addCurrentBinary(releases Releases, b Binary) (Releases, error) {
	// check if the binary is already in the release, if so return (no change to be made)
	for _, release := range releases.Releases {
		for _, binary := range release.Binaries {
			if binary.Name == b.Name && binary.OS == b.OS && binary.Arch == b.Arch && binary.Version == b.Version {
				return releases, fmt.Errorf("binary already exists in release")
			}
		}
	}

	var found bool
	// add binary to existing release
	for i, release := range releases.Releases {
		if release.Version == b.Version {
			releases.Releases[i].Binaries = append(releases.Releases[i].Binaries, b)
			found = true
			break
		}
	}

	// if no releases exist or there isn't one for the current binary version, create one
	if len(releases.Releases) == 0 || !found {
		releases.Releases = append(releases.Releases, Release{
			Version:  b.Version,
			Binaries: []Binary{b},
		})
	}

	// sort releases by version
	sort.Slice(releases.Releases, func(i, j int) bool {
		v1, _ := version.NewVersion(releases.Releases[i].Version)
		v2, _ := version.NewVersion(releases.Releases[j].Version)
		return v1.LessThan(v2)
	})

	// remove the oldest release to maintain a list of 5
	if len(releases.Releases) > releasesCount {
		releases.Releases = releases.Releases[1:]
	}

	return releases, nil
}

func writeJsonFile(releases Releases) error {
	data, err := json.MarshalIndent(releases, "", "  ")
	if err != nil {
		return fmt.Errorf("error marshalling json: %w", err)
	}

	err = os.WriteFile("releases.json", data, 0644)
	if err != nil {
		return fmt.Errorf("error writing file: %w", err)
	}

	return nil
}