From 50f1e4e7d55cbc0fdd250436b26a6cec7921af95 Mon Sep 17 00:00:00 2001 From: Kyle Sanderson Date: Sat, 16 Nov 2024 14:57:41 -0800 Subject: [PATCH] build(ci): implement PGO (#1812) * build(ci): implement pgo Implement PGO (performance guided optimizations) for Go builds. --- .github/workflows/release.yml | 172 +++++++++++++++++++++++++++++++--- .goreleaser.yml | 2 + ci.Dockerfile | 2 +- ciwindows.Dockerfile | 2 +- cmd/autobrr/main.go | 35 ++++++- 5 files changed, 199 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb5020c..1e7bfac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,8 +59,13 @@ jobs: path: web/dist test: - name: Test - runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + cgo: [ 1, 0 ] + name: Test${{ matrix.cgo == 1 && ' CGO'|| '' }} ${{ matrix.os }} + runs-on: ${{ matrix.os }} services: test_postgres: image: postgres:12.10 @@ -71,21 +76,35 @@ jobs: POSTGRES_PASSWORD: testdb POSTGRES_DB: autobrr options: --health-cmd pg_isready --health-interval 1s --health-timeout 5s --health-retries 60 + env: + CGO_ENABLED: ${{ matrix.cgo }} steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - # 1.20 is the last version to support Windows < 10, Server < 2016, and MacOS < 1.15. - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} cache: true + - name: Create Profile environment + run: + | + printf '#!/usr/bin/env bash\nset -eu\nfor pkg in $(go list "$@"); do\n\tgo test -json -cpuprofile="profile/$(echo $pkg | tr / -)-${{ matrix.cgo }}.pprof" ${{ startsWith(matrix.os, 'ubuntu') && '-tags=integration ' || '' }}"$pkg"\ndone' | tee -a profile.sh; + chmod +x profile.sh; + mkdir profile; + - name: Test - run: go run gotest.tools/gotestsum@latest --junitfile unit-tests.xml --format pkgname -- ./... -tags=integration + run: go run gotest.tools/gotestsum@latest --junitfile unit-tests.xml --format pkgname --raw-command ./profile.sh -- ./... + + - name: Upload pprof + uses: actions/upload-artifact@v4 + with: + name: pprof-test-${{ matrix.os }}-${{ matrix.cgo }} + path: profile - name: Test Summary uses: test-summary/action@v2 @@ -93,10 +112,71 @@ jobs: paths: "unit-tests.xml" if: always() - goreleaserbuild: - name: Build distribution binaries - runs-on: ubuntu-latest - needs: [web, test] + testother: + strategy: + fail-fast: true + matrix: + os: [macos-latest, windows-latest] + cgo: [ 1, 0 ] + name: Test${{ matrix.cgo == 1 && ' CGO'|| '' }} ${{ matrix.os }} + runs-on: ${{ matrix.os }} + env: + GOPATH: ${{ startsWith(matrix.os, 'windows') && 'D:\golang\go' || '' }} + GOCACHE: ${{ startsWith(matrix.os, 'windows') && 'D:\golang\cache' || '' }} + GOMODCACHE: ${{ startsWith(matrix.os, 'windows') && 'D:\golang\modcache' || '' }} + USERPROFILE: ${{ startsWith(matrix.os, 'windows') && 'D:\homedir' || '' }} + CGO_ENABLED: ${{ matrix.cgo }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Create Profile environment + shell: bash + run: + | + printf '#!/usr/bin/env bash\nset -eu\nfor pkg in $(go list "$@"); do\n\tgo test -json -cpuprofile="profile/$(echo $pkg | tr / -)-${{ matrix.cgo }}.pprof" ${{ startsWith(matrix.os, 'ubuntu') && '-tags=integration ' || '' }}"$pkg"\ndone' | tee -a profile.sh; + chmod +x profile.sh; + mkdir profile; + + - name: Profile + shell: bash + run: ${{ startsWith(matrix.os, 'windows') && './profile.sh ./...' || 'go run gotest.tools/gotestsum@latest --junitfile unit-tests.xml --format pkgname --raw-command ./profile.sh -- ./...' }} + + - name: Upload pprof + uses: actions/upload-artifact@v4 + with: + name: pprof-test-${{ matrix.os }}-${{ matrix.cgo }} + path: profile + + - name: Test Summary + uses: test-summary/action@v2 + with: + paths: "unit-tests.xml" + if: always() && startsWith(matrix.os, 'windows') == false + + pgo: + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + cgo: [ 1, 0 ] + name: Automatic PGO ${{ matrix.cgo == 1 && 'CGO ' || ''}}run ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: [web] + env: + GOPATH: ${{ startsWith(matrix.os, 'windows') && 'D:\golang\go' || '' }} + GOCACHE: ${{ startsWith(matrix.os, 'windows') && 'D:\golang\cache' || '' }} + GOMODCACHE: ${{ startsWith(matrix.os, 'windows') && 'D:\golang\modcache' || '' }} + USERPROFILE: ${{ startsWith(matrix.os, 'windows') && 'D:\homedir' || '' }} + CGO_ENABLED: ${{ matrix.cgo }} steps: - name: Checkout uses: actions/checkout@v4 @@ -109,7 +189,72 @@ jobs: name: web-dist path: web/dist -# 1.20 is the last version to support Windows < 10, Server < 2016, and MacOS < 1.15. + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Generate Profile + run: go run cmd/autobrr/main.go --pgo cpu-${{ matrix.os }}-${{ matrix.cgo }}.pprof + + - name: Upload pprof + uses: actions/upload-artifact@v4 + with: + name: pprof-pgo-${{ matrix.os }}-${{ matrix.cgo }} + path: cpu-${{ matrix.os }}-${{ matrix.cgo }}.pprof + + goprofilecombine: + name: Combine pprof profiles + runs-on: ubuntu-latest + needs: [pgo, test, testother] + steps: + - name: Download pprof profiles + uses: actions/download-artifact@v4 + with: + pattern: pprof-* + merge-multiple: true + path: profile + + - name: List contents + run: ls -la profile + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Merge Profiles + run: go tool pprof -proto profile/*.pprof | tee -a cpu.pprof + + - name: Upload pprof + uses: actions/upload-artifact@v4 + with: + name: pprof + path: cpu.pprof + + goreleaserbuild: + name: Build distribution binaries + runs-on: ubuntu-latest + needs: [web, goprofilecombine] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download web production build + uses: actions/download-artifact@v4 + with: + name: web-dist + path: web/dist + + - name: Download pprof profile + uses: actions/download-artifact@v4 + with: + name: pprof + - name: Set up Go uses: actions/setup-go@v5 with: @@ -178,7 +323,7 @@ jobs: # - linux/riscv64 - linux/s390x - windows/amd64 - needs: [web, test] + needs: [web, goprofilecombine] steps: - name: Checkout uses: actions/checkout@v4 @@ -191,6 +336,11 @@ jobs: name: web-dist path: web/dist + - name: Download pprof profile + uses: actions/download-artifact@v4 + with: + name: pprof + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -257,7 +407,7 @@ jobs: name: Publish Docker multi-arch manifest if: ${{ github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request' }} runs-on: ubuntu-latest - needs: [docker, test] + needs: [docker] steps: - name: Download image digests uses: actions/download-artifact@v4 diff --git a/.goreleaser.yml b/.goreleaser.yml index 903259d..3889b5c 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -8,6 +8,8 @@ builds: - id: autobrr env: - CGO_ENABLED=0 + flags: + - -pgo=cpu.pprof goos: - linux - windows diff --git a/ci.Dockerfile b/ci.Dockerfile index f5ff0b7..f21cc23 100644 --- a/ci.Dockerfile +++ b/ci.Dockerfile @@ -26,7 +26,7 @@ export GOARCH=$TARGETARCH; \ [[ "$GOARCH" == "arm" ]] && [[ "$TARGETVARIANT" == "v6" ]] && export GOARM=6; \ [[ "$GOARCH" == "arm" ]] && [[ "$TARGETVARIANT" == "v7" ]] && export GOARM=7; \ echo $GOARCH $GOOS $GOARM$GOAMD64; \ -go build -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${REVISION} -X main.date=${BUILDTIME}" -o /out/bin/autobrr cmd/autobrr/main.go && \ +go build -pgo=cpu.pprof -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${REVISION} -X main.date=${BUILDTIME}" -o /out/bin/autobrr cmd/autobrr/main.go && \ go build -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${REVISION} -X main.date=${BUILDTIME}" -o /out/bin/autobrrctl cmd/autobrrctl/main.go # build runner diff --git a/ciwindows.Dockerfile b/ciwindows.Dockerfile index d8d5bd1..c3c469c 100644 --- a/ciwindows.Dockerfile +++ b/ciwindows.Dockerfile @@ -26,7 +26,7 @@ export GOARCH=$TARGETARCH; \ [[ "$GOARCH" == "arm" ]] && [[ "$TARGETVARIANT" == "v6" ]] && export GOARM=6; \ [[ "$GOARCH" == "arm" ]] && [[ "$TARGETVARIANT" == "v7" ]] && export GOARM=7; \ echo $GOARCH $GOOS $GOARM$GOAMD64; \ -go build -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${REVISION} -X main.date=${BUILDTIME}" -o /out/bin/autobrr.exe cmd/autobrr/main.go && \ +go build -pgo=cpu.pprof -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${REVISION} -X main.date=${BUILDTIME}" -o /out/bin/autobrr.exe cmd/autobrr/main.go && \ go build -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${REVISION} -X main.date=${BUILDTIME}" -o /out/bin/autobrrctl.exe cmd/autobrrctl/main.go # build runner diff --git a/cmd/autobrr/main.go b/cmd/autobrr/main.go index bab486d..b36bf38 100644 --- a/cmd/autobrr/main.go +++ b/cmd/autobrr/main.go @@ -4,9 +4,12 @@ package main import ( + "log" "os" "os/signal" + "runtime/pprof" "syscall" + "time" _ "time/tzdata" "github.com/autobrr/autobrr/internal/action" @@ -47,10 +50,13 @@ var ( ) func main() { - var configPath string + var configPath, profilePath string pflag.StringVar(&configPath, "config", "", "path to configuration file") + pflag.StringVar(&profilePath, "pgo", "", "internal build flag") pflag.Parse() + shutdownFunc := pgoRun(profilePath) + // read config cfg := config.New(configPath, version) @@ -167,6 +173,11 @@ func main() { return } + if shutdownFunc != nil { + time.Sleep(5 * time.Second) + sigCh <- syscall.SIGQUIT + } + for sig := range sigCh { log.Info().Msgf("received signal: %v, shutting down server.", sig) @@ -174,8 +185,30 @@ func main() { if err := db.Close(); err != nil { log.Error().Err(err).Msg("failed to close the database connection properly") + shutdownFunc() os.Exit(1) } + shutdownFunc() os.Exit(0) } } + +func pgoRun(file string) func() { + if len(file) == 0 { + return nil + } + + f, err := os.Create(file) + if err != nil { + log.Fatalf("could not create CPU profile: %v", err) + } + + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatalf("could not create CPU profile: %v", err) + } + + return func() { + defer f.Close() + defer pprof.StopCPUProfile() + } +}