diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 483dda9..cb40c38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ permissions: jobs: web: name: Build web - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v3 @@ -44,7 +44,7 @@ jobs: goreleaserbuild: name: Build Go binaries if: github.event_name == 'pull_request' - runs-on: ubuntu-latest + runs-on: self-hosted needs: web steps: - name: Checkout @@ -82,7 +82,7 @@ jobs: goreleaser: name: Build & publish binaries and images if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest + runs-on: self-hosted needs: web steps: - name: Checkout diff --git a/internal/action/deluge.go b/internal/action/deluge.go index 0ce2d37..ae9f0d5 100644 --- a/internal/action/deluge.go +++ b/internal/action/deluge.go @@ -117,7 +117,7 @@ func (s *service) delugeV1(client *domain.DownloadClient, action domain.Action, } if release.TorrentTmpFile == "" { - if err = release.DownloadTorrentFile(); err != nil { + if err := release.DownloadTorrentFile(); err != nil { s.log.Error().Err(err).Msgf("could not download torrent file for release: %v", release.TorrentName) return nil, err } @@ -206,7 +206,7 @@ func (s *service) delugeV2(client *domain.DownloadClient, action domain.Action, } if release.TorrentTmpFile == "" { - if err = release.DownloadTorrentFile(); err != nil { + if err := release.DownloadTorrentFile(); err != nil { s.log.Error().Err(err).Msgf("could not download torrent file for release: %v", release.TorrentName) return nil, err } diff --git a/internal/action/qbittorrent.go b/internal/action/qbittorrent.go index fa431f7..a530601 100644 --- a/internal/action/qbittorrent.go +++ b/internal/action/qbittorrent.go @@ -68,8 +68,7 @@ func (s *service) qbittorrent(action domain.Action, release domain.Release) ([]s } if release.TorrentTmpFile == "" { - err = release.DownloadTorrentFile() - if err != nil { + if err := release.DownloadTorrentFile(); err != nil { return nil, errors.Wrap(err, "error downloading torrent file for release: %v", release.TorrentName) } } @@ -279,11 +278,12 @@ func (s *service) reannounceTorrent(qb *qbittorrent.Client, action domain.Action // Check if status not working or something else // https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1)#get-torrent-trackers -// 0 Tracker is disabled (used for DHT, PeX, and LSD) -// 1 Tracker has not been contacted yet -// 2 Tracker has been contacted and is working -// 3 Tracker is updating -// 4 Tracker has been contacted, but it is not working (or doesn't send proper replies) +// +// 0 Tracker is disabled (used for DHT, PeX, and LSD) +// 1 Tracker has not been contacted yet +// 2 Tracker has been contacted and is working +// 3 Tracker is updating +// 4 Tracker has been contacted, but it is not working (or doesn't send proper replies) func isTrackerStatusOK(trackers []qbittorrent.TorrentTracker) bool { for _, tracker := range trackers { if tracker.Status == qbittorrent.TrackerStatusDisabled { diff --git a/internal/action/rtorrent.go b/internal/action/rtorrent.go index 97ef720..2ce44aa 100644 --- a/internal/action/rtorrent.go +++ b/internal/action/rtorrent.go @@ -28,7 +28,7 @@ func (s *service) rtorrent(action domain.Action, release domain.Release) ([]stri var rejections []string if release.TorrentTmpFile == "" { - if err = release.DownloadTorrentFile(); err != nil { + if err := release.DownloadTorrentFile(); err != nil { s.log.Error().Err(err).Msgf("could not download torrent file for release: %v", release.TorrentName) return nil, err } diff --git a/internal/action/transmission.go b/internal/action/transmission.go index 4ba9a8b..fb6b3d5 100644 --- a/internal/action/transmission.go +++ b/internal/action/transmission.go @@ -28,8 +28,7 @@ func (s *service) transmission(action domain.Action, release domain.Release) ([] var rejections []string if release.TorrentTmpFile == "" { - err = release.DownloadTorrentFile() - if err != nil { + if err := release.DownloadTorrentFile(); err != nil { s.log.Error().Err(err).Msgf("could not download torrent file for release: %v", release.TorrentName) return nil, err } diff --git a/internal/client/http.go b/internal/client/http.go index 6e55b70..27572cb 100644 --- a/internal/client/http.go +++ b/internal/client/http.go @@ -1,14 +1,15 @@ package client import ( - "errors" "io" "net/http" "os" "time" - "github.com/anacrolix/torrent/metainfo" + "github.com/autobrr/autobrr/pkg/errors" + "github.com/anacrolix/torrent/metainfo" + "github.com/avast/retry-go" "github.com/rs/zerolog/log" ) @@ -35,58 +36,6 @@ func NewHttpClient() *HttpClient { } } -func (c *HttpClient) DownloadFile(url string, opts map[string]string) (*DownloadFileResponse, error) { - if url == "" { - return nil, errors.New("download_file: url can't be empty") - } - - // Create tmp file - tmpFile, err := os.CreateTemp("", "autobrr-") - if err != nil { - log.Error().Stack().Err(err).Msg("error creating temp file") - return nil, err - } - defer tmpFile.Close() - - // Get the data - resp, err := http.Get(url) - if err != nil { - log.Error().Stack().Err(err).Msgf("error downloading file from %v", url) - return nil, err - } - defer resp.Body.Close() - - // retry logic - - if resp.StatusCode != http.StatusOK { - log.Error().Stack().Err(err).Msgf("error downloading file from: %v - bad status: %d", url, resp.StatusCode) - return nil, err - } - - // Write the body to file - _, err = io.Copy(tmpFile, resp.Body) - if err != nil { - log.Error().Stack().Err(err).Msgf("error writing downloaded file: %v", tmpFile.Name()) - return nil, err - } - - // remove file if fail - - res := DownloadFileResponse{ - Body: &resp.Body, - FileName: tmpFile.Name(), - } - - if res.FileName == "" || res.Body == nil { - log.Error().Stack().Err(err).Msgf("tmp file error - empty body: %v", url) - return nil, errors.New("error downloading file, no tmp file") - } - - log.Debug().Msgf("successfully downloaded file: %v", tmpFile.Name()) - - return &res, nil -} - func (c *HttpClient) DownloadTorrentFile(url string, opts map[string]string) (*DownloadTorrentFileResponse, error) { if url == "" { return nil, errors.New("download_file: url can't be empty") @@ -100,47 +49,63 @@ func (c *HttpClient) DownloadTorrentFile(url string, opts map[string]string) (*D } defer tmpFile.Close() - // Get the data - resp, err := http.Get(url) + res := &DownloadTorrentFileResponse{} + // try request and if fail run 3 retries + err = retry.Do(func() error { + resp, err := http.Get(url) + if err != nil { + return errors.New("error downloading file: %q", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errors.New("error downloading file bad status: %d", resp.StatusCode) + } + + nuke := func() { + tmpFile.Seek(0, io.SeekStart) + tmpFile.Truncate(0) + } + + // Write the body to file + _, err = io.Copy(tmpFile, resp.Body) + if err != nil { + nuke() + return errors.New("error writing downloaded file: %v | %q", tmpFile.Name(), err) + } + + meta, err := metainfo.Load(resp.Body) + if err != nil { + nuke() + return errors.New("metainfo could not load file contents: %v | %q", tmpFile.Name(), err) + } + + res = &DownloadTorrentFileResponse{ + MetaInfo: meta, + TmpFileName: tmpFile.Name(), + } + + if res.TmpFileName == "" || res.MetaInfo == nil { + nuke() + return errors.New("tmp file error - empty body") + } + + if len(res.MetaInfo.InfoBytes) < 1 { + nuke() + return errors.New("could not read infohash") + } + + log.Debug().Msgf("successfully downloaded file: %v", tmpFile.Name()) + return nil + }, + //retry.OnRetry(func(n uint, err error) { c.log.Printf("%q: attempt %d - %v\n", err, n, url) }), + retry.Delay(time.Second*5), + retry.Attempts(3), + retry.MaxJitter(time.Second*1)) + if err != nil { - log.Error().Stack().Err(err).Msgf("error downloading file from %v", url) - return nil, err - } - defer resp.Body.Close() - - // retry logic - - if resp.StatusCode != http.StatusOK { - log.Error().Stack().Err(err).Msgf("error downloading file from: %v - bad status: %d", url, resp.StatusCode) - return nil, err + res = nil } - // Write the body to file - _, err = io.Copy(tmpFile, resp.Body) - if err != nil { - log.Error().Stack().Err(err).Msgf("error writing downloaded file: %v", tmpFile.Name()) - return nil, err - } - - meta, err := metainfo.Load(resp.Body) - if err != nil { - log.Error().Stack().Err(err).Msgf("metainfo could not load file contents: %v", tmpFile.Name()) - return nil, err - } - - // remove file if fail - - res := DownloadTorrentFileResponse{ - MetaInfo: meta, - TmpFileName: tmpFile.Name(), - } - - if res.TmpFileName == "" || res.MetaInfo == nil { - log.Error().Stack().Err(err).Msgf("tmp file error - empty body: %v", url) - return nil, errors.New("error downloading file, no tmp file") - } - - log.Debug().Msgf("successfully downloaded file: %v", tmpFile.Name()) - - return &res, nil + return res, err } diff --git a/internal/domain/release.go b/internal/domain/release.go index 23b8ec4..4038d97 100644 --- a/internal/domain/release.go +++ b/internal/domain/release.go @@ -17,6 +17,7 @@ import ( "github.com/autobrr/autobrr/pkg/errors" "github.com/anacrolix/torrent/metainfo" + "github.com/avast/retry-go" "github.com/dustin/go-humanize" "github.com/moistari/rls" "golang.org/x/net/publicsuffix" @@ -222,6 +223,8 @@ func (r *Release) ParseString(title string) { return } +var ErrUnrecoverableError = errors.New("unrecoverable error") + func (r *Release) ParseReleaseTagsString(tags string) { // trim delimiters and closest space re := regexp.MustCompile(`\| |/ |, `) @@ -292,9 +295,10 @@ func (r *Release) DownloadTorrentFile() error { client := &http.Client{ Transport: customTransport, Jar: jar, + Timeout: time.Second * 45, } - req, err := http.NewRequest("GET", r.TorrentURL, nil) + req, err := http.NewRequest(http.MethodGet, r.TorrentURL, nil) if err != nil { return errors.Wrap(err, "error downloading file") } @@ -305,19 +309,6 @@ func (r *Release) DownloadTorrentFile() error { req.Header.Set("Cookie", r.RawCookie) } - // Get the data - resp, err := client.Do(req) - if err != nil { - return errors.Wrap(err, "error downloading file") - } - defer resp.Body.Close() - - // retry logic - - if resp.StatusCode != http.StatusOK { - return errors.New("error downloading torrent (%v) file (%v) from '%v' - status code: %d", r.TorrentName, r.TorrentURL, r.Indexer, resp.StatusCode) - } - // Create tmp file tmpFile, err := os.CreateTemp("", "autobrr-") if err != nil { @@ -325,29 +316,65 @@ func (r *Release) DownloadTorrentFile() error { } defer tmpFile.Close() - // Write the body to file - _, err = io.Copy(tmpFile, resp.Body) - if err != nil { - return errors.Wrap(err, "error writing downloaded file: %v", tmpFile.Name()) - } + errFunc := retry.Do(func() error { + // Get the data + resp, err := client.Do(req) + if err != nil { + return errors.Wrap(err, "error downloading file") + } + defer resp.Body.Close() - meta, err := metainfo.LoadFromFile(tmpFile.Name()) - if err != nil { - return errors.Wrap(err, "metainfo could not load file contents: %v", tmpFile.Name()) - } + if resp.StatusCode != http.StatusOK { + unRecoverableErr := errors.Wrap(ErrUnrecoverableError, "unrecoverable error downloading torrent (%v) file (%v) from '%v' - status code: %d", r.TorrentName, r.TorrentURL, r.Indexer, resp.StatusCode) - torrentMetaInfo, err := meta.UnmarshalInfo() - if err != nil { - return errors.Wrap(err, "metainfo could not unmarshal info from torrent: %v", tmpFile.Name()) - } + if resp.StatusCode == 401 || resp.StatusCode == 403 || resp.StatusCode == 404 || resp.StatusCode == 405 { + return retry.Unrecoverable(unRecoverableErr) + } - r.TorrentTmpFile = tmpFile.Name() - r.TorrentHash = meta.HashInfoBytes().String() - r.Size = uint64(torrentMetaInfo.TotalLength()) + return errors.New("unexpected status: %v", resp.StatusCode) + } - // remove file if fail + resetTmpFile := func() { + tmpFile.Seek(0, io.SeekStart) + tmpFile.Truncate(0) + } - return nil + // Write the body to file + if _, err := io.Copy(tmpFile, resp.Body); err != nil { + resetTmpFile() + return errors.Wrap(err, "error writing downloaded file: %v", tmpFile.Name()) + } + + meta, err := metainfo.LoadFromFile(tmpFile.Name()) + if err != nil { + resetTmpFile() + return errors.Wrap(err, "metainfo could not load file contents: %v", tmpFile.Name()) + } + + torrentMetaInfo, err := meta.UnmarshalInfo() + if err != nil { + resetTmpFile() + return errors.Wrap(err, "metainfo could not unmarshal info from torrent: %v", tmpFile.Name()) + } + + hashInfoBytes := meta.HashInfoBytes().Bytes() + if len(hashInfoBytes) < 1 { + resetTmpFile() + return errors.New("could not read infohash") + } + + r.TorrentTmpFile = tmpFile.Name() + r.TorrentHash = meta.HashInfoBytes().String() + r.Size = uint64(torrentMetaInfo.TotalLength()) + + return nil + }, + retry.Delay(time.Second*3), + retry.Attempts(3), + retry.MaxJitter(time.Second*1), + ) + + return errFunc } func (r *Release) addRejection(reason string) { diff --git a/internal/domain/release_test.go b/internal/domain/release_test.go index 1285a45..b5a3649 100644 --- a/internal/domain/release_test.go +++ b/internal/domain/release_test.go @@ -1,8 +1,15 @@ package domain import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" "testing" + "time" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" ) @@ -630,3 +637,212 @@ func TestRelease_ParseString(t *testing.T) { }) } } + +var trackerLessTestTorrent = `d7:comment19:This is just a test10:created by12:Johnny Bravo13:creation datei1430648794e8:encoding5:UTF-84:infod6:lengthi1128e4:name12:testfile.bin12:piece lengthi32768e6:pieces20:Õˆë =‘UŒäiÎ^æ °Eâ?ÇÒe5:nodesl35:udp://tracker.openbittorrent.com:8035:udp://tracker.openbittorrent.com:80ee` + +func TestRelease_DownloadTorrentFile(t *testing.T) { + // disable logger + zerolog.SetGlobalLevel(zerolog.Disabled) + + mux := http.NewServeMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.RequestURI, "401") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("unauthorized")) + return + } + if strings.Contains(r.RequestURI, "403") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("forbidden")) + return + } + if strings.Contains(r.RequestURI, "404") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("not found")) + return + } + if strings.Contains(r.RequestURI, "405") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("method not allowed")) + return + } + + if strings.Contains(r.RequestURI, "file.torrent") { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/x-bittorrent") + payload, _ := os.ReadFile("testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent") + w.Write(payload) + return + } + + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("internal error")) + }) + + type fields struct { + ID int64 + FilterStatus ReleaseFilterStatus + Rejections []string + Indexer string + FilterName string + Protocol ReleaseProtocol + Implementation ReleaseImplementation + Timestamp time.Time + GroupID string + TorrentID string + TorrentURL string + TorrentTmpFile string + TorrentDataRawBytes []byte + TorrentHash string + TorrentName string + Size uint64 + Title string + Category string + Categories []string + Season int + Episode int + Year int + Resolution string + Source string + Codec []string + Container string + HDR []string + Audio []string + AudioChannels string + Group string + Region string + Language string + Proper bool + Repack bool + Website string + Artists string + Type string + LogScore int + IsScene bool + Origin string + Tags []string + ReleaseTags string + Freeleech bool + FreeleechPercent int + Bonus []string + Uploader string + PreTime string + Other []string + RawCookie string + AdditionalSizeCheckRequired bool + FilterID int + Filter *Filter + ActionStatus []ReleaseActionStatus + } + tests := []struct { + name string + fields fields + wantErr assert.ErrorAssertionFunc + }{ + { + name: "401", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + TorrentURL: fmt.Sprintf("%v/%v", ts.URL, 401), + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + if err != nil { + return true + } + return false + }, + }, + { + name: "403", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + TorrentURL: fmt.Sprintf("%v/%v", ts.URL, 403), + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + if err != nil { + return true + } + return false + }, + }, + { + name: "ok", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + TorrentURL: fmt.Sprintf("%v/%v", ts.URL, "file.torrent"), + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + if err != nil { + return true + } + return false + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Release{ + ID: tt.fields.ID, + FilterStatus: tt.fields.FilterStatus, + Rejections: tt.fields.Rejections, + Indexer: tt.fields.Indexer, + FilterName: tt.fields.FilterName, + Protocol: tt.fields.Protocol, + Implementation: tt.fields.Implementation, + Timestamp: tt.fields.Timestamp, + GroupID: tt.fields.GroupID, + TorrentID: tt.fields.TorrentID, + TorrentURL: tt.fields.TorrentURL, + TorrentTmpFile: tt.fields.TorrentTmpFile, + TorrentDataRawBytes: tt.fields.TorrentDataRawBytes, + TorrentHash: tt.fields.TorrentHash, + TorrentName: tt.fields.TorrentName, + Size: tt.fields.Size, + Title: tt.fields.Title, + Category: tt.fields.Category, + Categories: tt.fields.Categories, + Season: tt.fields.Season, + Episode: tt.fields.Episode, + Year: tt.fields.Year, + Resolution: tt.fields.Resolution, + Source: tt.fields.Source, + Codec: tt.fields.Codec, + Container: tt.fields.Container, + HDR: tt.fields.HDR, + Audio: tt.fields.Audio, + AudioChannels: tt.fields.AudioChannels, + Group: tt.fields.Group, + Region: tt.fields.Region, + Language: tt.fields.Language, + Proper: tt.fields.Proper, + Repack: tt.fields.Repack, + Website: tt.fields.Website, + Artists: tt.fields.Artists, + Type: tt.fields.Type, + LogScore: tt.fields.LogScore, + IsScene: tt.fields.IsScene, + Origin: tt.fields.Origin, + Tags: tt.fields.Tags, + ReleaseTags: tt.fields.ReleaseTags, + Freeleech: tt.fields.Freeleech, + FreeleechPercent: tt.fields.FreeleechPercent, + Bonus: tt.fields.Bonus, + Uploader: tt.fields.Uploader, + PreTime: tt.fields.PreTime, + Other: tt.fields.Other, + RawCookie: tt.fields.RawCookie, + AdditionalSizeCheckRequired: tt.fields.AdditionalSizeCheckRequired, + FilterID: tt.fields.FilterID, + Filter: tt.fields.Filter, + ActionStatus: tt.fields.ActionStatus, + } + tt.wantErr(t, r.DownloadTorrentFile(), fmt.Sprintf("DownloadTorrentFile()")) + }) + } +} diff --git a/internal/domain/testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent b/internal/domain/testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent new file mode 100644 index 0000000..9ce7748 Binary files /dev/null and b/internal/domain/testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent differ diff --git a/internal/filter/service.go b/internal/filter/service.go index b77301d..500bad4 100644 --- a/internal/filter/service.go +++ b/internal/filter/service.go @@ -409,8 +409,7 @@ func (s *service) AdditionalSizeCheck(f domain.Filter, release *domain.Release) s.log.Trace().Msgf("filter.Service.AdditionalSizeCheck: (%v) preparing to download torrent metafile", f.Name) // if indexer doesn't have api, download torrent and add to tmpPath - err := release.DownloadTorrentFile() - if err != nil { + if err := release.DownloadTorrentFile(); err != nil { s.log.Error().Stack().Err(err).Msgf("filter.Service.AdditionalSizeCheck: (%v) could not download torrent file with id: '%v' from: %v", f.Name, release.TorrentID, release.Indexer) return false, err }