From 9e5b7b0aa5e69434dde36fe75153a35423bd56dd Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Tue, 12 Apr 2022 16:45:46 +0200 Subject: [PATCH] fix(indexer): panic on size check via api (#239) * fix(indexer): panic on size check via api * feat(indexer): add mock api --- internal/domain/indexer.go | 1 + internal/filter/service.go | 75 ++++++++++++++---------------------- internal/indexer/api.go | 26 +++++++------ internal/mock/indexer_api.go | 46 ++++++++++++++++++++++ test/definitions/mock.yaml | 1 + 5 files changed, 91 insertions(+), 58 deletions(-) create mode 100644 internal/mock/indexer_api.go diff --git a/internal/domain/indexer.go b/internal/domain/indexer.go index 5748b1f..d2768ea 100644 --- a/internal/domain/indexer.go +++ b/internal/domain/indexer.go @@ -123,6 +123,7 @@ func (t TorrentBasic) ReleaseSizeBytes() uint64 { releaseSizeBytes, err := humanize.ParseBytes(t.Size) if err != nil { // log could not parse into bytes + return 0 } return releaseSizeBytes } diff --git a/internal/filter/service.go b/internal/filter/service.go index a4f2080..29ec0f7 100644 --- a/internal/filter/service.go +++ b/internal/filter/service.go @@ -249,7 +249,7 @@ func (s *service) CheckFilter(f domain.Filter, release *domain.Release) (bool, e // additional size check. Some indexers have api implemented to fetch this data and for the others // it will download the torrent file to parse and make the size check. This is all to minimize the amount of downloads. - // do additional size check against indexer api or torrent for size + // do additional size check against indexer api or download torrent for size check if release.AdditionalSizeCheckRequired { log.Debug().Msgf("filter.Service.CheckFilter: (%v) additional size check required", f.Name) @@ -286,70 +286,51 @@ func (s *service) CheckFilter(f domain.Filter, release *domain.Release) (bool, e return false, nil } +// AdditionalSizeCheck +// Some indexers do not announce the size and if size (min,max) is set in a filter then it will need +// additional size check. Some indexers have api implemented to fetch this data and for the others +// it will download the torrent file to parse and make the size check. This is all to minimize the amount of downloads. func (s *service) AdditionalSizeCheck(f domain.Filter, release *domain.Release) (bool, error) { - // save outside of loop to check multiple filters with only one fetch - // TODO put on filter to reuse - var torrentInfo *domain.TorrentBasic - - // Some indexers do not announce the size and if size (min,max) is set in a filter then it will need - // additional size check. Some indexers have api implemented to fetch this data and for the others - // it will download the torrent file to parse and make the size check. This is all to minimize the amount of downloads. - // do additional size check against indexer api or torrent for size - log.Debug().Msgf("filter-service.find_and_check_filters: (%v) additional size check required", f.Name) + log.Debug().Msgf("filter.Service.AdditionalSizeCheck: (%v) additional size check required", f.Name) - // check if indexer = btn, ptp, ggn or red - if release.Indexer == "ptp" || release.Indexer == "btn" || release.Indexer == "ggn" || release.Indexer == "redacted" { - // fetch torrent info from api - // save outside of loop to check multiple filters with only one fetch - if torrentInfo == nil { + switch release.Indexer { + case "ptp", "btn", "ggn", "redacted", "mock": + if release.Size == 0 { + log.Trace().Msgf("filter.Service.AdditionalSizeCheck: (%v) preparing to check via api", f.Name) torrentInfo, err := s.apiService.GetTorrentByID(release.Indexer, release.TorrentID) if err != nil || torrentInfo == nil { - log.Error().Stack().Err(err).Msgf("filter-service.find_and_check_filters: (%v) could not get torrent: '%v' from: %v", f.Name, release.TorrentID, release.Indexer) + log.Error().Stack().Err(err).Msgf("filter.Service.AdditionalSizeCheck: (%v) could not get torrent info from api: '%v' from: %v", f.Name, release.TorrentID, release.Indexer) return false, err } - log.Debug().Msgf("filter-service.find_and_check_filters: (%v) got torrent info: %+v", f.Name, torrentInfo) + log.Debug().Msgf("filter.Service.AdditionalSizeCheck: (%v) got torrent info from api: %+v", f.Name, torrentInfo) + + release.Size = torrentInfo.ReleaseSizeBytes() } - // compare size against filters - match, err := checkSizeFilter(f.MinSize, f.MaxSize, torrentInfo.ReleaseSizeBytes()) - if err != nil { - log.Error().Stack().Err(err).Msgf("filter-service.find_and_check_filters: (%v) could not check size filter", f.Name) - return false, err - } - - // no match, lets continue to next filter - if !match { - log.Debug().Msgf("filter-service.find_and_check_filters: (%v) filter did not match after additional size check, trying next", f.Name) - return false, nil - } - - // store size on the release - release.Size = torrentInfo.ReleaseSizeBytes() - } else { - log.Trace().Msgf("filter-service.find_and_check_filters: (%v) additional size check required: preparing to download metafile", f.Name) + default: + log.Trace().Msgf("filter.Service.AdditionalSizeCheck: (%v) preparing to download torrent metafile", f.Name) // if indexer doesn't have api, download torrent and add to tmpPath err := release.DownloadTorrentFile() if err != nil { - log.Error().Stack().Err(err).Msgf("filter-service.find_and_check_filters: (%v) could not download torrent file with id: '%v' from: %v", f.Name, release.TorrentID, release.Indexer) + log.Error().Stack().Err(err).Msgf("filter.Service.AdditionalSizeCheck: (%v) could not download torrent file with id: '%v' from: %v", f.Name, release.TorrentID, release.Indexer) return false, err } + } - // compare size against filter - match, err := checkSizeFilter(f.MinSize, f.MaxSize, release.Size) - if err != nil { - log.Error().Stack().Err(err).Msgf("filter-service.find_and_check_filters: (%v) could not check size filter", f.Name) - return false, err - } - - // no match, lets continue to next filter - if !match { - log.Debug().Msgf("filter-service.find_and_check_filters: (%v) filter did not match after additional size check, trying next", f.Name) - return false, nil - } + // compare size against filter + match, err := checkSizeFilter(f.MinSize, f.MaxSize, release.Size) + if err != nil { + log.Error().Stack().Err(err).Msgf("filter.Service.AdditionalSizeCheck: (%v) error checking extra size filter", f.Name) + return false, err + } + //no match, lets continue to next filter + if !match { + log.Debug().Msgf("filter.Service.AdditionalSizeCheck: (%v) filter did not match after additional size check, trying next", f.Name) + return false, nil } return true, nil diff --git a/internal/indexer/api.go b/internal/indexer/api.go index 0e1427b..44b8c35 100644 --- a/internal/indexer/api.go +++ b/internal/indexer/api.go @@ -3,13 +3,14 @@ package indexer import ( "fmt" - "github.com/rs/zerolog/log" - "github.com/autobrr/autobrr/internal/domain" + "github.com/autobrr/autobrr/internal/mock" "github.com/autobrr/autobrr/pkg/btn" "github.com/autobrr/autobrr/pkg/ggn" "github.com/autobrr/autobrr/pkg/ptp" "github.com/autobrr/autobrr/pkg/red" + + "github.com/rs/zerolog/log" ) type APIService interface { @@ -70,50 +71,53 @@ func (s *apiService) TestConnection(indexer string) (bool, error) { func (s *apiService) AddClient(indexer string, settings map[string]string) error { // basic validation if indexer == "" { - return fmt.Errorf("api_service.add_client: validation falied: indexer can't be empty") + return fmt.Errorf("api.Service.AddClient: validation falied: indexer can't be empty") } else if len(settings) == 0 { - return fmt.Errorf("api_service.add_client: validation falied: settings can't be empty") + return fmt.Errorf("api.Service.AddClient: validation falied: settings can't be empty") } - log.Trace().Msgf("api-service.add_client: init api client for '%v'", indexer) + log.Trace().Msgf("api.Service.AddClient: init api client for '%v'", indexer) // init client switch indexer { case "btn": key, ok := settings["api_key"] if !ok || key == "" { - return fmt.Errorf("api_service: could not initialize btn client: missing var 'api_key'") + return fmt.Errorf("api.Service.AddClient: could not initialize btn client: missing var 'api_key'") } s.apiClients[indexer] = btn.NewClient("", key) case "ptp": user, ok := settings["api_user"] if !ok || user == "" { - return fmt.Errorf("api_service: could not initialize ptp client: missing var 'api_user'") + return fmt.Errorf("api.Service.AddClient: could not initialize ptp client: missing var 'api_user'") } key, ok := settings["api_key"] if !ok || key == "" { - return fmt.Errorf("api_service: could not initialize ptp client: missing var 'api_key'") + return fmt.Errorf("api.Service.AddClient: could not initialize ptp client: missing var 'api_key'") } s.apiClients[indexer] = ptp.NewClient("", user, key) case "ggn": key, ok := settings["api_key"] if !ok || key == "" { - return fmt.Errorf("api_service: could not initialize ggn client: missing var 'api_key'") + return fmt.Errorf("api.Service.AddClient: could not initialize ggn client: missing var 'api_key'") } s.apiClients[indexer] = ggn.NewClient("", key) case "redacted": key, ok := settings["api_key"] if !ok || key == "" { - return fmt.Errorf("api_service: could not initialize red client: missing var 'api_key'") + return fmt.Errorf("api.Service.AddClient: could not initialize red client: missing var 'api_key'") } s.apiClients[indexer] = red.NewClient("", key) + case "mock": + s.apiClients[indexer] = mock.NewMockClient("", "mock") + default: - return fmt.Errorf("api_service: could not initialize client: unsupported indexer '%v'", indexer) + return fmt.Errorf("api.Service.AddClient: could not initialize client: unsupported indexer '%v'", indexer) } return nil diff --git a/internal/mock/indexer_api.go b/internal/mock/indexer_api.go new file mode 100644 index 0000000..74faaad --- /dev/null +++ b/internal/mock/indexer_api.go @@ -0,0 +1,46 @@ +package mock + +import ( + "fmt" + + "github.com/autobrr/autobrr/internal/domain" +) + +type IndexerApiClient interface { + GetTorrentByID(torrentID string) (*domain.TorrentBasic, error) + TestAPI() (bool, error) +} + +type IndexerClient struct { + URL string + APIKey string +} + +func NewMockClient(url string, apiKey string) IndexerApiClient { + c := &IndexerClient{ + URL: url, + APIKey: apiKey, + } + + return c +} + +func (c *IndexerClient) GetTorrentByID(torrentID string) (*domain.TorrentBasic, error) { + if torrentID == "" { + return nil, fmt.Errorf("mock client: must have torrentID") + } + + r := &domain.TorrentBasic{ + Id: torrentID, + InfoHash: "", + Size: "10GB", + } + + return r, nil + +} + +// TestAPI try api access against torrents page +func (c *IndexerClient) TestAPI() (bool, error) { + return true, nil +} diff --git a/test/definitions/mock.yaml b/test/definitions/mock.yaml index 9841c5f..750741a 100644 --- a/test/definitions/mock.yaml +++ b/test/definitions/mock.yaml @@ -10,6 +10,7 @@ privacy: private protocol: torrent supports: - irc + - api source: custom settings: - name: rsskey