idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 00:39:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(filters): skip duplicates (#1711)

Browse source 
* feat(filters): skip duplicates

* fix: add interface instead of any

* fix(filters): tonullint

* feat(filters): skip dupes check month day

* chore: cleanup

* feat(db): set autoincrement id

* feat(filters): add repack and proper to dupe profile

* feat(filters): add default dupe profiles

* feat(duplicates): check audio and website

* feat(duplicates): update tests

* feat(duplicates): add toggles on addform

* feat(duplicates): fix sqlite upgrade path and initialize duplicate profiles

* feat(duplicates): simplify sqlite upgrade

avoiding temp table and unwieldy select.  Besides, FK constraints
are turned off anyway in #229.

* feat(duplicates): change CheckIsDuplicateRelease treatment of PROPER and REPACK

"Proper" and "Repack" are not parallel to the other conditions like "Title",
so they do not belong as dedup conditions.  "PROPER" means there was an issue in
the previous release, and so a PROPER is never a duplicate, even if it replaces
another PROPER.  Similarly, "REPACK" means there was an issue in the previous
release by that group, and so it is a duplicate only if we previously took a
release from a DIFFERENT group.

I have not removed Proper and Repack from the UI or the schema yet.

* feat(duplicates): update postgres schema to match sqlite

* feat(duplicates): fix web build errors

* feat(duplicates): fix postgres errors

* feat(filters): do leftjoin for duplicate profile

* fix(filters): partial update dupe profile

* go fmt `internal/domain/filter.go`

* feat(duplicates): restore straightforward logic for proper/repack

* feat(duplicates): remove mostly duplicate TV duplicate profiles

Having one profile seems the cleanest.  If somebody wants multiple
resolutions then they can add Resolution to the duplicate profile.
Tested this profile with both weekly episodic releases and daily
show releases.

* feat(release): add db indexes and sub_title

* feat(release): add IsDuplicate tests

* feat(release): update action handler

* feat(release): add more tests for skip duplicates

* feat(duplicates): check audio

* feat(duplicates): add more tests

* feat(duplicates): match edition cut and more

* fix(duplicates): tests

* fix(duplicates): missing imports

* fix(duplicates): tests

* feat(duplicates): handle sub_title edition and language in ui

* fix(duplicates): tests

* feat(duplicates): check name against normalized hash

* fix(duplicates): tests

* chore: update .gitignore to ignore .pnpm-store

* fix: tests

* fix(filters): tests

* fix: bad conflict merge

* fix: update release type in test

* fix: use vendored hot-toast

* fix: release_test.go

* fix: rss_test.go

* feat(duplicates): improve title hashing for unique check

* feat(duplicates): further improve title hashing for unique check with lang

* feat(duplicates): fix tests

* feat(duplicates): add macros IsDuplicate and DuplicateProfile ID and name

* feat(duplicates): add normalized hash match option

* fix: headlessui-state prop warning

* fix(duplicates): add missing year in daily ep normalize

* fix(duplicates): check rejections len

---------

Co-authored-by: ze0s <ze0s@riseup.net>
This commit is contained in:
kenstir 2024-12-25 16:33:46 -05:00 • committed by GitHub
parent d153ac44b8
commit 4009554d10
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 3792 additions and 743 deletions
Download patch file Download diff file Expand all files Collapse all files

77
internal/release/service.go
View file

@ -32,6 +32,10 @@ type Service interface {
ProcessMultiple(releases []*domain.Release)
ProcessManual(ctx context.Context, req *domain.ReleaseProcessReq) error
Retry(ctx context.Context, req *domain.ReleaseActionRetryReq) error
StoreReleaseProfileDuplicate(ctx context.Context, profile *domain.DuplicateReleaseProfile) error
FindDuplicateReleaseProfiles(ctx context.Context) ([]*domain.DuplicateReleaseProfile, error)
DeleteReleaseProfileDuplicate(ctx context.Context, id int64) error
}
type actionClientTypeKey struct {
@ -94,6 +98,18 @@ func (s *service) Delete(ctx context.Context, req *domain.DeleteReleaseRequest)
return s.repo.Delete(ctx, req)
}
func (s *service) FindDuplicateReleaseProfiles(ctx context.Context) ([]*domain.DuplicateReleaseProfile, error) {
return s.repo.FindDuplicateReleaseProfiles(ctx)
}
func (s *service) StoreReleaseProfileDuplicate(ctx context.Context, profile *domain.DuplicateReleaseProfile) error {
return s.repo.StoreDuplicateProfile(ctx, profile)
}
func (s *service) DeleteReleaseProfileDuplicate(ctx context.Context, id int64) error {
return s.repo.DeleteReleaseProfileDuplicate(ctx, id)
}
func (s *service) ProcessManual(ctx context.Context, req *domain.ReleaseProcessReq) error {
// get indexer definition with data
def, err := s.indexerSvc.GetMappedDefinitionByName(req.IndexerIdentifier)
@ -183,8 +199,6 @@ func (s *service) Process(release *domain.Release) {
s.log.Error().Err(err).Msgf("release.Process: error processing filters for indexer: %s", release.Indexer.Name)
return
}
return
}
func (s *service) processFilters(ctx context.Context, filters []*domain.Filter, release *domain.Release) error {
@ -201,6 +215,11 @@ func (s *service) processFilters(ctx context.Context, filters []*domain.Filter,
release.FilterName = f.Name
release.FilterID = f.ID
// reset IsDuplicate
release.IsDuplicate = false
release.SkipDuplicateProfileID = 0
release.SkipDuplicateProfileName = ""
// test filter
match, err := s.filterSvc.CheckFilter(ctx, f, release)
if err != nil {
@ -208,10 +227,10 @@ func (s *service) processFilters(ctx context.Context, filters []*domain.Filter,
return err
}
if !match {
if !match || f.RejectReasons.Len() > 0 {
l.Trace().Msgf("release.Process: indexer: %s, filter: %s release: %s, no match. rejections: %s", release.Indexer.Name, release.FilterName, release.TorrentName, f.RejectReasons.String())
l.Debug().Msgf("filter %s rejected release: %s", f.Name, release.TorrentName)
l.Debug().Msgf("filter %s rejected release: %s with reasons: %s", f.Name, release.TorrentName, f.RejectReasons.String())
continue
}
@ -231,13 +250,6 @@ func (s *service) processFilters(ctx context.Context, filters []*domain.Filter,
continue
}
// sleep for the delay period specified in the filter before running actions
delay := release.Filter.Delay
if delay > 0 {
l.Debug().Msgf("release.Process: delaying processing of '%s' (%s) for %s by %d seconds as specified in the filter", release.TorrentName, release.FilterName, release.Indexer.Name, delay)
time.Sleep(time.Duration(delay) * time.Second)
}
// save release here to only save those with rejections from actions instead of all releases
if release.ID == 0 {
release.FilterStatus = domain.ReleaseStatusFilterApproved
@ -251,24 +263,40 @@ func (s *service) processFilters(ctx context.Context, filters []*domain.Filter,
var rejections []string
// run actions (watchFolder, test, exec, qBittorrent, Deluge, arr etc.)
for _, act := range actions {
for idx, act := range actions {
// only run enabled actions
if !act.Enabled {
l.Trace().Msgf("release.Process: indexer: %s, filter: %s release: %s action '%s' not enabled, skip", release.Indexer.Name, release.FilterName, release.TorrentName, act.Name)
continue
}
// add action status as pending
actionStatus := domain.NewReleaseActionStatus(act, release)
if err := s.StoreReleaseActionStatus(ctx, actionStatus); err != nil {
s.log.Error().Err(err).Msgf("release.runAction: error storing action for filter: %s", release.FilterName)
}
if idx == 0 {
// sleep for the delay period specified in the filter before running actions
delay := release.Filter.Delay
if delay > 0 {
l.Debug().Msgf("release.Process: delaying processing of '%s' (%s) for %s by %d seconds as specified in the filter", release.TorrentName, release.FilterName, release.Indexer.Name, delay)
time.Sleep(time.Duration(delay) * time.Second)
}
}
l.Trace().Msgf("release.Process: indexer: %s, filter: %s release: %s , run action: %s", release.Indexer.Name, release.FilterName, release.TorrentName, act.Name)
// keep track of action clients to avoid sending the same thing all over again
_, tried := triedActionClients[actionClientTypeKey{Type: act.Type, ClientID: act.ClientID}]
if tried {
l.Trace().Msgf("release.Process: indexer: %s, filter: %s release: %s action client already tried, skip", release.Indexer.Name, release.FilterName, release.TorrentName)
l.Debug().Msgf("release.Process: indexer: %s, filter: %s release: %s action client already tried, skip", release.Indexer.Name, release.FilterName, release.TorrentName)
continue
}
// run action
status, err := s.runAction(ctx, act, release)
status, err := s.runAction(ctx, act, release, actionStatus)
if err != nil {
l.Error().Err(err).Msgf("release.Process: error running actions for filter: %s", release.FilterName)
//continue
@ -320,13 +348,13 @@ func (s *service) ProcessMultiple(releases []*domain.Release) {
}
}
func (s *service) runAction(ctx context.Context, action *domain.Action, release *domain.Release) (*domain.ReleaseActionStatus, error) {
func (s *service) runAction(ctx context.Context, action *domain.Action, release *domain.Release, status *domain.ReleaseActionStatus) (*domain.ReleaseActionStatus, error) {
// add action status as pending
status := domain.NewReleaseActionStatus(action, release)
if err := s.StoreReleaseActionStatus(ctx, status); err != nil {
s.log.Error().Err(err).Msgf("release.runAction: error storing action for filter: %s", release.FilterName)
}
//status := domain.NewReleaseActionStatus(action, release)
//
//if err := s.StoreReleaseActionStatus(ctx, status); err != nil {
// s.log.Error().Err(err).Msgf("release.runAction: error storing action for filter: %s", release.FilterName)
//}
rejections, err := s.actionSvc.RunAction(ctx, action, release)
if err != nil {
@ -351,7 +379,14 @@ func (s *service) runAction(ctx context.Context, action *domain.Action, release
}
func (s *service) retryAction(ctx context.Context, action *domain.Action, release *domain.Release) error {
actionStatus, err := s.runAction(ctx, action, release)
// add action status as pending
status := domain.NewReleaseActionStatus(action, release)
if err := s.StoreReleaseActionStatus(ctx, status); err != nil {
s.log.Error().Err(err).Msgf("release.runAction: error storing action for filter: %s", release.FilterName)
}
actionStatus, err := s.runAction(ctx, action, release, status)
if err != nil {
s.log.Error().Err(err).Msgf("release.retryAction: error running actions for filter: %s", release.FilterName)