feat(filters): skip duplicates (#1711)

* feat(filters): skip duplicates

* fix: add interface instead of any

* fix(filters): tonullint

* feat(filters): skip dupes check month day

* chore: cleanup

* feat(db): set autoincrement id

* feat(filters): add repack and proper to dupe profile

* feat(filters): add default dupe profiles

* feat(duplicates): check audio and website

* feat(duplicates): update tests

* feat(duplicates): add toggles on addform

* feat(duplicates): fix sqlite upgrade path and initialize duplicate profiles

* feat(duplicates): simplify sqlite upgrade

avoiding temp table and unwieldy select.  Besides, FK constraints
are turned off anyway in #229.

* feat(duplicates): change CheckIsDuplicateRelease treatment of PROPER and REPACK

"Proper" and "Repack" are not parallel to the other conditions like "Title",
so they do not belong as dedup conditions.  "PROPER" means there was an issue in
the previous release, and so a PROPER is never a duplicate, even if it replaces
another PROPER.  Similarly, "REPACK" means there was an issue in the previous
release by that group, and so it is a duplicate only if we previously took a
release from a DIFFERENT group.

I have not removed Proper and Repack from the UI or the schema yet.

* feat(duplicates): update postgres schema to match sqlite

* feat(duplicates): fix web build errors

* feat(duplicates): fix postgres errors

* feat(filters): do leftjoin for duplicate profile

* fix(filters): partial update dupe profile

* go fmt `internal/domain/filter.go`

* feat(duplicates): restore straightforward logic for proper/repack

* feat(duplicates): remove mostly duplicate TV duplicate profiles

Having one profile seems the cleanest.  If somebody wants multiple
resolutions then they can add Resolution to the duplicate profile.
Tested this profile with both weekly episodic releases and daily
show releases.

* feat(release): add db indexes and sub_title

* feat(release): add IsDuplicate tests

* feat(release): update action handler

* feat(release): add more tests for skip duplicates

* feat(duplicates): check audio

* feat(duplicates): add more tests

* feat(duplicates): match edition cut and more

* fix(duplicates): tests

* fix(duplicates): missing imports

* fix(duplicates): tests

* feat(duplicates): handle sub_title edition and language in ui

* fix(duplicates): tests

* feat(duplicates): check name against normalized hash

* fix(duplicates): tests

* chore: update .gitignore to ignore .pnpm-store

* fix: tests

* fix(filters): tests

* fix: bad conflict merge

* fix: update release type in test

* fix: use vendored hot-toast

* fix: release_test.go

* fix: rss_test.go

* feat(duplicates): improve title hashing for unique check

* feat(duplicates): further improve title hashing for unique check with lang

* feat(duplicates): fix tests

* feat(duplicates): add macros IsDuplicate and DuplicateProfile ID and name

* feat(duplicates): add normalized hash match option

* fix: headlessui-state prop warning

* fix(duplicates): add missing year in daily ep normalize

* fix(duplicates): check rejections len

---------

Co-authored-by: ze0s <ze0s@riseup.net>
This commit is contained in:
kenstir 2024-12-25 16:33:46 -05:00 committed by GitHub
parent d153ac44b8
commit 4009554d10
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 3792 additions and 743 deletions

View file

@ -13,6 +13,7 @@ import (
"github.com/autobrr/autobrr/internal/domain"
"github.com/moistari/rls"
"github.com/stretchr/testify/assert"
)
@ -49,12 +50,13 @@ func getMockRelease() *domain.Release {
Proper: true,
Repack: false,
Website: "https://example.com",
Type: "Movie",
Type: rls.Movie,
Origin: "P2P",
Tags: []string{"Action", "Adventure"},
Uploader: "john_doe",
PreTime: "10m",
FilterID: 1,
Other: []string{},
}
}
@ -108,11 +110,11 @@ func TestReleaseRepo_Store(t *testing.T) {
// Execute
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -123,7 +125,7 @@ func TestReleaseRepo_Store(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -164,11 +166,11 @@ func TestReleaseRepo_StoreReleaseActionStatus(t *testing.T) {
// Execute
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -179,7 +181,7 @@ func TestReleaseRepo_StoreReleaseActionStatus(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -328,11 +330,11 @@ func TestReleaseRepo_GetIndexerOptions(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -347,7 +349,7 @@ func TestReleaseRepo_GetIndexerOptions(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -387,11 +389,11 @@ func TestReleaseRepo_GetActionStatusByReleaseID(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -407,7 +409,7 @@ func TestReleaseRepo_GetActionStatusByReleaseID(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -447,11 +449,11 @@ func TestReleaseRepo_Get(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -467,7 +469,7 @@ func TestReleaseRepo_Get(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -507,11 +509,11 @@ func TestReleaseRepo_Stats(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -527,7 +529,7 @@ func TestReleaseRepo_Stats(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -567,11 +569,11 @@ func TestReleaseRepo_Delete(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -584,7 +586,7 @@ func TestReleaseRepo_Delete(t *testing.T) {
assert.NoError(t, err)
// Cleanup
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -624,11 +626,11 @@ func TestReleaseRepo_CheckSmartEpisodeCanDownloadShow(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -652,9 +654,724 @@ func TestReleaseRepo_CheckSmartEpisodeCanDownloadShow(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
}
}
func getMockDuplicateReleaseProfileTV() *domain.DuplicateReleaseProfile {
return &domain.DuplicateReleaseProfile{
ID: 0,
Name: "TV",
Protocol: false,
ReleaseName: false,
Hash: false,
Title: true,
SubTitle: false,
Year: false,
Month: false,
Day: false,
Source: false,
Resolution: false,
Codec: false,
Container: false,
DynamicRange: false,
Audio: false,
Group: false,
Season: true,
Episode: true,
Website: false,
Proper: false,
Repack: false,
Edition: false,
Language: false,
}
}
func getMockDuplicateReleaseProfileTVDaily() *domain.DuplicateReleaseProfile {
return &domain.DuplicateReleaseProfile{
ID: 0,
Name: "TV",
Protocol: false,
ReleaseName: false,
Hash: false,
Title: true,
SubTitle: false,
Year: true,
Month: true,
Day: true,
Source: false,
Resolution: false,
Codec: false,
Container: false,
DynamicRange: false,
Audio: false,
Group: false,
Season: false,
Episode: false,
Website: false,
Proper: false,
Repack: false,
Edition: false,
Language: false,
}
}
func getMockFilterDuplicates() *domain.Filter {
return &domain.Filter{
Name: "New Filter",
Enabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
MinSize: "10mb",
MaxSize: "20mb",
Delay: 60,
Priority: 1,
MaxDownloads: 100,
MaxDownloadsUnit: domain.FilterMaxDownloadsHour,
MatchReleases: "BRRip",
ExceptReleases: "BRRip",
UseRegex: false,
MatchReleaseGroups: "AMIABLE",
ExceptReleaseGroups: "NTb",
Scene: false,
Origins: nil,
ExceptOrigins: nil,
Bonus: nil,
Freeleech: false,
FreeleechPercent: "100%",
SmartEpisode: false,
Shows: "Is It Wrong to Try to Pick Up Girls in a Dungeon?",
Seasons: "4",
Episodes: "500",
Resolutions: []string{"1080p"},
Codecs: []string{"x264"},
Sources: []string{"BluRay"},
Containers: []string{"mkv"},
MatchHDR: []string{"HDR10"},
ExceptHDR: []string{"HDR10"},
MatchOther: []string{"Atmos"},
ExceptOther: []string{"Atmos"},
Years: "2023",
Months: "",
Days: "",
Artists: "",
Albums: "",
MatchReleaseTypes: []string{"Remux"},
ExceptReleaseTypes: "Remux",
Formats: []string{"FLAC"},
Quality: []string{"Lossless"},
Media: []string{"CD"},
PerfectFlac: true,
Cue: true,
Log: true,
LogScore: 100,
MatchCategories: "Anime",
ExceptCategories: "Anime",
MatchUploaders: "SubsPlease",
ExceptUploaders: "SubsPlease",
MatchLanguage: []string{"English", "Japanese"},
ExceptLanguage: []string{"English", "Japanese"},
Tags: "Anime, x264",
ExceptTags: "Anime, x264",
TagsAny: "Anime, x264",
ExceptTagsAny: "Anime, x264",
TagsMatchLogic: "AND",
ExceptTagsMatchLogic: "AND",
MatchReleaseTags: "Anime, x264",
ExceptReleaseTags: "Anime, x264",
UseRegexReleaseTags: true,
MatchDescription: "Anime, x264",
ExceptDescription: "Anime, x264",
UseRegexDescription: true,
}
}
func TestReleaseRepo_CheckIsDuplicateRelease(t *testing.T) {
for dbType, db := range testDBs {
log := setupLoggerForTest()
downloadClientRepo := NewDownloadClientRepo(log, db)
filterRepo := NewFilterRepo(log, db)
actionRepo := NewActionRepo(log, db, downloadClientRepo)
releaseRepo := NewReleaseRepo(log, db)
// reset
//db.handler.Exec("DELETE FROM release")
//db.handler.Exec("DELETE FROM action")
//db.handler.Exec("DELETE FROM release_action_status")
mockIndexer := domain.IndexerMinimal{ID: 0, Name: "Mock", Identifier: "mock", IdentifierExternal: "Mock"}
actionMock := &domain.Action{Name: "Test", Type: domain.ActionTypeTest, Enabled: true}
filterMock := getMockFilterDuplicates()
// Setup
err := filterRepo.Store(context.Background(), filterMock)
assert.NoError(t, err)
createdFilters, err := filterRepo.ListFilters(context.Background())
assert.NoError(t, err)
assert.NotNil(t, createdFilters)
actionMock.FilterID = filterMock.ID
err = actionRepo.Store(context.Background(), actionMock)
assert.NoError(t, err)
type fields struct {
releaseTitles []string
releaseTitle string
profile *domain.DuplicateReleaseProfile
}
tests := []struct {
name string
fields fields
isDuplicate bool
}{
{
name: "1",
fields: fields{
releaseTitles: []string{
"Inkheart 2008 BluRay 1080p DD5.1 x264-BADGROUP",
},
releaseTitle: "Inkheart 2008 BluRay 1080p DD5.1 x264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Group: true},
},
isDuplicate: false,
},
{
name: "2",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.WEB.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Source: true, Resolution: true},
},
isDuplicate: true,
},
{
name: "3",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.WEB.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true},
},
isDuplicate: true,
},
{
name: "4",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.WEB.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: false,
},
{
name: "5",
fields: fields{
releaseTitles: []string{
"That.Tv.Show.2023.S01E01.BluRay.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Tv.Show.2023.S01E01.BluRay.2160p.x265.DTS-HD-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: true,
},
{
name: "6",
fields: fields{
releaseTitles: []string{
"That.Tv.Show.2023.S01E01.BluRay.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Tv.Show.2023.S01E02.BluRay.2160p.x265.DTS-HD-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: false,
},
{
name: "7",
fields: fields{
releaseTitles: []string{
"That.Tv.Show.2023.S01.BluRay.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Tv.Show.2023.S01.BluRay.2160p.x265.DTS-HD-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: true,
},
{
name: "8",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 SDR H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 SDR H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, Group: true},
},
isDuplicate: true,
},
{
name: "9",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.HULU.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 SDR H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, Group: true},
},
isDuplicate: false,
},
{
name: "10",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: true,
},
{
name: "11",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
"The Best Show 2020 S04E10 1080p amzn web-dl ddp 5.1 hdr dv h.264-group",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true},
},
isDuplicate: true,
},
{
name: "12",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: false,
},
{
name: "13",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: false,
},
{
name: "14",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: true,
},
{
name: "15",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Season: true, Episode: true, DynamicRange: true},
},
isDuplicate: true,
},
{
name: "16",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E11 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Season: true, Episode: true, DynamicRange: true},
},
isDuplicate: false,
},
{
name: "17",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title REPACK 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Season: true, Episode: true, DynamicRange: true},
},
isDuplicate: true,
},
{
name: "18",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.REPACK.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title REPACK 1080p AMZN WEB-DL DDP 5.1 DV H.264-OTHERGROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Repack: true},
},
isDuplicate: false, // not a match because REPACK checks for the same group
},
{
name: "19",
fields: fields{
releaseTitles: []string{
"The Daily Show 2024-09-21 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The Daily Show 2024-09-21.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The Daily Show 2024-09-21.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
},
releaseTitle: "The Daily Show 2024-09-21.Other.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Year: true, Month: true, Day: true},
},
isDuplicate: true,
},
{
name: "20",
fields: fields{
releaseTitles: []string{
"The Daily Show 2024-09-21 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The Daily Show 2024-09-21.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The Daily Show 2024-09-21.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
},
releaseTitle: "The Daily Show 2024-09-21 Other Guest 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Year: true, Month: true, Day: true, SubTitle: true},
},
isDuplicate: false,
},
{
name: "21",
fields: fields{
releaseTitles: []string{
"The Daily Show 2024-09-21 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The Daily Show 2024-09-21.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The Daily Show 2024-09-21.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
},
releaseTitle: "The Daily Show 2024-09-22 Other Guest 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Year: true, Month: true, Day: true, SubTitle: true},
},
isDuplicate: false,
},
{
name: "22",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.2160p.BluRay.DTS-HD.5.1.x265-GROUP",
},
releaseTitle: "That.Movie.2023.2160p.BluRay.DD.2.0.x265-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "23",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.2160p.BluRay.DTS-HD.5.1.x265-GROUP",
},
releaseTitle: "That.Movie.2023.2160p.BluRay.DTS-HD.5.1.x265-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "24",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.2160p.BluRay.DD.5.1.x265-GROUP",
},
releaseTitle: "That.Movie.2023.2160p.BluRay.AC3.5.1.x265-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "25",
fields: fields{
releaseTitles: []string{
//"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "26",
fields: fields{
releaseTitles: []string{
//"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "27",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Edition: false, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "28",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
"Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Edition: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "29",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "30",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "31",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "32",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "33",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 FRENCH UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "34",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 FRENCH UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true, Language: true},
},
isDuplicate: true,
},
{
name: "35",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 FRENCH UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "36",
fields: fields{
releaseTitles: []string{
"Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
},
releaseTitle: "Road House 1989 1080p Blu-ray AVC LPCM 2.0-MONUMENT",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "37",
fields: fields{
releaseTitles: []string{
"Road House 1989 1080p ITA Blu-ray AVC LPCM 2.0-MONUMENT",
"Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
},
releaseTitle: "Road House 1989 1080p NOR Blu-ray AVC LPCM 2.0-MONUMENT",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "38",
fields: fields{
releaseTitles: []string{
"Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
},
releaseTitle: "Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true, Language: true},
},
isDuplicate: true,
},
{
name: "39",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{ReleaseName: true},
},
isDuplicate: true,
},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("Check_Is_Duplicate_Release %s [%s]", tt.name, dbType), func(t *testing.T) {
ctx := context.Background()
// Setup
for _, rel := range tt.fields.releaseTitles {
mockRel := domain.NewRelease(mockIndexer)
mockRel.ParseString(rel)
mockRel.FilterID = filterMock.ID
err = releaseRepo.Store(ctx, mockRel)
assert.NoError(t, err)
ras := &domain.ReleaseActionStatus{
ID: 0,
Status: domain.ReleasePushStatusApproved,
Action: "test",
ActionID: int64(actionMock.ID),
Type: domain.ActionTypeTest,
Client: "",
Filter: "Test filter",
FilterID: int64(filterMock.ID),
Rejections: []string{},
ReleaseID: mockRel.ID,
Timestamp: time.Now(),
}
err = releaseRepo.StoreReleaseActionStatus(ctx, ras)
assert.NoError(t, err)
}
releases, err := releaseRepo.Find(ctx, domain.ReleaseQueryParams{})
assert.NoError(t, err)
assert.Len(t, releases.Data, len(tt.fields.releaseTitles))
compareRel := domain.NewRelease(mockIndexer)
compareRel.ParseString(tt.fields.releaseTitle)
// Execute
isDuplicate, err := releaseRepo.CheckIsDuplicateRelease(ctx, tt.fields.profile, compareRel)
// Verify
assert.NoError(t, err)
assert.Equal(t, tt.isDuplicate, isDuplicate)
// Cleanup
_ = releaseRepo.Delete(ctx, &domain.DeleteReleaseRequest{OlderThan: 0})
})
}
// Cleanup
//_ = releaseRepo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMock.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
}
}