feat(filters): skip duplicates (#1711)

* feat(filters): skip duplicates

* fix: add interface instead of any

* fix(filters): tonullint

* feat(filters): skip dupes check month day

* chore: cleanup

* feat(db): set autoincrement id

* feat(filters): add repack and proper to dupe profile

* feat(filters): add default dupe profiles

* feat(duplicates): check audio and website

* feat(duplicates): update tests

* feat(duplicates): add toggles on addform

* feat(duplicates): fix sqlite upgrade path and initialize duplicate profiles

* feat(duplicates): simplify sqlite upgrade

avoiding temp table and unwieldy select.  Besides, FK constraints
are turned off anyway in #229.

* feat(duplicates): change CheckIsDuplicateRelease treatment of PROPER and REPACK

"Proper" and "Repack" are not parallel to the other conditions like "Title",
so they do not belong as dedup conditions.  "PROPER" means there was an issue in
the previous release, and so a PROPER is never a duplicate, even if it replaces
another PROPER.  Similarly, "REPACK" means there was an issue in the previous
release by that group, and so it is a duplicate only if we previously took a
release from a DIFFERENT group.

I have not removed Proper and Repack from the UI or the schema yet.

* feat(duplicates): update postgres schema to match sqlite

* feat(duplicates): fix web build errors

* feat(duplicates): fix postgres errors

* feat(filters): do leftjoin for duplicate profile

* fix(filters): partial update dupe profile

* go fmt `internal/domain/filter.go`

* feat(duplicates): restore straightforward logic for proper/repack

* feat(duplicates): remove mostly duplicate TV duplicate profiles

Having one profile seems the cleanest.  If somebody wants multiple
resolutions then they can add Resolution to the duplicate profile.
Tested this profile with both weekly episodic releases and daily
show releases.

* feat(release): add db indexes and sub_title

* feat(release): add IsDuplicate tests

* feat(release): update action handler

* feat(release): add more tests for skip duplicates

* feat(duplicates): check audio

* feat(duplicates): add more tests

* feat(duplicates): match edition cut and more

* fix(duplicates): tests

* fix(duplicates): missing imports

* fix(duplicates): tests

* feat(duplicates): handle sub_title edition and language in ui

* fix(duplicates): tests

* feat(duplicates): check name against normalized hash

* fix(duplicates): tests

* chore: update .gitignore to ignore .pnpm-store

* fix: tests

* fix(filters): tests

* fix: bad conflict merge

* fix: update release type in test

* fix: use vendored hot-toast

* fix: release_test.go

* fix: rss_test.go

* feat(duplicates): improve title hashing for unique check

* feat(duplicates): further improve title hashing for unique check with lang

* feat(duplicates): fix tests

* feat(duplicates): add macros IsDuplicate and DuplicateProfile ID and name

* feat(duplicates): add normalized hash match option

* fix: headlessui-state prop warning

* fix(duplicates): add missing year in daily ep normalize

* fix(duplicates): check rejections len

---------

Co-authored-by: ze0s <ze0s@riseup.net>
This commit is contained in:
kenstir 2024-12-25 16:33:46 -05:00 committed by GitHub
parent d153ac44b8
commit 4009554d10
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 3792 additions and 743 deletions

View file

@ -706,7 +706,7 @@ func (r *ActionRepo) DeleteByFilterID(ctx context.Context, filterID int) error {
return nil
}
func (r *ActionRepo) Store(ctx context.Context, action domain.Action) (*domain.Action, error) {
func (r *ActionRepo) Store(ctx context.Context, action *domain.Action) error {
queryBuilder := r.db.squirrel.
Insert("action").
Columns(
@ -783,14 +783,14 @@ func (r *ActionRepo) Store(ctx context.Context, action domain.Action) (*domain.A
var retID int64
if err := queryBuilder.QueryRowContext(ctx).Scan(&retID); err != nil {
return nil, errors.Wrap(err, "error executing query")
return errors.Wrap(err, "error executing query")
}
action.ID = int(retID)
r.log.Debug().Msgf("action.store: added new %d", retID)
return &action, nil
return nil
}
func (r *ActionRepo) Update(ctx context.Context, action domain.Action) (*domain.Action, error) {

View file

@ -16,8 +16,8 @@ import (
"github.com/stretchr/testify/assert"
)
func getMockAction() domain.Action {
return domain.Action{
func getMockAction() *domain.Action {
return &domain.Action{
Name: "randomAction",
Type: domain.ActionTypeTest,
Enabled: true,
@ -78,29 +78,29 @@ func TestActionRepo_Store(t *testing.T) {
mockData.FilterID = createdFilters[0].ID
// Actual test for Store
createdAction, err := repo.Store(context.Background(), mockData)
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
assert.NotNil(t, createdAction)
assert.NotNil(t, mockData)
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = repo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: mockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
t.Run(fmt.Sprintf("Store_Succeeds_With_Missing_or_empty_fields [%s]", dbType), func(t *testing.T) {
mockData := domain.Action{}
createdAction, err := repo.Store(context.Background(), mockData)
mockData := &domain.Action{}
err := repo.Store(context.Background(), mockData)
assert.NoError(t, err)
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = repo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: mockData.ID})
})
t.Run(fmt.Sprintf("Store_Fails_With_Invalid_ClientID [%s]", dbType), func(t *testing.T) {
mockData := getMockAction()
mockData.ClientID = 9999
_, err := repo.Store(context.Background(), mockData)
err := repo.Store(context.Background(), mockData)
assert.Error(t, err)
})
@ -110,7 +110,7 @@ func TestActionRepo_Store(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
defer cancel()
_, err := repo.Store(ctx, mockData)
err := repo.Store(ctx, mockData)
assert.Error(t, err)
})
}
@ -142,7 +142,7 @@ func TestActionRepo_StoreFilterActions(t *testing.T) {
mockData.FilterID = createdFilters[0].ID
// Actual test for StoreFilterActions
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{&mockData})
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.NoError(t, err)
assert.NotNil(t, createdActions)
@ -154,7 +154,7 @@ func TestActionRepo_StoreFilterActions(t *testing.T) {
})
t.Run(fmt.Sprintf("StoreFilterActions_Fails_Invalid_FilterID [%s]", dbType), func(t *testing.T) {
_, err := repo.StoreFilterActions(context.Background(), 9999, []*domain.Action{&mockData})
_, err := repo.StoreFilterActions(context.Background(), 9999, []*domain.Action{mockData})
assert.NoError(t, err)
})
@ -186,7 +186,7 @@ func TestActionRepo_StoreFilterActions(t *testing.T) {
assert.NoError(t, err)
assert.NotNil(t, createdFilters)
_, err = repo.StoreFilterActions(ctx, int64(createdFilters[0].ID), []*domain.Action{&mockData})
_, err = repo.StoreFilterActions(ctx, int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.Error(t, err)
// Cleanup
@ -219,7 +219,7 @@ func TestActionRepo_FindByFilterID(t *testing.T) {
mockData.ClientID = mock.ID
mockData.FilterID = createdFilters[0].ID
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{&mockData})
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.NoError(t, err)
// Actual test for FindByFilterID
@ -294,7 +294,7 @@ func TestActionRepo_List(t *testing.T) {
mockData.ClientID = mock.ID
mockData.FilterID = createdFilters[0].ID
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{&mockData})
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.NoError(t, err)
// Actual test for List
@ -344,7 +344,7 @@ func TestActionRepo_Get(t *testing.T) {
mockData.ClientID = mock.ID
mockData.FilterID = createdFilters[0].ID
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{&mockData})
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.NoError(t, err)
// Actual test for Get
@ -401,7 +401,7 @@ func TestActionRepo_Delete(t *testing.T) {
mockData.ClientID = mock.ID
mockData.FilterID = createdFilters[0].ID
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{&mockData})
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.NoError(t, err)
// Actual test for Delete
@ -455,7 +455,7 @@ func TestActionRepo_DeleteByFilterID(t *testing.T) {
mockData.ClientID = mock.ID
mockData.FilterID = createdFilters[0].ID
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{&mockData})
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.NoError(t, err)
err = repo.DeleteByFilterID(context.Background(), mockData.FilterID)
@ -508,7 +508,7 @@ func TestActionRepo_ToggleEnabled(t *testing.T) {
mockData.ClientID = mock.ID
mockData.FilterID = createdFilters[0].ID
mockData.Enabled = false
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{&mockData})
createdActions, err := repo.StoreFilterActions(context.Background(), int64(createdFilters[0].ID), []*domain.Action{mockData})
assert.NoError(t, err)
// Actual test for ToggleEnabled

View file

@ -129,13 +129,9 @@ type Tx struct {
handler *DB
}
type ILikeDynamic interface {
ToSql() (sql string, args []interface{}, err error)
}
// ILike is a wrapper for sq.Like and sq.ILike
// SQLite does not support ILike but postgres does so this checks what database is being used
func (db *DB) ILike(col string, val string) ILikeDynamic {
func (db *DB) ILike(col string, val string) sq.Sqlizer {
//if databaseDriver == "sqlite" {
if db.Driver == "sqlite" {
return sq.Like{col: val}

View file

@ -240,6 +240,7 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
"f.max_seeders",
"f.min_leechers",
"f.max_leechers",
"f.release_profile_duplicate_id",
"f.created_at",
"f.updated_at",
).
@ -266,6 +267,7 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
var minSize, maxSize, maxDownloadsUnit, matchReleases, exceptReleases, matchReleaseGroups, exceptReleaseGroups, matchReleaseTags, exceptReleaseTags, matchDescription, exceptDescription, freeleechPercent, shows, seasons, episodes, years, months, days, artists, albums, matchCategories, exceptCategories, matchUploaders, exceptUploaders, matchRecordLabels, exceptRecordLabels, tags, exceptTags, tagsMatchLogic, exceptTagsMatchLogic sql.NullString
var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac sql.NullBool
var delay, maxDownloads, logScore sql.NullInt32
var releaseProfileDuplicateId sql.NullInt64
err = row.Scan(
&f.ID,
@ -335,6 +337,7 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
&f.MaxSeeders,
&f.MinLeechers,
&f.MaxLeechers,
&releaseProfileDuplicateId,
&f.CreatedAt,
&f.UpdatedAt,
)
@ -385,6 +388,7 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
f.UseRegex = useRegex.Bool
f.Scene = scene.Bool
f.Freeleech = freeleech.Bool
f.ReleaseProfileDuplicateID = releaseProfileDuplicateId.Int64
return &f, nil
}
@ -466,10 +470,35 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
"f.max_leechers",
"f.created_at",
"f.updated_at",
"f.release_profile_duplicate_id",
"rdp.id",
"rdp.name",
"rdp.release_name",
"rdp.hash",
"rdp.title",
"rdp.sub_title",
"rdp.year",
"rdp.month",
"rdp.day",
"rdp.source",
"rdp.resolution",
"rdp.codec",
"rdp.container",
"rdp.dynamic_range",
"rdp.audio",
"rdp.release_group",
"rdp.season",
"rdp.episode",
"rdp.website",
"rdp.proper",
"rdp.repack",
"rdp.edition",
"rdp.language",
).
From("filter f").
Join("filter_indexer fi ON f.id = fi.filter_id").
Join("indexer i ON i.id = fi.indexer_id").
LeftJoin("release_profile_duplicate rdp ON rdp.id = f.release_profile_duplicate_id").
Where(sq.Eq{"i.identifier": indexer}).
Where(sq.Eq{"i.enabled": true}).
Where(sq.Eq{"f.enabled": true}).
@ -495,6 +524,10 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
var minSize, maxSize, maxDownloadsUnit, matchReleases, exceptReleases, matchReleaseGroups, exceptReleaseGroups, matchReleaseTags, exceptReleaseTags, matchDescription, exceptDescription, freeleechPercent, shows, seasons, episodes, years, months, days, artists, albums, matchCategories, exceptCategories, matchUploaders, exceptUploaders, matchRecordLabels, exceptRecordLabels, tags, exceptTags, tagsMatchLogic, exceptTagsMatchLogic sql.NullString
var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac sql.NullBool
var delay, maxDownloads, logScore sql.NullInt32
var releaseProfileDuplicateID, rdpId sql.NullInt64
var rdpName sql.NullString
var rdpRelName, rdpHash, rdpTitle, rdpSubTitle, rdpYear, rdpMonth, rdpDay, rdpSource, rdpResolution, rdpCodec, rdpContainer, rdpDynRange, rdpAudio, rdpGroup, rdpSeason, rdpEpisode, rdpWebsite, rdpProper, rdpRepack, rdpEdition, rdpLanguage sql.NullBool
err := rows.Scan(
&f.ID,
@ -566,6 +599,30 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
&f.MaxLeechers,
&f.CreatedAt,
&f.UpdatedAt,
&releaseProfileDuplicateID,
&rdpId,
&rdpName,
&rdpRelName,
&rdpHash,
&rdpTitle,
&rdpSubTitle,
&rdpYear,
&rdpMonth,
&rdpDay,
&rdpSource,
&rdpResolution,
&rdpCodec,
&rdpContainer,
&rdpDynRange,
&rdpAudio,
&rdpGroup,
&rdpSeason,
&rdpEpisode,
&rdpWebsite,
&rdpProper,
&rdpRepack,
&rdpEdition,
&rdpLanguage,
)
if err != nil {
return nil, errors.Wrap(err, "error scanning row")
@ -610,9 +667,40 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
f.UseRegex = useRegex.Bool
f.Scene = scene.Bool
f.Freeleech = freeleech.Bool
f.ReleaseProfileDuplicateID = releaseProfileDuplicateID.Int64
f.Rejections = []string{}
if releaseProfileDuplicateID.Valid {
profile := domain.DuplicateReleaseProfile{
ID: rdpId.Int64,
//Protocol: rdpName.String,
Name: rdpName.String,
ReleaseName: rdpRelName.Bool,
Hash: rdpHash.Bool,
Title: rdpTitle.Bool,
SubTitle: rdpSubTitle.Bool,
Year: rdpYear.Bool,
Month: rdpMonth.Bool,
Day: rdpDay.Bool,
Source: rdpSource.Bool,
Resolution: rdpResolution.Bool,
Codec: rdpCodec.Bool,
Container: rdpContainer.Bool,
DynamicRange: rdpDynRange.Bool,
Audio: rdpAudio.Bool,
Group: rdpGroup.Bool,
Season: rdpSeason.Bool,
Episode: rdpEpisode.Bool,
Website: rdpWebsite.Bool,
Proper: rdpProper.Bool,
Repack: rdpRepack.Bool,
Edition: rdpEdition.Bool,
Language: rdpLanguage.Bool,
}
f.DuplicateHandling = &profile
}
filters = append(filters, &f)
}
@ -774,6 +862,7 @@ func (r *FilterRepo) Store(ctx context.Context, filter *domain.Filter) error {
"max_seeders",
"min_leechers",
"max_leechers",
"release_profile_duplicate_id",
).
Values(
filter.Name,
@ -842,6 +931,7 @@ func (r *FilterRepo) Store(ctx context.Context, filter *domain.Filter) error {
filter.MaxSeeders,
filter.MinLeechers,
filter.MaxLeechers,
toNullInt64(filter.ReleaseProfileDuplicateID),
).
Suffix("RETURNING id").RunWith(r.db.handler)
@ -928,6 +1018,7 @@ func (r *FilterRepo) Update(ctx context.Context, filter *domain.Filter) error {
Set("max_seeders", filter.MaxSeeders).
Set("min_leechers", filter.MinLeechers).
Set("max_leechers", filter.MaxLeechers).
Set("release_profile_duplicate_id", toNullInt64(filter.ReleaseProfileDuplicateID)).
Set("updated_at", time.Now().Format(time.RFC3339)).
Where(sq.Eq{"id": filter.ID})
@ -1153,6 +1244,9 @@ func (r *FilterRepo) UpdatePartial(ctx context.Context, filter domain.FilterUpda
if filter.MaxLeechers != nil {
q = q.Set("max_leechers", filter.MaxLeechers)
}
if filter.ReleaseProfileDuplicateID != nil {
q = q.Set("release_profile_duplicate_id", filter.ReleaseProfileDuplicateID)
}
q = q.Where(sq.Eq{"id": filter.ID})

View file

@ -800,7 +800,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
mockAction.FilterID = mockData.ID
mockAction.ClientID = mockClient.ID
action, err := actionRepo.Store(context.Background(), mockAction)
err = actionRepo.Store(context.Background(), mockAction)
mockReleaseActionStatus.FilterID = int64(mockData.ID)
mockRelease.FilterID = mockData.ID
@ -808,7 +808,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
err = releaseRepo.Store(context.Background(), mockRelease)
assert.NoError(t, err)
mockReleaseActionStatus.ActionID = int64(action.ID)
mockReleaseActionStatus.ActionID = int64(mockAction.ID)
mockReleaseActionStatus.ReleaseID = mockRelease.ID
err = releaseRepo.StoreReleaseActionStatus(context.Background(), mockReleaseActionStatus)
@ -827,7 +827,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
})
// Cleanup
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: action.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: mockAction.ID})
_ = repo.Delete(context.Background(), mockData.ID)
_ = downloadClientRepo.Delete(context.Background(), mockClient.ID)
_ = releaseRepo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
@ -861,13 +861,15 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
mockAction1.FilterID = mockData.ID
mockAction1.ClientID = mockClient.ID
action1, err := actionRepo.Store(context.Background(), mockAction1)
actionErr := actionRepo.Store(context.Background(), mockAction1)
assert.NoError(t, actionErr)
mockAction2 := getMockAction()
mockAction2.FilterID = mockData.ID
mockAction2.ClientID = mockClient.ID
action2, err := actionRepo.Store(context.Background(), mockAction2)
action2Err := actionRepo.Store(context.Background(), mockAction2)
assert.NoError(t, action2Err)
mockRelease.FilterID = mockData.ID
@ -875,7 +877,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
assert.NoError(t, err)
mockReleaseActionStatus1 := getMockReleaseActionStatus()
mockReleaseActionStatus1.ActionID = int64(action1.ID)
mockReleaseActionStatus1.ActionID = int64(mockAction1.ID)
mockReleaseActionStatus1.FilterID = int64(mockData.ID)
mockReleaseActionStatus1.ReleaseID = mockRelease.ID
@ -883,7 +885,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
assert.NoError(t, err)
mockReleaseActionStatus2 := getMockReleaseActionStatus()
mockReleaseActionStatus2.ActionID = int64(action2.ID)
mockReleaseActionStatus2.ActionID = int64(mockAction2.ID)
mockReleaseActionStatus2.FilterID = int64(mockData.ID)
mockReleaseActionStatus2.ReleaseID = mockRelease.ID
@ -903,8 +905,8 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
})
// Cleanup
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: action1.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: action2.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: mockAction1.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: mockAction2.ID})
_ = repo.Delete(context.Background(), mockData.ID)
_ = downloadClientRepo.Delete(context.Background(), mockClient.ID)
_ = releaseRepo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
@ -924,13 +926,15 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
mockAction.FilterID = mockData.ID
mockAction.ClientID = mockClient.ID
action, err := actionRepo.Store(context.Background(), mockAction)
err = actionRepo.Store(context.Background(), mockAction)
assert.NoError(t, err)
mockAction2 := getMockAction()
mockAction2.FilterID = mockData.ID
mockAction2.ClientID = mockClient.ID
action2, err := actionRepo.Store(context.Background(), mockAction2)
err = actionRepo.Store(context.Background(), mockAction2)
assert.NoError(t, err)
mockRelease.FilterID = mockData.ID
@ -938,7 +942,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
assert.NoError(t, err)
mockReleaseActionStatus = getMockReleaseActionStatus()
mockReleaseActionStatus.ActionID = int64(action.ID)
mockReleaseActionStatus.ActionID = int64(mockAction.ID)
mockReleaseActionStatus.FilterID = int64(mockData.ID)
mockReleaseActionStatus.ReleaseID = mockRelease.ID
mockReleaseActionStatus.Timestamp = mockReleaseActionStatus.Timestamp.AddDate(0, -1, 0)
@ -947,7 +951,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
assert.NoError(t, err)
mockReleaseActionStatus2 := getMockReleaseActionStatus()
mockReleaseActionStatus2.ActionID = int64(action2.ID)
mockReleaseActionStatus2.ActionID = int64(mockAction2.ID)
mockReleaseActionStatus2.FilterID = int64(mockData.ID)
mockReleaseActionStatus2.ReleaseID = mockRelease.ID
mockReleaseActionStatus2.Timestamp = mockReleaseActionStatus2.Timestamp.AddDate(0, -1, 0)
@ -968,7 +972,7 @@ func TestFilterRepo_GetDownloadsByFilterId(t *testing.T) {
})
// Cleanup
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: action.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: mockAction.ID})
_ = repo.Delete(context.Background(), mockData.ID)
_ = downloadClientRepo.Delete(context.Background(), mockClient.ID)
_ = releaseRepo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})

View file

@ -88,6 +88,39 @@ CREATE TABLE irc_channel
UNIQUE (network_id, name)
);
CREATE TABLE release_profile_duplicate
(
id SERIAL PRIMARY KEY,
name TEXT NOT NULL,
protocol BOOLEAN DEFAULT FALSE,
release_name BOOLEAN DEFAULT FALSE,
hash BOOLEAN DEFAULT FALSE,
title BOOLEAN DEFAULT FALSE,
sub_title BOOLEAN DEFAULT FALSE,
year BOOLEAN DEFAULT FALSE,
month BOOLEAN DEFAULT FALSE,
day BOOLEAN DEFAULT FALSE,
source BOOLEAN DEFAULT FALSE,
resolution BOOLEAN DEFAULT FALSE,
codec BOOLEAN DEFAULT FALSE,
container BOOLEAN DEFAULT FALSE,
dynamic_range BOOLEAN DEFAULT FALSE,
audio BOOLEAN DEFAULT FALSE,
release_group BOOLEAN DEFAULT FALSE,
season BOOLEAN DEFAULT FALSE,
episode BOOLEAN DEFAULT FALSE,
website BOOLEAN DEFAULT FALSE,
proper BOOLEAN DEFAULT FALSE,
repack BOOLEAN DEFAULT FALSE,
edition BOOLEAN DEFAULT FALSE,
language BOOLEAN DEFAULT FALSE
);
INSERT INTO release_profile_duplicate (id, name, protocol, release_name, hash, title, sub_title, year, month, day, source, resolution, codec, container, dynamic_range, audio, release_group, season, episode, website, proper, repack, edition, language)
VALUES (1, 'Exact release', 'f', 't', 't', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f'),
(2, 'Movie', 'f', 'f', 'f', 't', 'f', 't', 'f', 'f', 'f', 't', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f'),
(3, 'TV', 'f', 'f', 'f', 't', 'f', 't', 't', 't', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 't', 't', 'f', 'f', 'f', 'f', 'f');
CREATE TABLE filter
(
id SERIAL PRIMARY KEY,
@ -159,7 +192,9 @@ CREATE TABLE filter
min_seeders INTEGER DEFAULT 0,
max_seeders INTEGER DEFAULT 0,
min_leechers INTEGER DEFAULT 0,
max_leechers INTEGER DEFAULT 0
max_leechers INTEGER DEFAULT 0,
release_profile_duplicate_id INTEGER,
FOREIGN KEY (release_profile_duplicate_id) REFERENCES release_profile_duplicate(id) ON DELETE SET NULL
);
CREATE INDEX filter_enabled_index
@ -270,9 +305,11 @@ CREATE TABLE "release"
group_id TEXT,
torrent_id TEXT,
torrent_name TEXT,
normalized_hash TEXT,
size BIGINT,
raw TEXT,
title TEXT,
sub_title TEXT,
category TEXT,
season INTEGER,
episode INTEGER,
@ -285,15 +322,18 @@ CREATE TABLE "release"
container TEXT,
hdr TEXT,
audio TEXT,
audio_channels TEXT,
release_group TEXT,
region TEXT,
language TEXT,
edition TEXT,
cut TEXT,
unrated BOOLEAN,
hybrid BOOLEAN,
proper BOOLEAN,
repack BOOLEAN,
website TEXT,
media_processing TEXT,
artists TEXT [] DEFAULT '{}' NOT NULL,
type TEXT,
format TEXT,
@ -308,6 +348,7 @@ CREATE TABLE "release"
freeleech_percent INTEGER,
uploader TEXT,
pre_time TEXT,
other TEXT [] DEFAULT '{}' NOT NULL,
filter_id INTEGER
CONSTRAINT release_filter_id_fk
REFERENCES filter
@ -326,6 +367,81 @@ CREATE INDEX release_timestamp_index
CREATE INDEX release_torrent_name_index
ON "release" (torrent_name);
CREATE INDEX release_normalized_hash_index
ON "release" (normalized_hash);
CREATE INDEX release_title_index
ON "release" (title);
CREATE INDEX release_sub_title_index
ON "release" (sub_title);
CREATE INDEX release_season_index
ON "release" (season);
CREATE INDEX release_episode_index
ON "release" (episode);
CREATE INDEX release_year_index
ON "release" (year);
CREATE INDEX release_month_index
ON "release" (month);
CREATE INDEX release_day_index
ON "release" (day);
CREATE INDEX release_resolution_index
ON "release" (resolution);
CREATE INDEX release_source_index
ON "release" (source);
CREATE INDEX release_codec_index
ON "release" (codec);
CREATE INDEX release_container_index
ON "release" (container);
CREATE INDEX release_hdr_index
ON "release" (hdr);
CREATE INDEX release_audio_index
ON "release" (audio);
CREATE INDEX release_audio_channels_index
ON "release" (audio_channels);
CREATE INDEX release_release_group_index
ON "release" (release_group);
CREATE INDEX release_language_index
ON "release" (language);
CREATE INDEX release_proper_index
ON "release" (proper);
CREATE INDEX release_repack_index
ON "release" (repack);
CREATE INDEX release_website_index
ON "release" (website);
CREATE INDEX release_media_processing_index
ON "release" (media_processing);
CREATE INDEX release_region_index
ON "release" (region);
CREATE INDEX release_edition_index
ON "release" (edition);
CREATE INDEX release_cut_index
ON "release" (cut);
CREATE INDEX release_hybrid_index
ON "release" (hybrid);
CREATE TABLE release_action_status
(
id SERIAL PRIMARY KEY,
@ -1074,5 +1190,154 @@ CREATE TABLE list_filter
ALTER TABLE filter
ADD COLUMN except_record_labels TEXT DEFAULT '';
`,
`CREATE TABLE release_profile_duplicate
(
id SERIAL PRIMARY KEY,
name TEXT NOT NULL,
protocol BOOLEAN DEFAULT FALSE,
release_name BOOLEAN DEFAULT FALSE,
hash BOOLEAN DEFAULT FALSE,
title BOOLEAN DEFAULT FALSE,
sub_title BOOLEAN DEFAULT FALSE,
year BOOLEAN DEFAULT FALSE,
month BOOLEAN DEFAULT FALSE,
day BOOLEAN DEFAULT FALSE,
source BOOLEAN DEFAULT FALSE,
resolution BOOLEAN DEFAULT FALSE,
codec BOOLEAN DEFAULT FALSE,
container BOOLEAN DEFAULT FALSE,
dynamic_range BOOLEAN DEFAULT FALSE,
audio BOOLEAN DEFAULT FALSE,
release_group BOOLEAN DEFAULT FALSE,
season BOOLEAN DEFAULT FALSE,
episode BOOLEAN DEFAULT FALSE,
website BOOLEAN DEFAULT FALSE,
proper BOOLEAN DEFAULT FALSE,
repack BOOLEAN DEFAULT FALSE,
edition BOOLEAN DEFAULT FALSE,
language BOOLEAN DEFAULT FALSE
);
INSERT INTO release_profile_duplicate (id, name, protocol, release_name, hash, title, sub_title, year, month, day, source, resolution, codec, container, dynamic_range, audio, release_group, season, episode, website, proper, repack, edition, language)
VALUES (1, 'Exact release', 'f', 't', 't', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f'),
(2, 'Movie', 'f', 'f', 'f', 't', 'f', 't', 'f', 'f', 'f', 't', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f'),
(3, 'TV', 'f', 'f', 'f', 't', 'f', 't', 't', 't', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 't', 't', 'f', 'f', 'f', 'f', 'f');
ALTER TABLE filter
ADD release_profile_duplicate_id INTEGER;
ALTER TABLE filter
ADD CONSTRAINT filter_release_profile_duplicate_id_fk
FOREIGN KEY (release_profile_duplicate_id) REFERENCES release_profile_duplicate (id)
ON DELETE SET NULL;
ALTER TABLE "release"
ADD normalized_hash TEXT;
ALTER TABLE "release"
ADD sub_title TEXT;
ALTER TABLE "release"
ADD COLUMN IF NOT EXISTS audio TEXT;
ALTER TABLE "release"
ADD audio_channels TEXT;
ALTER TABLE "release"
ADD IF NOT EXISTS language TEXT;
ALTER TABLE "release"
ADD media_processing TEXT;
ALTER TABLE "release"
ADD IF NOT EXISTS edition TEXT;
ALTER TABLE "release"
ADD IF NOT EXISTS cut TEXT;
ALTER TABLE "release"
ADD IF NOT EXISTS hybrid TEXT;
ALTER TABLE "release"
ADD IF NOT EXISTS region TEXT;
ALTER TABLE "release"
ADD IF NOT EXISTS other TEXT [] DEFAULT '{}' NOT NULL;
CREATE INDEX release_normalized_hash_index
ON "release" (normalized_hash);
CREATE INDEX release_title_index
ON "release" (title);
CREATE INDEX release_sub_title_index
ON "release" (sub_title);
CREATE INDEX release_season_index
ON "release" (season);
CREATE INDEX release_episode_index
ON "release" (episode);
CREATE INDEX release_year_index
ON "release" (year);
CREATE INDEX release_month_index
ON "release" (month);
CREATE INDEX release_day_index
ON "release" (day);
CREATE INDEX release_resolution_index
ON "release" (resolution);
CREATE INDEX release_source_index
ON "release" (source);
CREATE INDEX release_codec_index
ON "release" (codec);
CREATE INDEX release_container_index
ON "release" (container);
CREATE INDEX release_hdr_index
ON "release" (hdr);
CREATE INDEX release_audio_index
ON "release" (audio);
CREATE INDEX release_audio_channels_index
ON "release" (audio_channels);
CREATE INDEX release_release_group_index
ON "release" (release_group);
CREATE INDEX release_proper_index
ON "release" (proper);
CREATE INDEX release_repack_index
ON "release" (repack);
CREATE INDEX release_website_index
ON "release" (website);
CREATE INDEX release_media_processing_index
ON "release" (media_processing);
CREATE INDEX release_language_index
ON "release" (language);
CREATE INDEX release_region_index
ON "release" (region);
CREATE INDEX release_edition_index
ON "release" (edition);
CREATE INDEX release_cut_index
ON "release" (cut);
CREATE INDEX release_hybrid_index
ON "release" (hybrid);
`,
}

View file

@ -33,23 +33,31 @@ func NewReleaseRepo(log logger.Logger, db *DB) domain.ReleaseRepo {
}
func (repo *ReleaseRepo) Store(ctx context.Context, r *domain.Release) error {
codecStr := strings.Join(r.Codec, ",")
hdrStr := strings.Join(r.HDR, ",")
var (
codecStr = strings.Join(r.Codec, ",")
hdrStr = strings.Join(r.HDR, ",")
audioStr = strings.Join(r.Audio, ",")
editionStr = strings.Join(r.Edition, ",")
cutStr = strings.Join(r.Cut, ",")
languageStr = strings.Join(r.Language, ",")
)
queryBuilder := repo.db.squirrel.
Insert("release").
Columns("filter_status", "rejections", "indexer", "filter", "protocol", "implementation", "timestamp", "announce_type", "group_id", "torrent_id", "info_url", "download_url", "torrent_name", "size", "title", "category", "season", "episode", "year", "month", "day", "resolution", "source", "codec", "container", "hdr", "release_group", "proper", "repack", "website", "type", "origin", "tags", "uploader", "pre_time", "filter_id").
Values(r.FilterStatus, pq.Array(r.Rejections), r.Indexer.Identifier, r.FilterName, r.Protocol, r.Implementation, r.Timestamp.Format(time.RFC3339), r.AnnounceType, r.GroupID, r.TorrentID, r.InfoURL, r.DownloadURL, r.TorrentName, r.Size, r.Title, r.Category, r.Season, r.Episode, r.Year, r.Month, r.Day, r.Resolution, r.Source, codecStr, r.Container, hdrStr, r.Group, r.Proper, r.Repack, r.Website, r.Type, r.Origin, pq.Array(r.Tags), r.Uploader, r.PreTime, r.FilterID).
Columns("filter_status", "rejections", "indexer", "filter", "protocol", "implementation", "timestamp", "announce_type", "group_id", "torrent_id", "info_url", "download_url", "torrent_name", "normalized_hash", "size", "title", "sub_title", "category", "season", "episode", "year", "month", "day", "resolution", "source", "codec", "container", "hdr", "audio", "audio_channels", "release_group", "proper", "repack", "region", "language", "cut", "edition", "hybrid", "media_processing", "website", "type", "origin", "tags", "uploader", "pre_time", "other", "filter_id").
Values(r.FilterStatus, pq.Array(r.Rejections), r.Indexer.Identifier, r.FilterName, r.Protocol, r.Implementation, r.Timestamp.Format(time.RFC3339), r.AnnounceType, r.GroupID, r.TorrentID, r.InfoURL, r.DownloadURL, r.TorrentName, r.NormalizedHash, r.Size, r.Title, r.SubTitle, r.Category, r.Season, r.Episode, r.Year, r.Month, r.Day, r.Resolution, r.Source, codecStr, r.Container, hdrStr, audioStr, r.AudioChannels, r.Group, r.Proper, r.Repack, r.Region, languageStr, cutStr, editionStr, r.Hybrid, r.MediaProcessing, r.Website, r.Type.String(), r.Origin, pq.Array(r.Tags), r.Uploader, r.PreTime, pq.Array(r.Other), r.FilterID).
Suffix("RETURNING id").RunWith(repo.db.handler)
// return values
var retID int64
if err := queryBuilder.QueryRowContext(ctx).Scan(&retID); err != nil {
return errors.Wrap(err, "error executing query")
q, args, err := queryBuilder.ToSql()
if err != nil {
return errors.Wrap(err, "error building query")
}
r.ID = retID
repo.log.Debug().Msgf("release.store: %s %v", q, args)
if err := queryBuilder.QueryRowContext(ctx).Scan(&r.ID); err != nil {
return errors.Wrap(err, "error executing query")
}
repo.log.Debug().Msgf("release.store: %+v", r)
@ -102,14 +110,9 @@ func (repo *ReleaseRepo) StoreReleaseActionStatus(ctx context.Context, status *d
Values(status.Status, status.Action, status.ActionID, status.Type, status.Client, status.Filter, status.FilterID, pq.Array(status.Rejections), status.Timestamp.Format(time.RFC3339), status.ReleaseID).
Suffix("RETURNING id").RunWith(repo.db.handler)
// return values
var retID int64
if err := queryBuilder.QueryRowContext(ctx).Scan(&retID); err != nil {
if err := queryBuilder.QueryRowContext(ctx).Scan(&status.ID); err != nil {
return errors.Wrap(err, "error executing query")
}
status.ID = retID
}
repo.log.Trace().Msgf("release.store_release_action_status: %+v", status)
@ -117,6 +120,62 @@ func (repo *ReleaseRepo) StoreReleaseActionStatus(ctx context.Context, status *d
return nil
}
func (repo *ReleaseRepo) StoreDuplicateProfile(ctx context.Context, profile *domain.DuplicateReleaseProfile) error {
if profile.ID == 0 {
queryBuilder := repo.db.squirrel.
Insert("release_profile_duplicate").
Columns("name", "protocol", "release_name", "hash", "title", "sub_title", "season", "episode", "year", "month", "day", "resolution", "source", "codec", "container", "dynamic_range", "audio", "release_group", "website", "proper", "repack").
Values(profile.Name, profile.Protocol, profile.ReleaseName, profile.Hash, profile.Title, profile.SubTitle, profile.Season, profile.Episode, profile.Year, profile.Month, profile.Day, profile.Resolution, profile.Source, profile.Codec, profile.Container, profile.DynamicRange, profile.Audio, profile.Group, profile.Website, profile.Proper, profile.Repack).
Suffix("RETURNING id").
RunWith(repo.db.handler)
// return values
var retID int64
err := queryBuilder.QueryRowContext(ctx).Scan(&retID)
if err != nil {
return errors.Wrap(err, "error executing query")
}
profile.ID = retID
} else {
queryBuilder := repo.db.squirrel.
Update("release_profile_duplicate").
Set("name", profile.Name).
Set("protocol", profile.Protocol).
Set("release_name", profile.ReleaseName).
Set("hash", profile.Hash).
Set("title", profile.Title).
Set("sub_title", profile.SubTitle).
Set("season", profile.Season).
Set("episode", profile.Episode).
Set("year", profile.Year).
Set("month", profile.Month).
Set("day", profile.Day).
Set("resolution", profile.Resolution).
Set("source", profile.Source).
Set("codec", profile.Codec).
Set("container", profile.Container).
Set("dynamic_range", profile.DynamicRange).
Set("audio", profile.Audio).
Set("release_group", profile.Group).
Set("website", profile.Website).
Set("proper", profile.Proper).
Set("repack", profile.Repack).
Where(sq.Eq{"id": profile.ID}).
RunWith(repo.db.handler)
_, err := queryBuilder.ExecContext(ctx)
if err != nil {
return errors.Wrap(err, "error executing query")
}
}
repo.log.Debug().Msgf("release.StoreDuplicateProfile: %+v", profile)
return nil
}
func (repo *ReleaseRepo) Find(ctx context.Context, params domain.ReleaseQueryParams) (*domain.FindReleasesResponse, error) {
tx, err := repo.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
@ -192,7 +251,7 @@ func (repo *ReleaseRepo) findReleases(ctx context.Context, tx *Tx, params domain
whereQuery, _, err := whereQueryBuilder.ToSql()
if err != nil {
return nil, errors.Wrap(err, "error building wherequery")
return nil, errors.Wrap(err, "error building where query")
}
subQueryBuilder := repo.db.squirrel.
@ -230,8 +289,49 @@ func (repo *ReleaseRepo) findReleases(ctx context.Context, tx *Tx, params domain
}
queryBuilder := repo.db.squirrel.
Select("r.id", "r.filter_status", "r.rejections", "r.indexer", "i.id", "i.name", "i.identifier_external", "r.filter", "r.protocol", "r.announce_type", "r.info_url", "r.download_url", "r.title", "r.torrent_name", "r.size", "r.category", "r.season", "r.episode", "r.year", "r.resolution", "r.source", "r.codec", "r.container", "r.release_group", "r.timestamp",
"ras.id", "ras.status", "ras.action", "ras.action_id", "ras.type", "ras.client", "ras.filter", "ras.filter_id", "ras.release_id", "ras.rejections", "ras.timestamp").
Select(
"r.id",
"r.filter_status",
"r.rejections",
"r.indexer",
"i.id",
"i.name",
"i.identifier_external",
"r.filter",
"r.protocol",
"r.announce_type",
"r.info_url",
"r.download_url",
"r.title",
"r.sub_title",
"r.torrent_name",
"r.normalized_hash",
"r.size",
"r.category",
"r.season",
"r.episode",
"r.year",
"r.resolution",
"r.source",
"r.codec",
"r.container",
"r.hdr",
"r.audio",
"r.audio_channels",
"r.release_group",
"r.region",
"r.language",
"r.edition",
"r.cut",
"r.hybrid",
"r.proper",
"r.repack",
"r.website",
"r.media_processing",
"r.type",
"r.timestamp",
"ras.id", "ras.status", "ras.action", "ras.action_id", "ras.type", "ras.client", "ras.filter", "ras.filter_id", "ras.release_id", "ras.rejections", "ras.timestamp",
).
Column(sq.Alias(countQuery, "page_total")).
From("release r").
OrderBy("r.id DESC").
@ -267,7 +367,7 @@ func (repo *ReleaseRepo) findReleases(ctx context.Context, tx *Tx, params domain
var rls domain.Release
var ras domain.ReleaseActionStatus
var rlsIndexer, rlsIndexerName, rlsIndexerExternalName, rlsFilter, rlsAnnounceType, infoUrl, downloadUrl, codec sql.NullString
var rlsIndexer, rlsIndexerName, rlsIndexerExternalName, rlsFilter, rlsAnnounceType, infoUrl, downloadUrl, subTitle, normalizedHash, codec, hdr, rlsType, audioStr, languageStr, editionStr, cutStr, website sql.NullString
var rlsIndexerID sql.NullInt64
var rasId, rasFilterId, rasReleaseId, rasActionId sql.NullInt64
@ -275,7 +375,49 @@ func (repo *ReleaseRepo) findReleases(ctx context.Context, tx *Tx, params domain
var rasRejections []sql.NullString
var rasTimestamp sql.NullTime
if err := rows.Scan(&rls.ID, &rls.FilterStatus, pq.Array(&rls.Rejections), &rlsIndexer, &rlsIndexerID, &rlsIndexerName, &rlsIndexerExternalName, &rlsFilter, &rls.Protocol, &rlsAnnounceType, &infoUrl, &downloadUrl, &rls.Title, &rls.TorrentName, &rls.Size, &rls.Category, &rls.Season, &rls.Episode, &rls.Year, &rls.Resolution, &rls.Source, &codec, &rls.Container, &rls.Group, &rls.Timestamp, &rasId, &rasStatus, &rasAction, &rasActionId, &rasType, &rasClient, &rasFilter, &rasFilterId, &rasReleaseId, pq.Array(&rasRejections), &rasTimestamp, &resp.TotalCount); err != nil {
if err := rows.Scan(
&rls.ID,
&rls.FilterStatus,
pq.Array(&rls.Rejections),
&rlsIndexer,
&rlsIndexerID,
&rlsIndexerName,
&rlsIndexerExternalName,
&rlsFilter,
&rls.Protocol,
&rlsAnnounceType,
&infoUrl,
&downloadUrl,
&rls.Title,
&subTitle,
&rls.TorrentName,
&normalizedHash,
&rls.Size,
&rls.Category,
&rls.Season,
&rls.Episode,
&rls.Year,
&rls.Resolution,
&rls.Source,
&codec,
&rls.Container,
&hdr,
&audioStr,
&rls.AudioChannels,
&rls.Group,
&rls.Region,
&languageStr,
&editionStr,
&cutStr,
&rls.Hybrid,
&rls.Proper,
&rls.Repack,
&website,
&rls.MediaProcessing,
&rlsType,
&rls.Timestamp,
&rasId, &rasStatus, &rasAction, &rasActionId, &rasType, &rasClient, &rasFilter, &rasFilterId, &rasReleaseId, pq.Array(&rasRejections), &rasTimestamp, &resp.TotalCount,
); err != nil {
return resp, errors.Wrap(err, "error scanning row")
}
@ -324,7 +466,19 @@ func (repo *ReleaseRepo) findReleases(ctx context.Context, tx *Tx, params domain
rls.ActionStatus = make([]domain.ReleaseActionStatus, 0)
rls.InfoURL = infoUrl.String
rls.DownloadURL = downloadUrl.String
rls.SubTitle = subTitle.String
rls.NormalizedHash = normalizedHash.String
rls.Codec = strings.Split(codec.String, ",")
rls.HDR = strings.Split(hdr.String, ",")
rls.Audio = strings.Split(audioStr.String, ",")
rls.Language = strings.Split(languageStr.String, ",")
rls.Edition = strings.Split(editionStr.String, ",")
rls.Cut = strings.Split(cutStr.String, ",")
rls.Website = website.String
//rls.Type = rlsType.String
if rlsType.Valid {
rls.ParseType(rlsType.String)
}
// only add ActionStatus if it's not empty
if ras.ID > 0 {
@ -342,6 +496,66 @@ func (repo *ReleaseRepo) findReleases(ctx context.Context, tx *Tx, params domain
return resp, nil
}
func (repo *ReleaseRepo) FindDuplicateReleaseProfiles(ctx context.Context) ([]*domain.DuplicateReleaseProfile, error) {
queryBuilder := repo.db.squirrel.
Select(
"id",
"name",
"protocol",
"release_name",
"hash",
"title",
"sub_title",
"year",
"month",
"day",
"source",
"resolution",
"codec",
"container",
"dynamic_range",
"audio",
"release_group",
"season",
"episode",
"website",
"proper",
"repack",
).
From("release_profile_duplicate")
query, args, err := queryBuilder.ToSql()
if err != nil {
return nil, errors.Wrap(err, "error building query")
}
rows, err := repo.db.handler.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, "error executing query")
}
defer rows.Close()
if err := rows.Err(); err != nil {
return nil, errors.Wrap(err, "error rows FindDuplicateReleaseProfiles")
}
res := make([]*domain.DuplicateReleaseProfile, 0)
for rows.Next() {
var p domain.DuplicateReleaseProfile
err := rows.Scan(&p.ID, &p.Name, &p.Protocol, &p.ReleaseName, &p.Hash, &p.Title, &p.SubTitle, &p.Year, &p.Month, &p.Day, &p.Source, &p.Resolution, &p.Codec, &p.Container, &p.DynamicRange, &p.Audio, &p.Group, &p.Season, &p.Episode, &p.Website, &p.Proper, &p.Repack)
if err != nil {
return nil, errors.Wrap(err, "error scanning row")
}
res = append(res, &p)
}
return res, nil
}
func (repo *ReleaseRepo) GetIndexerOptions(ctx context.Context) ([]string, error) {
query := `SELECT DISTINCT indexer FROM "release" UNION SELECT DISTINCT identifier indexer FROM indexer;`
@ -420,7 +634,7 @@ func (repo *ReleaseRepo) GetActionStatusByReleaseID(ctx context.Context, release
func (repo *ReleaseRepo) Get(ctx context.Context, req *domain.GetReleaseRequest) (*domain.Release, error) {
queryBuilder := repo.db.squirrel.
Select("r.id", "r.filter_status", "r.rejections", "r.indexer", "r.filter", "r.filter_id", "r.protocol", "r.implementation", "r.announce_type", "r.info_url", "r.download_url", "r.title", "r.torrent_name", "r.category", "r.size", "r.group_id", "r.torrent_id", "r.uploader", "r.timestamp").
Select("r.id", "r.filter_status", "r.rejections", "r.indexer", "r.filter", "r.filter_id", "r.protocol", "r.implementation", "r.announce_type", "r.info_url", "r.download_url", "r.title", "r.sub_title", "r.torrent_name", "r.category", "r.size", "r.group_id", "r.torrent_id", "r.uploader", "r.timestamp").
From("release r").
OrderBy("r.id DESC").
Where(sq.Eq{"r.id": req.Id})
@ -439,10 +653,10 @@ func (repo *ReleaseRepo) Get(ctx context.Context, req *domain.GetReleaseRequest)
var rls domain.Release
var indexerName, filterName, announceType, infoUrl, downloadUrl, groupId, torrentId, category, uploader sql.NullString
var indexerName, filterName, announceType, infoUrl, downloadUrl, subTitle, groupId, torrentId, category, uploader sql.NullString
var filterId sql.NullInt64
if err := row.Scan(&rls.ID, &rls.FilterStatus, pq.Array(&rls.Rejections), &indexerName, &filterName, &filterId, &rls.Protocol, &rls.Implementation, &announceType, &infoUrl, &downloadUrl, &rls.Title, &rls.TorrentName, &category, &rls.Size, &groupId, &torrentId, &uploader, &rls.Timestamp); err != nil {
if err := row.Scan(&rls.ID, &rls.FilterStatus, pq.Array(&rls.Rejections), &indexerName, &filterName, &filterId, &rls.Protocol, &rls.Implementation, &announceType, &infoUrl, &downloadUrl, &rls.Title, &subTitle, &rls.TorrentName, &category, &rls.Size, &groupId, &torrentId, &uploader, &rls.Timestamp); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, domain.ErrRecordNotFound
}
@ -457,6 +671,7 @@ func (repo *ReleaseRepo) Get(ctx context.Context, req *domain.GetReleaseRequest)
rls.AnnounceType = domain.AnnounceType(announceType.String)
rls.InfoURL = infoUrl.String
rls.DownloadURL = downloadUrl.String
rls.SubTitle = subTitle.String
rls.Category = category.String
rls.GroupID = groupId.String
rls.TorrentID = torrentId.String
@ -670,6 +885,31 @@ func (repo *ReleaseRepo) Delete(ctx context.Context, req *domain.DeleteReleaseRe
return nil
}
func (repo *ReleaseRepo) DeleteReleaseProfileDuplicate(ctx context.Context, id int64) error {
qb := repo.db.squirrel.Delete("release_profile_duplicate").Where(sq.Eq{"id": id})
query, args, err := qb.ToSql()
if err != nil {
return errors.Wrap(err, "error building SQL query")
}
_, err = repo.db.handler.ExecContext(ctx, query, args...)
if err != nil {
return errors.Wrap(err, "error executing delete query")
}
//deletedRows, err := result.RowsAffected()
//if err != nil {
// return errors.Wrap(err, "error fetching rows affected")
//}
//
//repo.log.Debug().Msgf("deleted %d rows from release table", deletedRows)
repo.log.Debug().Msgf("deleted duplicate release profile: %d", id)
return nil
}
func (repo *ReleaseRepo) CheckSmartEpisodeCanDownload(ctx context.Context, p *domain.SmartEpisodeParams) (bool, error) {
queryBuilder := repo.db.squirrel.
Select("COUNT(*)").
@ -793,3 +1033,200 @@ func (repo *ReleaseRepo) UpdateBaseURL(ctx context.Context, indexer string, oldB
return nil
}
func (repo *ReleaseRepo) CheckIsDuplicateRelease(ctx context.Context, profile *domain.DuplicateReleaseProfile, release *domain.Release) (bool, error) {
queryBuilder := repo.db.squirrel.
Select("r.id, r.torrent_name, r.normalized_hash, r.title, ras.action, ras.status").
From("release r").
LeftJoin("release_action_status ras ON r.id = ras.release_id").
Where("ras.status = 'PUSH_APPROVED'")
if profile.ReleaseName && profile.Hash {
//queryBuilder = queryBuilder.Where(repo.db.ILike("r.torrent_name", release.TorrentName))
queryBuilder = queryBuilder.Where(sq.Eq{"r.normalized_hash": release.NormalizedHash})
} else {
if profile.Title {
queryBuilder = queryBuilder.Where(repo.db.ILike("r.title", release.Title))
}
if profile.SubTitle {
queryBuilder = queryBuilder.Where(repo.db.ILike("r.sub_title", release.SubTitle))
}
if profile.ReleaseName && profile.Hash {
//queryBuilder = queryBuilder.Where(repo.db.ILike("r.torrent_name", release.TorrentName))
queryBuilder = queryBuilder.Where(sq.Eq{"r.normalized_hash": release.NormalizedHash})
}
if profile.Year {
queryBuilder = queryBuilder.Where(sq.Eq{"r.year": release.Year})
}
if profile.Month {
queryBuilder = queryBuilder.Where(sq.Eq{"r.month": release.Month})
}
if profile.Day {
queryBuilder = queryBuilder.Where(sq.Eq{"r.day": release.Day})
}
if profile.Source {
queryBuilder = queryBuilder.Where(sq.Eq{"r.source": release.Source})
}
if profile.Container {
queryBuilder = queryBuilder.Where(sq.Eq{"r.container": release.Container})
}
if profile.Edition {
//queryBuilder = queryBuilder.Where(sq.Eq{"r.cut": release.Cut})
if len(release.Cut) > 1 {
var and sq.And
for _, cut := range release.Cut {
//and = append(and, sq.Eq{"r.cut": "%" + cut + "%"})
and = append(and, repo.db.ILike("r.cut", "%"+cut+"%"))
}
queryBuilder = queryBuilder.Where(and)
} else if len(release.Cut) == 1 {
queryBuilder = queryBuilder.Where(repo.db.ILike("r.cut", "%"+release.Cut[0]+"%"))
}
//queryBuilder = queryBuilder.Where(sq.Eq{"r.edition": release.Edition})
if len(release.Edition) > 1 {
var and sq.And
for _, edition := range release.Edition {
and = append(and, repo.db.ILike("r.edition", "%"+edition+"%"))
}
queryBuilder = queryBuilder.Where(and)
} else if len(release.Edition) == 1 {
queryBuilder = queryBuilder.Where(repo.db.ILike("r.edition", "%"+release.Edition[0]+"%"))
}
}
// video features (hybrid, remux)
if release.IsTypeVideo() {
queryBuilder = queryBuilder.Where(sq.Eq{"r.hybrid": release.Hybrid})
queryBuilder = queryBuilder.Where(sq.Eq{"r.media_processing": release.MediaProcessing})
}
if profile.Language {
queryBuilder = queryBuilder.Where(sq.Eq{"r.region": release.Region})
if len(release.Language) > 0 {
var and sq.And
for _, lang := range release.Language {
and = append(and, repo.db.ILike("r.language", "%"+lang+"%"))
}
queryBuilder = queryBuilder.Where(and)
} else {
queryBuilder = queryBuilder.Where(sq.Eq{"r.language": ""})
}
}
if profile.Codec {
if len(release.Codec) > 1 {
var and sq.And
for _, codec := range release.Codec {
and = append(and, repo.db.ILike("r.codec", "%"+codec+"%"))
}
queryBuilder = queryBuilder.Where(and)
} else {
// FIXME this does an IN (arg)
queryBuilder = queryBuilder.Where(sq.Eq{"r.codec": release.Codec})
}
}
if profile.Resolution {
queryBuilder = queryBuilder.Where(sq.Eq{"r.resolution": release.Resolution})
}
if profile.DynamicRange {
//if len(release.HDR) > 1 {
// var and sq.And
// for _, hdr := range release.HDR {
// and = append(and, repo.db.ILike("r.hdr", "%"+hdr+"%"))
// }
// queryBuilder = queryBuilder.Where(and)
//} else {
// queryBuilder = queryBuilder.Where(sq.Eq{"r.hdr": release.HDR})
//}
queryBuilder = queryBuilder.Where(sq.Eq{"r.hdr": strings.Join(release.HDR, ",")})
}
if profile.Audio {
queryBuilder = queryBuilder.Where(sq.Eq{"r.audio": strings.Join(release.Audio, ",")})
queryBuilder = queryBuilder.Where(sq.Eq{"r.audio_channels": release.AudioChannels})
}
if profile.Group {
queryBuilder = queryBuilder.Where(repo.db.ILike("r.release_group", release.Group))
}
if profile.Season {
queryBuilder = queryBuilder.Where(sq.Eq{"r.season": release.Season})
}
if profile.Episode {
queryBuilder = queryBuilder.Where(sq.Eq{"r.episode": release.Episode})
}
if profile.Website {
queryBuilder = queryBuilder.Where(sq.Eq{"r.website": release.Website})
}
if profile.Proper {
queryBuilder = queryBuilder.Where(sq.Eq{"r.proper": release.Proper})
}
if profile.Repack {
queryBuilder = queryBuilder.Where(sq.And{
sq.Eq{"r.repack": release.Repack},
repo.db.ILike("r.release_group", release.Group),
})
}
}
query, args, err := queryBuilder.ToSql()
if err != nil {
return false, errors.Wrap(err, "error building query")
}
repo.log.Trace().Str("database", "release.FindDuplicateReleases").Msgf("query: %q, args: %q", query, args)
rows, err := repo.db.handler.QueryContext(ctx, query, args...)
if err != nil {
return false, err
}
if err := rows.Err(); err != nil {
return false, errors.Wrap(err, "error rows CheckIsDuplicateRelease")
}
type result struct {
id int
release string
hash string
title string
action string
status string
}
var res []result
for rows.Next() {
r := result{}
if err := rows.Scan(&r.id, &r.release, &r.hash, &r.title, &r.action, &r.status); err != nil {
return false, errors.Wrap(err, "error scan CheckIsDuplicateRelease")
}
res = append(res, r)
}
repo.log.Trace().Str("database", "release.FindDuplicateReleases").Msgf("found duplicate releases: %+v", res)
if len(res) == 0 {
return false, nil
}
return true, nil
}

View file

@ -13,6 +13,7 @@ import (
"github.com/autobrr/autobrr/internal/domain"
"github.com/moistari/rls"
"github.com/stretchr/testify/assert"
)
@ -49,12 +50,13 @@ func getMockRelease() *domain.Release {
Proper: true,
Repack: false,
Website: "https://example.com",
Type: "Movie",
Type: rls.Movie,
Origin: "P2P",
Tags: []string{"Action", "Adventure"},
Uploader: "john_doe",
PreTime: "10m",
FilterID: 1,
Other: []string{},
}
}
@ -108,11 +110,11 @@ func TestReleaseRepo_Store(t *testing.T) {
// Execute
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -123,7 +125,7 @@ func TestReleaseRepo_Store(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -164,11 +166,11 @@ func TestReleaseRepo_StoreReleaseActionStatus(t *testing.T) {
// Execute
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -179,7 +181,7 @@ func TestReleaseRepo_StoreReleaseActionStatus(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -328,11 +330,11 @@ func TestReleaseRepo_GetIndexerOptions(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -347,7 +349,7 @@ func TestReleaseRepo_GetIndexerOptions(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -387,11 +389,11 @@ func TestReleaseRepo_GetActionStatusByReleaseID(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -407,7 +409,7 @@ func TestReleaseRepo_GetActionStatusByReleaseID(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -447,11 +449,11 @@ func TestReleaseRepo_Get(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -467,7 +469,7 @@ func TestReleaseRepo_Get(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -507,11 +509,11 @@ func TestReleaseRepo_Stats(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -527,7 +529,7 @@ func TestReleaseRepo_Stats(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -567,11 +569,11 @@ func TestReleaseRepo_Delete(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -584,7 +586,7 @@ func TestReleaseRepo_Delete(t *testing.T) {
assert.NoError(t, err)
// Cleanup
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
@ -624,11 +626,11 @@ func TestReleaseRepo_CheckSmartEpisodeCanDownloadShow(t *testing.T) {
err = repo.Store(context.Background(), mockData)
assert.NoError(t, err)
createdAction, err := actionRepo.Store(context.Background(), actionMockData)
err = actionRepo.Store(context.Background(), actionMockData)
assert.NoError(t, err)
releaseActionMockData.ReleaseID = mockData.ID
releaseActionMockData.ActionID = int64(createdAction.ID)
releaseActionMockData.ActionID = int64(actionMockData.ID)
releaseActionMockData.FilterID = int64(createdFilters[0].ID)
err = repo.StoreReleaseActionStatus(context.Background(), releaseActionMockData)
@ -652,9 +654,724 @@ func TestReleaseRepo_CheckSmartEpisodeCanDownloadShow(t *testing.T) {
// Cleanup
_ = repo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: createdAction.ID})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMockData.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
_ = downloadClientRepo.Delete(context.Background(), mock.ID)
})
}
}
func getMockDuplicateReleaseProfileTV() *domain.DuplicateReleaseProfile {
return &domain.DuplicateReleaseProfile{
ID: 0,
Name: "TV",
Protocol: false,
ReleaseName: false,
Hash: false,
Title: true,
SubTitle: false,
Year: false,
Month: false,
Day: false,
Source: false,
Resolution: false,
Codec: false,
Container: false,
DynamicRange: false,
Audio: false,
Group: false,
Season: true,
Episode: true,
Website: false,
Proper: false,
Repack: false,
Edition: false,
Language: false,
}
}
func getMockDuplicateReleaseProfileTVDaily() *domain.DuplicateReleaseProfile {
return &domain.DuplicateReleaseProfile{
ID: 0,
Name: "TV",
Protocol: false,
ReleaseName: false,
Hash: false,
Title: true,
SubTitle: false,
Year: true,
Month: true,
Day: true,
Source: false,
Resolution: false,
Codec: false,
Container: false,
DynamicRange: false,
Audio: false,
Group: false,
Season: false,
Episode: false,
Website: false,
Proper: false,
Repack: false,
Edition: false,
Language: false,
}
}
func getMockFilterDuplicates() *domain.Filter {
return &domain.Filter{
Name: "New Filter",
Enabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
MinSize: "10mb",
MaxSize: "20mb",
Delay: 60,
Priority: 1,
MaxDownloads: 100,
MaxDownloadsUnit: domain.FilterMaxDownloadsHour,
MatchReleases: "BRRip",
ExceptReleases: "BRRip",
UseRegex: false,
MatchReleaseGroups: "AMIABLE",
ExceptReleaseGroups: "NTb",
Scene: false,
Origins: nil,
ExceptOrigins: nil,
Bonus: nil,
Freeleech: false,
FreeleechPercent: "100%",
SmartEpisode: false,
Shows: "Is It Wrong to Try to Pick Up Girls in a Dungeon?",
Seasons: "4",
Episodes: "500",
Resolutions: []string{"1080p"},
Codecs: []string{"x264"},
Sources: []string{"BluRay"},
Containers: []string{"mkv"},
MatchHDR: []string{"HDR10"},
ExceptHDR: []string{"HDR10"},
MatchOther: []string{"Atmos"},
ExceptOther: []string{"Atmos"},
Years: "2023",
Months: "",
Days: "",
Artists: "",
Albums: "",
MatchReleaseTypes: []string{"Remux"},
ExceptReleaseTypes: "Remux",
Formats: []string{"FLAC"},
Quality: []string{"Lossless"},
Media: []string{"CD"},
PerfectFlac: true,
Cue: true,
Log: true,
LogScore: 100,
MatchCategories: "Anime",
ExceptCategories: "Anime",
MatchUploaders: "SubsPlease",
ExceptUploaders: "SubsPlease",
MatchLanguage: []string{"English", "Japanese"},
ExceptLanguage: []string{"English", "Japanese"},
Tags: "Anime, x264",
ExceptTags: "Anime, x264",
TagsAny: "Anime, x264",
ExceptTagsAny: "Anime, x264",
TagsMatchLogic: "AND",
ExceptTagsMatchLogic: "AND",
MatchReleaseTags: "Anime, x264",
ExceptReleaseTags: "Anime, x264",
UseRegexReleaseTags: true,
MatchDescription: "Anime, x264",
ExceptDescription: "Anime, x264",
UseRegexDescription: true,
}
}
func TestReleaseRepo_CheckIsDuplicateRelease(t *testing.T) {
for dbType, db := range testDBs {
log := setupLoggerForTest()
downloadClientRepo := NewDownloadClientRepo(log, db)
filterRepo := NewFilterRepo(log, db)
actionRepo := NewActionRepo(log, db, downloadClientRepo)
releaseRepo := NewReleaseRepo(log, db)
// reset
//db.handler.Exec("DELETE FROM release")
//db.handler.Exec("DELETE FROM action")
//db.handler.Exec("DELETE FROM release_action_status")
mockIndexer := domain.IndexerMinimal{ID: 0, Name: "Mock", Identifier: "mock", IdentifierExternal: "Mock"}
actionMock := &domain.Action{Name: "Test", Type: domain.ActionTypeTest, Enabled: true}
filterMock := getMockFilterDuplicates()
// Setup
err := filterRepo.Store(context.Background(), filterMock)
assert.NoError(t, err)
createdFilters, err := filterRepo.ListFilters(context.Background())
assert.NoError(t, err)
assert.NotNil(t, createdFilters)
actionMock.FilterID = filterMock.ID
err = actionRepo.Store(context.Background(), actionMock)
assert.NoError(t, err)
type fields struct {
releaseTitles []string
releaseTitle string
profile *domain.DuplicateReleaseProfile
}
tests := []struct {
name string
fields fields
isDuplicate bool
}{
{
name: "1",
fields: fields{
releaseTitles: []string{
"Inkheart 2008 BluRay 1080p DD5.1 x264-BADGROUP",
},
releaseTitle: "Inkheart 2008 BluRay 1080p DD5.1 x264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Group: true},
},
isDuplicate: false,
},
{
name: "2",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.WEB.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Source: true, Resolution: true},
},
isDuplicate: true,
},
{
name: "3",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.WEB.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true},
},
isDuplicate: true,
},
{
name: "4",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.WEB.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: false,
},
{
name: "5",
fields: fields{
releaseTitles: []string{
"That.Tv.Show.2023.S01E01.BluRay.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Tv.Show.2023.S01E01.BluRay.2160p.x265.DTS-HD-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: true,
},
{
name: "6",
fields: fields{
releaseTitles: []string{
"That.Tv.Show.2023.S01E01.BluRay.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Tv.Show.2023.S01E02.BluRay.2160p.x265.DTS-HD-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: false,
},
{
name: "7",
fields: fields{
releaseTitles: []string{
"That.Tv.Show.2023.S01.BluRay.2160p.x265.DTS-HD-GROUP",
},
releaseTitle: "That.Tv.Show.2023.S01.BluRay.2160p.x265.DTS-HD-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Group: true},
},
isDuplicate: true,
},
{
name: "8",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 SDR H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 SDR H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, Group: true},
},
isDuplicate: true,
},
{
name: "9",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.HULU.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 SDR H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, Group: true},
},
isDuplicate: false,
},
{
name: "10",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: true,
},
{
name: "11",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
"The Best Show 2020 S04E10 1080p amzn web-dl ddp 5.1 hdr dv h.264-group",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true},
},
isDuplicate: true,
},
{
name: "12",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: false,
},
{
name: "13",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: false,
},
{
name: "14",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Year: true, Season: true, Episode: true, Source: true, Codec: true, Resolution: true, Website: true, DynamicRange: true, Group: true},
},
isDuplicate: true,
},
{
name: "15",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Season: true, Episode: true, DynamicRange: true},
},
isDuplicate: true,
},
{
name: "16",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E11 Episode Title 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Season: true, Episode: true, DynamicRange: true},
},
isDuplicate: false,
},
{
name: "17",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title REPACK 1080p AMZN WEB-DL DDP 5.1 HDR DV H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, SubTitle: true, Season: true, Episode: true, DynamicRange: true},
},
isDuplicate: true,
},
{
name: "18",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The.Best.Show.2020.S04E10.Episode.Title.REPACK.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 Episode Title REPACK 1080p AMZN WEB-DL DDP 5.1 DV H.264-OTHERGROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Repack: true},
},
isDuplicate: false, // not a match because REPACK checks for the same group
},
{
name: "19",
fields: fields{
releaseTitles: []string{
"The Daily Show 2024-09-21 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The Daily Show 2024-09-21.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The Daily Show 2024-09-21.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
},
releaseTitle: "The Daily Show 2024-09-21.Other.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Year: true, Month: true, Day: true},
},
isDuplicate: true,
},
{
name: "20",
fields: fields{
releaseTitles: []string{
"The Daily Show 2024-09-21 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The Daily Show 2024-09-21.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The Daily Show 2024-09-21.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
},
releaseTitle: "The Daily Show 2024-09-21 Other Guest 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Year: true, Month: true, Day: true, SubTitle: true},
},
isDuplicate: false,
},
{
name: "21",
fields: fields{
releaseTitles: []string{
"The Daily Show 2024-09-21 1080p HULU WEB-DL DDP 5.1 SDR H.264-GROUP",
"The Daily Show 2024-09-21.1080p.AMZN.WEB-DL.DDP.5.1.SDR.H.264-GROUP",
"The Daily Show 2024-09-21.Guest.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP1",
},
releaseTitle: "The Daily Show 2024-09-22 Other Guest 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP1",
profile: &domain.DuplicateReleaseProfile{Title: true, Season: true, Episode: true, Year: true, Month: true, Day: true, SubTitle: true},
},
isDuplicate: false,
},
{
name: "22",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.2160p.BluRay.DTS-HD.5.1.x265-GROUP",
},
releaseTitle: "That.Movie.2023.2160p.BluRay.DD.2.0.x265-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "23",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.2160p.BluRay.DTS-HD.5.1.x265-GROUP",
},
releaseTitle: "That.Movie.2023.2160p.BluRay.DTS-HD.5.1.x265-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "24",
fields: fields{
releaseTitles: []string{
"That.Movie.2023.BluRay.2160p.x265.DTS-HD-GROUP",
"That.Movie.2023.BluRay.720p.x265.DTS-HD-GROUP",
"That.Movie.2023.2160p.BluRay.DD.5.1.x265-GROUP",
},
releaseTitle: "That.Movie.2023.2160p.BluRay.AC3.5.1.x265-GROUP",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "25",
fields: fields{
releaseTitles: []string{
//"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "26",
fields: fields{
releaseTitles: []string{
//"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "27",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Edition: false, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "28",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX-FraMeSToR",
"Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 Collectors Edition UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Edition: true, Source: true, Codec: true, Resolution: true, Audio: true, Group: true},
},
isDuplicate: true,
},
{
name: "29",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "30",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR DV HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "31",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 DV HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "32",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR10 HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HDR HEVC REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true},
},
isDuplicate: false,
},
{
name: "33",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 FRENCH UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "34",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 FRENCH UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true, Language: true},
},
isDuplicate: true,
},
{
name: "35",
fields: fields{
releaseTitles: []string{
"Despicable Me 4 2024 FRENCH UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
"Despicable Me 4 2024 GERMAN UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
},
releaseTitle: "Despicable Me 4 2024 UHD BluRay 2160p TrueHD Atmos 7.1 HEVC DV REMUX Hybrid-FraMeSToR",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, DynamicRange: true, Audio: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "36",
fields: fields{
releaseTitles: []string{
"Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
},
releaseTitle: "Road House 1989 1080p Blu-ray AVC LPCM 2.0-MONUMENT",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "37",
fields: fields{
releaseTitles: []string{
"Road House 1989 1080p ITA Blu-ray AVC LPCM 2.0-MONUMENT",
"Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
},
releaseTitle: "Road House 1989 1080p NOR Blu-ray AVC LPCM 2.0-MONUMENT",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true, Language: true},
},
isDuplicate: false,
},
{
name: "38",
fields: fields{
releaseTitles: []string{
"Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
},
releaseTitle: "Road House 1989 1080p GER Blu-ray AVC LPCM 2.0-MONUMENT",
profile: &domain.DuplicateReleaseProfile{Title: true, Year: true, Source: true, Codec: true, Resolution: true, Group: true, Language: true},
},
isDuplicate: true,
},
{
name: "39",
fields: fields{
releaseTitles: []string{
"The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.H.264-GROUP",
"The.Best.Show.2020.S04E10.1080p.AMZN.WEB-DL.DDP.5.1.HDR.DV.H.264-GROUP",
},
releaseTitle: "The Best Show 2020 S04E10 1080p AMZN WEB-DL DDP 5.1 H.264-GROUP",
profile: &domain.DuplicateReleaseProfile{ReleaseName: true},
},
isDuplicate: true,
},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("Check_Is_Duplicate_Release %s [%s]", tt.name, dbType), func(t *testing.T) {
ctx := context.Background()
// Setup
for _, rel := range tt.fields.releaseTitles {
mockRel := domain.NewRelease(mockIndexer)
mockRel.ParseString(rel)
mockRel.FilterID = filterMock.ID
err = releaseRepo.Store(ctx, mockRel)
assert.NoError(t, err)
ras := &domain.ReleaseActionStatus{
ID: 0,
Status: domain.ReleasePushStatusApproved,
Action: "test",
ActionID: int64(actionMock.ID),
Type: domain.ActionTypeTest,
Client: "",
Filter: "Test filter",
FilterID: int64(filterMock.ID),
Rejections: []string{},
ReleaseID: mockRel.ID,
Timestamp: time.Now(),
}
err = releaseRepo.StoreReleaseActionStatus(ctx, ras)
assert.NoError(t, err)
}
releases, err := releaseRepo.Find(ctx, domain.ReleaseQueryParams{})
assert.NoError(t, err)
assert.Len(t, releases.Data, len(tt.fields.releaseTitles))
compareRel := domain.NewRelease(mockIndexer)
compareRel.ParseString(tt.fields.releaseTitle)
// Execute
isDuplicate, err := releaseRepo.CheckIsDuplicateRelease(ctx, tt.fields.profile, compareRel)
// Verify
assert.NoError(t, err)
assert.Equal(t, tt.isDuplicate, isDuplicate)
// Cleanup
_ = releaseRepo.Delete(ctx, &domain.DeleteReleaseRequest{OlderThan: 0})
})
}
// Cleanup
//_ = releaseRepo.Delete(context.Background(), &domain.DeleteReleaseRequest{OlderThan: 0})
_ = actionRepo.Delete(context.Background(), &domain.DeleteActionRequest{ActionId: actionMock.ID})
_ = filterRepo.Delete(context.Background(), createdFilters[0].ID)
}
}

View file

@ -88,9 +88,42 @@ CREATE TABLE irc_channel
UNIQUE (network_id, name)
);
CREATE TABLE release_profile_duplicate
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
protocol BOOLEAN DEFAULT FALSE,
release_name BOOLEAN DEFAULT FALSE,
hash BOOLEAN DEFAULT FALSE,
title BOOLEAN DEFAULT FALSE,
sub_title BOOLEAN DEFAULT FALSE,
year BOOLEAN DEFAULT FALSE,
month BOOLEAN DEFAULT FALSE,
day BOOLEAN DEFAULT FALSE,
source BOOLEAN DEFAULT FALSE,
resolution BOOLEAN DEFAULT FALSE,
codec BOOLEAN DEFAULT FALSE,
container BOOLEAN DEFAULT FALSE,
dynamic_range BOOLEAN DEFAULT FALSE,
audio BOOLEAN DEFAULT FALSE,
release_group BOOLEAN DEFAULT FALSE,
season BOOLEAN DEFAULT FALSE,
episode BOOLEAN DEFAULT FALSE,
website BOOLEAN DEFAULT FALSE,
proper BOOLEAN DEFAULT FALSE,
repack BOOLEAN DEFAULT FALSE,
edition BOOLEAN DEFAULT FALSE,
language BOOLEAN DEFAULT FALSE
);
INSERT INTO release_profile_duplicate (id, name, protocol, release_name, hash, title, sub_title, year, month, day, source, resolution, codec, container, dynamic_range, audio, release_group, season, episode, website, proper, repack, edition, language)
VALUES (1, 'Exact release', 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(2, 'Movie', 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(3, 'TV', 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0);
CREATE TABLE filter
(
id INTEGER PRIMARY KEY,
id INTEGER PRIMARY KEY AUTOINCREMENT,
enabled BOOLEAN,
name TEXT NOT NULL,
min_size TEXT,
@ -159,7 +192,9 @@ CREATE TABLE filter
min_seeders INTEGER DEFAULT 0,
max_seeders INTEGER DEFAULT 0,
min_leechers INTEGER DEFAULT 0,
max_leechers INTEGER DEFAULT 0
max_leechers INTEGER DEFAULT 0,
release_profile_duplicate_id INTEGER,
FOREIGN KEY (release_profile_duplicate_id) REFERENCES release_profile_duplicate(id) ON DELETE SET NULL
);
CREATE INDEX filter_enabled_index
@ -273,8 +308,10 @@ CREATE TABLE "release"
group_id TEXT,
torrent_id TEXT,
torrent_name TEXT,
normalized_hash TEXT,
size INTEGER,
title TEXT,
sub_title TEXT,
category TEXT,
season INTEGER,
episode INTEGER,
@ -286,15 +323,24 @@ CREATE TABLE "release"
codec TEXT,
container TEXT,
hdr TEXT,
audio TEXT,
audio_channels TEXT,
release_group TEXT,
region TEXT,
language TEXT,
edition TEXT,
cut TEXT,
hybrid BOOLEAN,
proper BOOLEAN,
repack BOOLEAN,
website TEXT,
media_processing TEXT,
type TEXT,
origin TEXT,
tags TEXT [] DEFAULT '{}' NOT NULL,
uploader TEXT,
pre_time TEXT,
other TEXT [] DEFAULT '{}' NOT NULL,
filter_id INTEGER
REFERENCES filter
ON DELETE SET NULL
@ -312,6 +358,81 @@ CREATE INDEX release_timestamp_index
CREATE INDEX release_torrent_name_index
ON "release" (torrent_name);
CREATE INDEX release_normalized_hash_index
ON "release" (normalized_hash);
CREATE INDEX release_title_index
ON "release" (title);
CREATE INDEX release_sub_title_index
ON "release" (sub_title);
CREATE INDEX release_season_index
ON "release" (season);
CREATE INDEX release_episode_index
ON "release" (episode);
CREATE INDEX release_year_index
ON "release" (year);
CREATE INDEX release_month_index
ON "release" (month);
CREATE INDEX release_day_index
ON "release" (day);
CREATE INDEX release_resolution_index
ON "release" (resolution);
CREATE INDEX release_source_index
ON "release" (source);
CREATE INDEX release_codec_index
ON "release" (codec);
CREATE INDEX release_container_index
ON "release" (container);
CREATE INDEX release_hdr_index
ON "release" (hdr);
CREATE INDEX release_audio_index
ON "release" (audio);
CREATE INDEX release_audio_channels_index
ON "release" (audio_channels);
CREATE INDEX release_release_group_index
ON "release" (release_group);
CREATE INDEX release_language_index
ON "release" (language);
CREATE INDEX release_proper_index
ON "release" (proper);
CREATE INDEX release_repack_index
ON "release" (repack);
CREATE INDEX release_website_index
ON "release" (website);
CREATE INDEX release_media_processing_index
ON "release" (media_processing);
CREATE INDEX release_region_index
ON "release" (region);
CREATE INDEX release_edition_index
ON "release" (edition);
CREATE INDEX release_cut_index
ON "release" (cut);
CREATE INDEX release_hybrid_index
ON "release" (hybrid);
CREATE TABLE release_action_status
(
id INTEGER PRIMARY KEY,
@ -1716,5 +1837,152 @@ CREATE TABLE list_filter
ALTER TABLE filter
ADD COLUMN except_record_labels TEXT DEFAULT '';
`,
`CREATE TABLE release_profile_duplicate
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
protocol BOOLEAN DEFAULT FALSE,
release_name BOOLEAN DEFAULT FALSE,
hash BOOLEAN DEFAULT FALSE,
title BOOLEAN DEFAULT FALSE,
sub_title BOOLEAN DEFAULT FALSE,
year BOOLEAN DEFAULT FALSE,
month BOOLEAN DEFAULT FALSE,
day BOOLEAN DEFAULT FALSE,
source BOOLEAN DEFAULT FALSE,
resolution BOOLEAN DEFAULT FALSE,
codec BOOLEAN DEFAULT FALSE,
container BOOLEAN DEFAULT FALSE,
dynamic_range BOOLEAN DEFAULT FALSE,
audio BOOLEAN DEFAULT FALSE,
release_group BOOLEAN DEFAULT FALSE,
season BOOLEAN DEFAULT FALSE,
episode BOOLEAN DEFAULT FALSE,
website BOOLEAN DEFAULT FALSE,
proper BOOLEAN DEFAULT FALSE,
repack BOOLEAN DEFAULT FALSE,
edition BOOLEAN DEFAULT FALSE,
language BOOLEAN DEFAULT FALSE
);
INSERT INTO release_profile_duplicate (id, name, protocol, release_name, hash, title, sub_title, year, month, day, source, resolution, codec, container, dynamic_range, audio, release_group, season, episode, website, proper, repack, edition, language)
VALUES (1, 'Exact release', 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(2, 'Movie', 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(3, 'TV', 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0);
ALTER TABLE filter
ADD COLUMN release_profile_duplicate_id INTEGER
CONSTRAINT filter_release_profile_duplicate_id_fk
REFERENCES release_profile_duplicate (id)
ON DELETE SET NULL;
ALTER TABLE "release"
ADD normalized_hash TEXT;
ALTER TABLE "release"
ADD sub_title TEXT;
ALTER TABLE "release"
ADD audio TEXT;
ALTER TABLE "release"
ADD audio_channels TEXT;
ALTER TABLE "release"
ADD language TEXT;
ALTER TABLE "release"
ADD media_processing TEXT;
ALTER TABLE "release"
ADD edition TEXT;
ALTER TABLE "release"
ADD cut TEXT;
ALTER TABLE "release"
ADD hybrid TEXT;
ALTER TABLE "release"
ADD region TEXT;
ALTER TABLE "release"
ADD other TEXT [] DEFAULT '{}' NOT NULL;
CREATE INDEX release_normalized_hash_index
ON "release" (normalized_hash);
CREATE INDEX release_title_index
ON "release" (title);
CREATE INDEX release_sub_title_index
ON "release" (sub_title);
CREATE INDEX release_season_index
ON "release" (season);
CREATE INDEX release_episode_index
ON "release" (episode);
CREATE INDEX release_year_index
ON "release" (year);
CREATE INDEX release_month_index
ON "release" (month);
CREATE INDEX release_day_index
ON "release" (day);
CREATE INDEX release_resolution_index
ON "release" (resolution);
CREATE INDEX release_source_index
ON "release" (source);
CREATE INDEX release_codec_index
ON "release" (codec);
CREATE INDEX release_container_index
ON "release" (container);
CREATE INDEX release_hdr_index
ON "release" (hdr);
CREATE INDEX release_audio_index
ON "release" (audio);
CREATE INDEX release_audio_channels_index
ON "release" (audio_channels);
CREATE INDEX release_release_group_index
ON "release" (release_group);
CREATE INDEX release_proper_index
ON "release" (proper);
CREATE INDEX release_repack_index
ON "release" (repack);
CREATE INDEX release_website_index
ON "release" (website);
CREATE INDEX release_media_processing_index
ON "release" (media_processing);
CREATE INDEX release_language_index
ON "release" (language);
CREATE INDEX release_region_index
ON "release" (region);
CREATE INDEX release_edition_index
ON "release" (edition);
CREATE INDEX release_cut_index
ON "release" (cut);
CREATE INDEX release_hybrid_index
ON "release" (hybrid);
`,
}

View file

@ -29,6 +29,7 @@ func toNullInt32(s int32) sql.NullInt32 {
Valid: s != 0,
}
}
func toNullInt64(s int64) sql.NullInt64 {
return sql.NullInt64{
Int64: s,