From 5183f7683ac34688175cbb3ea22ec37ec06c351b Mon Sep 17 00:00:00 2001 From: ze0s <43699394+zze0s@users.noreply.github.com> Date: Tue, 18 Oct 2022 23:52:36 +0200 Subject: [PATCH] feat(filters): filtering for raw releaseTags (#506) feat(filters): add releaseTags filtering --- internal/database/filter.go | 36 ++++++++++++++++-- internal/database/postgres_migrate.go | 12 ++++++ internal/database/sqlite_migrate.go | 12 ++++++ internal/domain/filter.go | 26 +++++++++++++ internal/domain/filter_test.go | 42 +++++++++++++++++++++ web/src/screens/filters/details.tsx | 53 ++++++++++++++++++++------- web/src/types/Filter.d.ts | 3 ++ 7 files changed, 166 insertions(+), 18 deletions(-) diff --git a/internal/database/filter.go b/internal/database/filter.go index 77dce66..58c7f6f 100644 --- a/internal/database/filter.go +++ b/internal/database/filter.go @@ -184,6 +184,9 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter "use_regex", "match_release_groups", "except_release_groups", + "match_release_tags", + "except_release_tags", + "use_regex_release_tags", "scene", "freeleech", "freeleech_percent", @@ -242,11 +245,11 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter } var f domain.Filter - var minSize, maxSize, maxDownloadsUnit, matchReleases, exceptReleases, matchReleaseGroups, exceptReleaseGroups, freeleechPercent, shows, seasons, episodes, years, artists, albums, matchCategories, exceptCategories, matchUploaders, exceptUploaders, tags, exceptTags, extScriptCmd, extScriptArgs, extWebhookHost, extWebhookData sql.NullString + var minSize, maxSize, maxDownloadsUnit, matchReleases, exceptReleases, matchReleaseGroups, exceptReleaseGroups, matchReleaseTags, exceptReleaseTags, freeleechPercent, shows, seasons, episodes, years, artists, albums, matchCategories, exceptCategories, matchUploaders, exceptUploaders, tags, exceptTags, extScriptCmd, extScriptArgs, extWebhookHost, extWebhookData sql.NullString var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac, extScriptEnabled, extWebhookEnabled sql.NullBool var delay, maxDownloads, logScore, extWebhookStatus, extScriptStatus sql.NullInt32 - if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &f.Priority, &maxDownloads, &maxDownloadsUnit, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), pq.Array(&f.MatchOther), pq.Array(&f.ExceptOther), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, pq.Array(&f.Origins), pq.Array(&f.ExceptOrigins), &extScriptEnabled, &extScriptCmd, &extScriptArgs, &extScriptStatus, &extWebhookEnabled, &extWebhookHost, &extWebhookData, &extWebhookStatus, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &f.Priority, &maxDownloads, &maxDownloadsUnit, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &matchReleaseTags, &exceptReleaseTags, &f.UseRegexReleaseTags, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), pq.Array(&f.MatchOther), pq.Array(&f.ExceptOther), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, pq.Array(&f.Origins), pq.Array(&f.ExceptOrigins), &extScriptEnabled, &extScriptCmd, &extScriptArgs, &extScriptStatus, &extWebhookEnabled, &extWebhookHost, &extWebhookData, &extWebhookStatus, &f.CreatedAt, &f.UpdatedAt); err != nil { return nil, errors.Wrap(err, "error scanning row") } @@ -259,6 +262,8 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter f.ExceptReleases = exceptReleases.String f.MatchReleaseGroups = matchReleaseGroups.String f.ExceptReleaseGroups = exceptReleaseGroups.String + f.MatchReleaseTags = matchReleaseTags.String + f.ExceptReleaseTags = exceptReleaseTags.String f.FreeleechPercent = freeleechPercent.String f.Shows = shows.String f.Seasons = seasons.String @@ -335,6 +340,9 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, tx *Tx, indexe "f.use_regex", "f.match_release_groups", "f.except_release_groups", + "f.match_release_tags", + "f.except_release_tags", + "f.use_regex_release_tags", "f.scene", "f.freeleech", "f.freeleech_percent", @@ -403,11 +411,11 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, tx *Tx, indexe for rows.Next() { var f domain.Filter - var minSize, maxSize, maxDownloadsUnit, matchReleases, exceptReleases, matchReleaseGroups, exceptReleaseGroups, freeleechPercent, shows, seasons, episodes, years, artists, albums, matchCategories, exceptCategories, matchUploaders, exceptUploaders, tags, exceptTags, extScriptCmd, extScriptArgs, extWebhookHost, extWebhookData sql.NullString + var minSize, maxSize, maxDownloadsUnit, matchReleases, exceptReleases, matchReleaseGroups, exceptReleaseGroups, matchReleaseTags, exceptReleaseTags, freeleechPercent, shows, seasons, episodes, years, artists, albums, matchCategories, exceptCategories, matchUploaders, exceptUploaders, tags, exceptTags, extScriptCmd, extScriptArgs, extWebhookHost, extWebhookData sql.NullString var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac, extScriptEnabled, extWebhookEnabled sql.NullBool var delay, maxDownloads, logScore, extWebhookStatus, extScriptStatus sql.NullInt32 - if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &f.Priority, &maxDownloads, &maxDownloadsUnit, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), pq.Array(&f.MatchOther), pq.Array(&f.ExceptOther), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, pq.Array(&f.Origins), pq.Array(&f.ExceptOrigins), &extScriptEnabled, &extScriptCmd, &extScriptArgs, &extScriptStatus, &extWebhookEnabled, &extWebhookHost, &extWebhookData, &extWebhookStatus, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &f.Priority, &maxDownloads, &maxDownloadsUnit, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &matchReleaseTags, &exceptReleaseTags, &f.UseRegexReleaseTags, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), pq.Array(&f.MatchOther), pq.Array(&f.ExceptOther), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, pq.Array(&f.Origins), pq.Array(&f.ExceptOrigins), &extScriptEnabled, &extScriptCmd, &extScriptArgs, &extScriptStatus, &extWebhookEnabled, &extWebhookHost, &extWebhookData, &extWebhookStatus, &f.CreatedAt, &f.UpdatedAt); err != nil { return nil, errors.Wrap(err, "error scanning row") } @@ -420,6 +428,8 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, tx *Tx, indexe f.ExceptReleases = exceptReleases.String f.MatchReleaseGroups = matchReleaseGroups.String f.ExceptReleaseGroups = exceptReleaseGroups.String + f.MatchReleaseTags = matchReleaseTags.String + f.ExceptReleaseTags = exceptReleaseTags.String f.FreeleechPercent = freeleechPercent.String f.Shows = shows.String f.Seasons = seasons.String @@ -474,6 +484,9 @@ func (r *FilterRepo) Store(ctx context.Context, filter domain.Filter) (*domain.F "use_regex", "match_release_groups", "except_release_groups", + "match_release_tags", + "except_release_tags", + "use_regex_release_tags", "scene", "freeleech", "freeleech_percent", @@ -530,6 +543,9 @@ func (r *FilterRepo) Store(ctx context.Context, filter domain.Filter) (*domain.F filter.UseRegex, filter.MatchReleaseGroups, filter.ExceptReleaseGroups, + filter.MatchReleaseTags, + filter.ExceptReleaseTags, + filter.UseRegexReleaseTags, filter.Scene, filter.Freeleech, filter.FreeleechPercent, @@ -605,6 +621,9 @@ func (r *FilterRepo) Update(ctx context.Context, filter domain.Filter) (*domain. Set("except_releases", filter.ExceptReleases). Set("match_release_groups", filter.MatchReleaseGroups). Set("except_release_groups", filter.ExceptReleaseGroups). + Set("match_release_tags", filter.MatchReleaseTags). + Set("except_release_tags", filter.ExceptReleaseTags). + Set("use_regex_release_tags", filter.UseRegexReleaseTags). Set("scene", filter.Scene). Set("freeleech", filter.Freeleech). Set("freeleech_percent", filter.FreeleechPercent). @@ -706,6 +725,15 @@ func (r *FilterRepo) UpdatePartial(ctx context.Context, filter domain.FilterUpda if filter.ExceptReleaseGroups != nil { q = q.Set("except_release_groups", filter.ExceptReleaseGroups) } + if filter.MatchReleaseTags != nil { + q = q.Set("match_release_tags", filter.MatchReleaseTags) + } + if filter.ExceptReleaseTags != nil { + q = q.Set("except_release_tags", filter.ExceptReleaseTags) + } + if filter.UseRegexReleaseTags != nil { + q = q.Set("use_regex_release_tags", filter.UseRegexReleaseTags) + } if filter.Scene != nil { q = q.Set("scene", filter.Scene) } diff --git a/internal/database/postgres_migrate.go b/internal/database/postgres_migrate.go index f62c186..aaf0c93 100644 --- a/internal/database/postgres_migrate.go +++ b/internal/database/postgres_migrate.go @@ -74,6 +74,9 @@ CREATE TABLE filter use_regex BOOLEAN, match_release_groups TEXT, except_release_groups TEXT, + match_release_tags TEXT, + except_release_tags TEXT, + use_regex_release_tags BOOLEAN DEFAULT FALSE, scene BOOLEAN, freeleech BOOLEAN, freeleech_percent TEXT, @@ -577,4 +580,13 @@ CREATE INDEX indexer_identifier_index ALTER TABLE feed ADD COLUMN cookie TEXT; `, + `ALTER TABLE filter + ADD COLUMN match_release_tags TEXT; + + ALTER TABLE filter + ADD COLUMN except_release_tags TEXT; + + ALTER TABLE filter + ADD COLUMN use_regex_release_tags BOOLEAN DEFAULT FALSE; + `, } diff --git a/internal/database/sqlite_migrate.go b/internal/database/sqlite_migrate.go index b0a73f3..f72e4e0 100644 --- a/internal/database/sqlite_migrate.go +++ b/internal/database/sqlite_migrate.go @@ -74,6 +74,9 @@ CREATE TABLE filter use_regex BOOLEAN, match_release_groups TEXT, except_release_groups TEXT, + match_release_tags TEXT, + except_release_tags TEXT, + use_regex_release_tags BOOLEAN DEFAULT FALSE, scene BOOLEAN, freeleech BOOLEAN, freeleech_percent TEXT, @@ -897,4 +900,13 @@ CREATE INDEX indexer_identifier_index ALTER TABLE feed ADD COLUMN cookie TEXT; `, + `ALTER TABLE filter + ADD COLUMN match_release_tags TEXT; + + ALTER TABLE filter + ADD COLUMN except_release_tags TEXT; + + ALTER TABLE filter + ADD COLUMN use_regex_release_tags BOOLEAN DEFAULT FALSE; + `, } diff --git a/internal/domain/filter.go b/internal/domain/filter.go index 2a2e020..b74045f 100644 --- a/internal/domain/filter.go +++ b/internal/domain/filter.go @@ -112,6 +112,9 @@ type Filter struct { ExceptTags string `json:"except_tags,omitempty"` TagsAny string `json:"tags_any,omitempty"` ExceptTagsAny string `json:"except_tags_any,omitempty"` + MatchReleaseTags string `json:"match_release_tags,omitempty"` + ExceptReleaseTags string `json:"except_release_tags,omitempty"` + UseRegexReleaseTags bool `json:"use_regex_release_tags,omitempty"` ExternalScriptEnabled bool `json:"external_script_enabled,omitempty"` ExternalScriptCmd string `json:"external_script_cmd,omitempty"` ExternalScriptArgs string `json:"external_script_args,omitempty"` @@ -141,6 +144,9 @@ type FilterUpdate struct { UseRegex *bool `json:"use_regex,omitempty"` MatchReleaseGroups *string `json:"match_release_groups,omitempty"` ExceptReleaseGroups *string `json:"except_release_groups,omitempty"` + MatchReleaseTags *string `json:"match_release_tags,omitempty"` + ExceptReleaseTags *string `json:"except_release_tags,omitempty"` + UseRegexReleaseTags *bool `json:"use_regex_release_tags,omitempty"` Scene *bool `json:"scene,omitempty"` Origins *[]string `json:"origins,omitempty"` ExceptOrigins *[]string `json:"except_origins,omitempty"` @@ -261,6 +267,26 @@ func (f Filter) CheckFilter(r *Release) ([]string, bool) { r.addRejectionF("unwanted release group. got: %v unwanted: %v", r.Group, f.ExceptReleaseGroups) } + // check raw releaseTags string + if f.UseRegexReleaseTags { + if f.MatchReleaseTags != "" && !matchRegex(r.ReleaseTags, f.MatchReleaseTags) { + r.addRejectionF("match release tags regex not matching. got: %v want: %v", r.ReleaseTags, f.MatchReleaseTags) + } + + if f.ExceptReleaseTags != "" && matchRegex(r.ReleaseTags, f.ExceptReleaseTags) { + r.addRejectionF("except release tags regex: unwanted release. got: %v want: %v", r.ReleaseTags, f.ExceptReleaseTags) + } + + } else { + if f.MatchReleaseTags != "" && !containsFuzzy(r.ReleaseTags, f.MatchReleaseTags) { + r.addRejectionF("match release tags not matching. got: %v want: %v", r.ReleaseTags, f.MatchReleaseTags) + } + + if f.ExceptReleaseTags != "" && containsFuzzy(r.ReleaseTags, f.ExceptReleaseTags) { + r.addRejectionF("except release tags: unwanted release. got: %v want: %v", r.ReleaseTags, f.ExceptReleaseTags) + } + } + if f.MatchUploaders != "" && !contains(r.Uploader, f.MatchUploaders) { r.addRejectionF("uploaders not matching. got: %v want: %v", r.Uploader, f.MatchUploaders) } diff --git a/internal/domain/filter_test.go b/internal/domain/filter_test.go index bb9aea9..eaecda2 100644 --- a/internal/domain/filter_test.go +++ b/internal/domain/filter_test.go @@ -1581,6 +1581,45 @@ func TestFilter_CheckFilter1(t *testing.T) { wantRejections: nil, wantMatch: true, }, + { + name: "test_39", + fields: fields{ + UseRegexReleaseTags: true, + MatchReleaseTags: ".*1080p.+(group1|group3)", + }, + args: args{&Release{TorrentName: "Show.Name.S01.DV.2160p.ATVP.WEB-DL.DDPA5.1.x265-GROUP2", ReleaseTags: "MKV | x264 | WEB | P2P"}}, + wantRejections: []string{"match release tags regex not matching. got: MKV | x264 | WEB | P2P want: .*1080p.+(group1|group3)"}, + wantMatch: false, + }, + { + name: "test_40", + fields: fields{ + UseRegexReleaseTags: true, + MatchReleaseTags: "foreign - 16", + }, + args: args{&Release{TorrentName: "Show.Name.S01.DV.2160p.ATVP.WEB-DL.DDPA5.1.x265-GROUP2", ReleaseTags: "MKV | x264 | WEB | P2P | Foreign - 17"}}, + wantRejections: []string{"match release tags regex not matching. got: MKV | x264 | WEB | P2P | Foreign - 17 want: foreign - 16"}, + wantMatch: false, + }, + { + name: "test_41", + fields: fields{ + UseRegexReleaseTags: true, + MatchReleaseTags: "foreign - 17", + }, + args: args{&Release{TorrentName: "Show.Name.S01.DV.2160p.ATVP.WEB-DL.DDPA5.1.x265-GROUP2", ReleaseTags: "MKV | x264 | WEB | P2P | Foreign - 17"}}, + wantMatch: true, + }, + { + name: "test_42", + fields: fields{ + UseRegexReleaseTags: true, + MatchReleaseTags: "foreign - 17", + }, + args: args{&Release{TorrentName: "Show.Name.S01.DV.2160p.ATVP.WEB-DL.DDPA5.1.x265-GROUP2", ReleaseTags: ""}}, + wantRejections: []string{"match release tags regex not matching. got: want: foreign - 17"}, + wantMatch: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1601,6 +1640,9 @@ func TestFilter_CheckFilter1(t *testing.T) { UseRegex: tt.fields.UseRegex, MatchReleaseGroups: tt.fields.MatchReleaseGroups, ExceptReleaseGroups: tt.fields.ExceptReleaseGroups, + MatchReleaseTags: tt.fields.MatchReleaseTags, + ExceptReleaseTags: tt.fields.ExceptReleaseTags, + UseRegexReleaseTags: tt.fields.UseRegexReleaseTags, Scene: tt.fields.Scene, Origins: tt.fields.Origins, ExceptOrigins: tt.fields.ExceptOrigins, diff --git a/web/src/screens/filters/details.tsx b/web/src/screens/filters/details.tsx index d79ed5f..df4f392 100644 --- a/web/src/screens/filters/details.tsx +++ b/web/src/screens/filters/details.tsx @@ -250,6 +250,9 @@ export default function FilterDetails() { except_releases: filter.except_releases, match_release_groups: filter.match_release_groups, except_release_groups: filter.except_release_groups, + match_release_tags: filter.match_release_tags, + except_release_tags: filter.except_release_tags, + use_regex_release_tags: filter.use_regex_release_tags, match_categories: filter.match_categories, except_categories: filter.except_categories, tags: filter.tags, @@ -454,20 +457,7 @@ export function Advanced() {
-
- - Info -
- Warning! autobrr has extensive filtering built-in - only use this if nothing else works. If you need help please ask. -
-
+ @@ -500,6 +490,18 @@ export function Advanced() { + +
+ + + + +
+ +
+
+ +
@@ -511,6 +513,29 @@ export function Advanced() { ); } +interface WarningAlertProps { + text: string; +} + +function WarningAlert({ text }: WarningAlertProps) { + return ( +
+ + Info +
+ Warning! {text} +
+
+ ); +} + interface CollapsableSectionProps { title: string; subtitle: string; diff --git a/web/src/types/Filter.d.ts b/web/src/types/Filter.d.ts index 8e4705c..9733143 100644 --- a/web/src/types/Filter.d.ts +++ b/web/src/types/Filter.d.ts @@ -15,6 +15,9 @@ interface Filter { use_regex: boolean; match_release_groups: string; except_release_groups: string; + match_release_tags: string; + except_release_tags: string; + use_regex_release_tags: boolean; scene: boolean; origins: string[]; except_origins: string[];