From 284a2f95904cdf810f1c034afe3cdeaf6e17eecc Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Sun, 16 Jan 2022 13:50:21 +0100 Subject: [PATCH] feat: add hdr filtering (#86) * feat: filter hdr content * feat: check filter hdr * feat: improve hdr parse and filter and tests --- internal/database/filter.go | 18 ++- internal/database/migrate.go | 9 ++ internal/domain/filter.go | 2 + internal/domain/release.go | 40 ++++++- internal/domain/release_test.go | 168 +++++++++++++++++++++++++++ web/src/components/inputs/select.tsx | 4 +- web/src/domain/constants.ts | 14 +++ web/src/domain/interfaces.ts | 2 + web/src/index.css | 40 ++++--- web/src/screens/filters/details.tsx | 9 +- 10 files changed, 284 insertions(+), 22 deletions(-) diff --git a/internal/database/filter.go b/internal/database/filter.go index 31f86d9..7b7ae6c 100644 --- a/internal/database/filter.go +++ b/internal/database/filter.go @@ -59,7 +59,7 @@ func (r *FilterRepo) FindByID(filterID int) (*domain.Filter, error) { //r.db.lock.RLock() //defer r.db.lock.RUnlock() - row := r.db.handler.QueryRow("SELECT id, enabled, name, min_size, max_size, delay, match_releases, except_releases, use_regex, match_release_groups, except_release_groups, scene, freeleech, freeleech_percent, shows, seasons, episodes, resolutions, codecs, sources, containers, years, match_categories, except_categories, match_uploaders, except_uploaders, tags, except_tags, created_at, updated_at FROM filter WHERE id = ?", filterID) + row := r.db.handler.QueryRow("SELECT id, enabled, name, min_size, max_size, delay, match_releases, except_releases, use_regex, match_release_groups, except_release_groups, scene, freeleech, freeleech_percent, shows, seasons, episodes, resolutions, codecs, sources, containers, match_hdr, except_hdr, years, match_categories, except_categories, match_uploaders, except_uploaders, tags, except_tags, created_at, updated_at FROM filter WHERE id = ?", filterID) var f domain.Filter @@ -71,7 +71,7 @@ func (r *FilterRepo) FindByID(filterID int) (*domain.Filter, error) { var useRegex, scene, freeleech sql.NullBool var delay sql.NullInt32 - if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), &years, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { log.Error().Stack().Err(err).Msgf("filter: %v : error scanning data to struct", filterID) return nil, err } @@ -129,6 +129,8 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e f.codecs, f.sources, f.containers, + f.match_hdr, + f.except_hdr, f.years, f.match_categories, f.except_categories, @@ -158,7 +160,7 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e var useRegex, scene, freeleech sql.NullBool var delay sql.NullInt32 - if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), &years, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { log.Error().Stack().Err(err).Msg("error scanning data to struct") return nil, err } @@ -225,6 +227,8 @@ func (r *FilterRepo) Store(filter domain.Filter) (*domain.Filter, error) { codecs, sources, containers, + match_hdr, + except_hdr, years, match_categories, except_categories, @@ -233,7 +237,7 @@ func (r *FilterRepo) Store(filter domain.Filter) (*domain.Filter, error) { tags, except_tags ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27) ON CONFLICT DO NOTHING`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29) ON CONFLICT DO NOTHING`, filter.Name, filter.Enabled, filter.MinSize, @@ -254,6 +258,8 @@ func (r *FilterRepo) Store(filter domain.Filter) (*domain.Filter, error) { pq.Array(filter.Codecs), pq.Array(filter.Sources), pq.Array(filter.Containers), + pq.Array(filter.MatchHDR), + pq.Array(filter.ExceptHDR), filter.Years, filter.MatchCategories, filter.ExceptCategories, @@ -303,6 +309,8 @@ func (r *FilterRepo) Update(ctx context.Context, filter domain.Filter) (*domain. codecs = ?, sources = ?, containers = ?, + match_hdr = ?, + except_hdr = ?, years = ?, match_categories = ?, except_categories = ?, @@ -332,6 +340,8 @@ func (r *FilterRepo) Update(ctx context.Context, filter domain.Filter) (*domain. pq.Array(filter.Codecs), pq.Array(filter.Sources), pq.Array(filter.Containers), + pq.Array(filter.MatchHDR), + pq.Array(filter.ExceptHDR), filter.Years, filter.MatchCategories, filter.ExceptCategories, diff --git a/internal/database/migrate.go b/internal/database/migrate.go index b8c57b3..9d31b28 100644 --- a/internal/database/migrate.go +++ b/internal/database/migrate.go @@ -81,6 +81,8 @@ CREATE TABLE filter codecs TEXT [] DEFAULT '{}' NOT NULL, sources TEXT [] DEFAULT '{}' NOT NULL, containers TEXT [] DEFAULT '{}' NOT NULL, + match_hdr TEXT [] DEFAULT '{}', + except_hdr TEXT [] DEFAULT '{}', years TEXT, match_categories TEXT, except_categories TEXT, @@ -280,6 +282,13 @@ var migrations = []string{ ALTER TABLE "release" DROP COLUMN push_status; `, + ` + ALTER TABLE "filter" + ADD COLUMN match_hdr TEXT [] DEFAULT '{}'; + + ALTER TABLE "filter" + ADD COLUMN except_hdr TEXT [] DEFAULT '{}'; + `, } func (db *SqliteDB) migrate() error { diff --git a/internal/domain/filter.go b/internal/domain/filter.go index beba471..a260385 100644 --- a/internal/domain/filter.go +++ b/internal/domain/filter.go @@ -48,6 +48,8 @@ type Filter struct { Codecs []string `json:"codecs"` // XviD, DivX, x264, h.264 (or h264), mpeg2 (or mpeg-2), VC-1 (or VC1), WMV, Remux, h.264 Remux (or h264 Remux), VC-1 Remux (or VC1 Remux). Sources []string `json:"sources"` // DSR, PDTV, HDTV, HR.PDTV, HR.HDTV, DVDRip, DVDScr, BDr, BD5, BD9, BDRip, BRRip, DVDR, MDVDR, HDDVD, HDDVDRip, BluRay, WEB-DL, TVRip, CAM, R5, TELESYNC, TS, TELECINE, TC. TELESYNC and TS are synonyms (you don't need both). Same for TELECINE and TC Containers []string `json:"containers"` + MatchHDR []string `json:"match_hdr"` + ExceptHDR []string `json:"except_hdr"` Years string `json:"years"` Artists string `json:"artists"` Albums string `json:"albums"` diff --git a/internal/domain/release.go b/internal/domain/release.go index 3a637e1..bddcf4f 100644 --- a/internal/domain/release.go +++ b/internal/domain/release.go @@ -248,7 +248,7 @@ func (r *Release) extractContainerFromTags(tag string) error { } func (r *Release) extractHDR() error { - v, err := findLast(r.TorrentName, `(?i)(HDR10\+|HDR10|DoVi HDR|DV HDR|HDR|DV|DoVi|Dolby Vision \+ HDR10|Dolby Vision)`) + v, err := findLast(r.TorrentName, `(?i)[\. ](HDR10\+|HDR10|DoVi[\. ]HDR|DV[\. ]HDR10\+|DV[\. ]HDR10|DV[\. ]HDR|HDR|DV|DoVi|Dolby[\. ]Vision[\. ]\+[\. ]HDR10|Dolby[\. ]Vision)[\. ]`) if err != nil { return err } @@ -711,6 +711,16 @@ func (r *Release) CheckFilter(filter Filter) bool { return false } + if len(filter.MatchHDR) > 0 && !checkMultipleFilterHDR(filter.MatchHDR, r.HDR, r.TorrentName) { + r.addRejection("hdr not matching") + return false + } + + if len(filter.ExceptHDR) > 0 && checkMultipleFilterHDR(filter.ExceptHDR, r.HDR, r.TorrentName) { + r.addRejection("unwanted hdr") + return false + } + if filter.Years != "" && !checkFilterIntStrings(r.Year, filter.Years) { r.addRejection("year not matching") return false @@ -1017,6 +1027,34 @@ func checkMultipleFilterGroups(filterList string, vars ...string) bool { return false } +func checkMultipleFilterHDR(filterList []string, vars ...string) bool { + for _, name := range vars { + name = strings.ToLower(name) + + for _, s := range filterList { + s = strings.ToLower(strings.Trim(s, " ")) + // check if line contains * or ?, if so try wildcard match, otherwise try substring match + a := strings.ContainsAny(s, "?|*") + if a { + match := wildcard.Match(s, name) + if match { + return true + } + } else { + split := SplitAny(name, " .-") + for _, c := range split { + if c == s { + return true + } + } + continue + } + } + } + + return false +} + func checkFilterSource(name string, filterList []string) bool { // remove dash (-) in blu-ray web-dl and make lowercase name = strings.ToLower(strings.ReplaceAll(name, "-", "")) diff --git a/internal/domain/release_test.go b/internal/domain/release_test.go index c4f9700..e1a7f01 100644 --- a/internal/domain/release_test.go +++ b/internal/domain/release_test.go @@ -863,6 +863,174 @@ func TestRelease_CheckFilter(t *testing.T) { }, want: false, }, + { + name: "match_hdr_1", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + MatchHDR: []string{"DV", "HDR"}, + }, + }, + want: true, + }, + { + name: "match_hdr_2", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DoVi HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + MatchHDR: []string{"DV", "HDR"}, + }, + }, + want: false, + }, + { + name: "match_hdr_3", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DoVi HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + ExceptHDR: []string{"DV", "HDR", "DoVi"}, + }, + }, + want: false, + }, + { + name: "match_hdr_4", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + MatchHDR: []string{"DV", "HDR", "DoVi"}, + }, + }, + want: false, + }, + { + name: "match_hdr_5", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + ExceptHDR: []string{"DV", "HDR", "DoVi"}, + }, + }, + want: true, + }, + { + name: "match_hdr_6", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos HDR HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + ExceptHDR: []string{"DV", "DoVi"}, + }, + }, + want: true, + }, + { + name: "match_hdr_7", + fields: &Release{ + TorrentName: "Good show dvorak shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos HDR HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show dvorak shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + ExceptHDR: []string{"DV", "DoVi"}, + }, + }, + want: true, + }, + { + name: "match_hdr_8", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos HDR10+ HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "GROUP", + ExceptReleases: "NORDiC", + MatchHDR: []string{"DV", "DoVi", "HDR10+"}, + }, + }, + want: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/web/src/components/inputs/select.tsx b/web/src/components/inputs/select.tsx index ed4a875..9de7a74 100644 --- a/web/src/components/inputs/select.tsx +++ b/web/src/components/inputs/select.tsx @@ -19,7 +19,7 @@ const MultiSelect: React.FC = ({ label, options, className, - columns + columns, }) => (
= ({ setFieldValue(field.name, am) }} - className="dark:bg-gray-700" + className="dark:bg-gray-700 dark" /> )} diff --git a/web/src/domain/constants.ts b/web/src/domain/constants.ts index 8d20405..fab9a44 100644 --- a/web/src/domain/constants.ts +++ b/web/src/domain/constants.ts @@ -62,6 +62,20 @@ export const containers = [ export const CONTAINER_OPTIONS = containers.map(v => ({ value: v, label: v, key: v})); +export const hdr = [ + "HDR", + "HDR10", + "HDR10+", + "DV", + "DV HDR", + "DV HDR10", + "DV HDR10+", + "DoVi", + "Dolby Vision", +]; + +export const HDR_OPTIONS = hdr.map(v => ({ value: v, label: v, key: v})); + export interface radioFieldsetOption { label: string; description: string; diff --git a/web/src/domain/interfaces.ts b/web/src/domain/interfaces.ts index 10debd6..f903407 100644 --- a/web/src/domain/interfaces.ts +++ b/web/src/domain/interfaces.ts @@ -83,6 +83,8 @@ export interface Filter { sources: string[]; codecs: string[]; containers: string[]; + match_hdr: string[]; + except_hdr: string[]; seasons: string; episodes: string; match_releases: string; diff --git a/web/src/index.css b/web/src/index.css index 2b3ea29..f6e5836 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -4,46 +4,58 @@ body { margin: 0; - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', - 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen", + "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } code { - font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', + font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", monospace; } @keyframes enter { 0% { - transform: scale(.9); - opacity: 0 + transform: scale(0.9); + opacity: 0; } to { - transform: scale(1); - opacity: 1 + transform: scale(1); + opacity: 1; } } .animate-enter { - animation: enter .2s ease-out + animation: enter 0.2s ease-out; } @keyframes leave { 0% { - transform: scale(1); - opacity: 1 + transform: scale(1); + opacity: 1; } to { - transform: scale(.9); - opacity: 0 + transform: scale(0.9); + opacity: 0; } } .animate-leave { - animation: leave .15s ease-in forwards -} \ No newline at end of file + animation: leave 0.15s ease-in forwards; +} + +@media (prefers-color-scheme: dark) { + .rmsc.dark { + --rmsc-main: #4285f4; + --rmsc-hover: #0e0c0a; + --rmsc-selected: #1d1915; + --rmsc-border: #35353a; + --rmsc-gray: #555555; + --rmsc-bg: #27272a; + color: #fff; + } +} diff --git a/web/src/screens/filters/details.tsx b/web/src/screens/filters/details.tsx index 28271ae..bc74d9f 100644 --- a/web/src/screens/filters/details.tsx +++ b/web/src/screens/filters/details.tsx @@ -16,7 +16,7 @@ import { Action, ActionType, DownloadClient, Filter, Indexer } from "../../domai import { useToggle } from "../../hooks/hooks"; import { useMutation, useQuery } from "react-query"; import { queryClient } from "../../App"; -import { CONTAINER_OPTIONS, CODECS_OPTIONS, RESOLUTION_OPTIONS, SOURCES_OPTIONS, ActionTypeNameMap, ActionTypeOptions } from "../../domain/constants"; +import { CONTAINER_OPTIONS, CODECS_OPTIONS, RESOLUTION_OPTIONS, SOURCES_OPTIONS, ActionTypeNameMap, ActionTypeOptions, HDR_OPTIONS } from "../../domain/constants"; import DEBUG from "../../components/debug"; import { TitleSubtitle } from "../../components/headings"; @@ -233,6 +233,8 @@ export default function FilterDetails() { sources: data.sources || [], codecs: data.codecs || [], containers: data.containers || [], + match_hdr: data.match_hdr || [], + except_hdr: data.except_hdr || [], seasons: data.seasons, episodes: data.episodes, match_releases: data.match_releases, @@ -393,6 +395,11 @@ function MoviesTv() {
+ +
+ + +
)