From 9b85f512e5e797e11a61127f92d8939d2da669f4 Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Mon, 31 Jan 2022 19:16:47 +0100 Subject: [PATCH] refactor(filters): music sources to media (#104) --- internal/database/filter.go | 13 ++-- internal/database/migrate.go | 107 ++++++++++++++++++++++++++++ internal/domain/filter.go | 30 ++++---- internal/domain/release.go | 5 ++ internal/domain/release_test.go | 12 ++-- web/src/domain/constants.ts | 9 --- web/src/screens/filters/details.tsx | 3 +- web/src/types/Action.d.ts | 1 + 8 files changed, 145 insertions(+), 35 deletions(-) diff --git a/internal/database/filter.go b/internal/database/filter.go index 549a78f..c243187 100644 --- a/internal/database/filter.go +++ b/internal/database/filter.go @@ -58,7 +58,7 @@ func (r *FilterRepo) FindByID(filterID int) (*domain.Filter, error) { //r.db.lock.RLock() //defer r.db.lock.RUnlock() - row := r.db.handler.QueryRow("SELECT id, enabled, name, min_size, max_size, delay, match_releases, except_releases, use_regex, match_release_groups, except_release_groups, scene, freeleech, freeleech_percent, shows, seasons, episodes, resolutions, codecs, sources, containers, match_hdr, except_hdr, years, artists, albums, release_types_match, formats, quality, log_score, has_log, has_cue, perfect_flac, match_categories, except_categories, match_uploaders, except_uploaders, tags, except_tags, created_at, updated_at FROM filter WHERE id = ?", filterID) + row := r.db.handler.QueryRow("SELECT id, enabled, name, min_size, max_size, delay, match_releases, except_releases, use_regex, match_release_groups, except_release_groups, scene, freeleech, freeleech_percent, shows, seasons, episodes, resolutions, codecs, sources, containers, match_hdr, except_hdr, years, artists, albums, release_types_match, formats, quality, media, log_score, has_log, has_cue, perfect_flac, match_categories, except_categories, match_uploaders, except_uploaders, tags, except_tags, created_at, updated_at FROM filter WHERE id = ?", filterID) var f domain.Filter @@ -70,7 +70,7 @@ func (r *FilterRepo) FindByID(filterID int) (*domain.Filter, error) { var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac sql.NullBool var delay, logScore sql.NullInt32 - if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { log.Error().Stack().Err(err).Msgf("filter: %v : error scanning data to struct", filterID) return nil, err } @@ -142,6 +142,7 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e f.release_types_match, f.formats, f.quality, + f.media, f.log_score, f.has_log, f.has_cue, @@ -174,7 +175,7 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac sql.NullBool var delay, logScore sql.NullInt32 - if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil { log.Error().Stack().Err(err).Msg("error scanning data to struct") return nil, err } @@ -261,12 +262,13 @@ func (r *FilterRepo) Store(filter domain.Filter) (*domain.Filter, error) { release_types_match, formats, quality, + media, log_score, has_log, has_cue, perfect_flac ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38) ON CONFLICT DO NOTHING`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39) ON CONFLICT DO NOTHING`, filter.Name, filter.Enabled, filter.MinSize, @@ -301,6 +303,7 @@ func (r *FilterRepo) Store(filter domain.Filter) (*domain.Filter, error) { pq.Array(filter.MatchReleaseTypes), pq.Array(filter.Formats), pq.Array(filter.Quality), + pq.Array(filter.Media), filter.LogScore, filter.Log, filter.Cue, @@ -361,6 +364,7 @@ func (r *FilterRepo) Update(ctx context.Context, filter domain.Filter) (*domain. release_types_match = ?, formats = ?, quality = ?, + media = ?, log_score = ?, has_log = ?, has_cue = ?, @@ -401,6 +405,7 @@ func (r *FilterRepo) Update(ctx context.Context, filter domain.Filter) (*domain. pq.Array(filter.MatchReleaseTypes), pq.Array(filter.Formats), pq.Array(filter.Quality), + pq.Array(filter.Media), filter.LogScore, filter.Log, filter.Cue, diff --git a/internal/database/migrate.go b/internal/database/migrate.go index 74ff18d..64295f6 100644 --- a/internal/database/migrate.go +++ b/internal/database/migrate.go @@ -1,7 +1,9 @@ package database import ( + "database/sql" "fmt" + "github.com/lib/pq" ) const schema = ` @@ -90,6 +92,7 @@ CREATE TABLE filter release_types_ignore TEXT [] DEFAULT '{}', formats TEXT [] DEFAULT '{}', quality TEXT [] DEFAULT '{}', + media TEXT [] DEFAULT '{}', log_score INTEGER, has_log BOOLEAN, has_cue BOOLEAN, @@ -333,6 +336,10 @@ var migrations = []string{ ALTER TABLE "filter" ADD COLUMN perfect_flac BOOLEAN; `, + ` + ALTER TABLE "filter" + ADD COLUMN media TEXT [] DEFAULT '{}'; + `, } func (db *SqliteDB) migrate() error { @@ -368,6 +375,16 @@ func (db *SqliteDB) migrate() error { } } + // temp custom data migration + // get data from filter.sources, check if specific types, move to new table and clear + // if migration 6 + // TODO 2022-01-30 remove this in future version + if version == 5 && len(migrations) == 6 { + if err := customMigrateCopySourcesToMedia(tx); err != nil { + return fmt.Errorf("could not run custom data migration: %v", err) + } + } + _, err = tx.Exec(fmt.Sprintf("PRAGMA user_version = %d", len(migrations))) if err != nil { return fmt.Errorf("failed to bump schema version: %v", err) @@ -375,3 +392,93 @@ func (db *SqliteDB) migrate() error { return tx.Commit() } + +// customMigrateCopySourcesToMedia move music specific sources to media +func customMigrateCopySourcesToMedia(tx *sql.Tx) error { + rows, err := tx.Query(` + SELECT id, sources + FROM filter + WHERE sources LIKE '%"CD"%' + OR sources LIKE '%"WEB"%' + OR sources LIKE '%"DVD"%' + OR sources LIKE '%"Vinyl"%' + OR sources LIKE '%"Soundboard"%' + OR sources LIKE '%"DAT"%' + OR sources LIKE '%"Cassette"%' + OR sources LIKE '%"Blu-Ray"%' + OR sources LIKE '%"SACD"%' + ;`) + if err != nil { + return fmt.Errorf("could not run custom data migration: %v", err) + } + + defer rows.Close() + + type tmpDataStruct struct { + id int + sources []string + } + + var tmpData []tmpDataStruct + + // scan data + for rows.Next() { + var t tmpDataStruct + + if err := rows.Scan(&t.id, pq.Array(&t.sources)); err != nil { + return err + } + + tmpData = append(tmpData, t) + } + if err := rows.Err(); err != nil { + return err + } + + // manipulate data + for _, d := range tmpData { + // create new slice with only music source if they exist in d.sources + mediaSources := []string{} + for _, source := range d.sources { + switch source { + case "CD": + mediaSources = append(mediaSources, source) + case "DVD": + mediaSources = append(mediaSources, source) + case "Vinyl": + mediaSources = append(mediaSources, source) + case "Soundboard": + mediaSources = append(mediaSources, source) + case "DAT": + mediaSources = append(mediaSources, source) + case "Cassette": + mediaSources = append(mediaSources, source) + case "Blu-Ray": + mediaSources = append(mediaSources, source) + case "SACD": + mediaSources = append(mediaSources, source) + } + } + _, err = tx.Exec(`UPDATE filter SET media = ? WHERE id = ?`, pq.Array(mediaSources), d.id) + if err != nil { + return err + } + + // remove all music specific sources + cleanSources := []string{} + for _, source := range d.sources { + switch source { + case "CD", "WEB", "DVD", "Vinyl", "Soundboard", "DAT", "Cassette", "Blu-Ray", "SACD": + continue + } + cleanSources = append(cleanSources, source) + } + _, err := tx.Exec(`UPDATE filter SET sources = ? WHERE id = ?`, pq.Array(cleanSources), d.id) + if err != nil { + return err + } + + } + + return nil +} diff --git a/internal/domain/filter.go b/internal/domain/filter.go index 9061304..0ee3b23 100644 --- a/internal/domain/filter.go +++ b/internal/domain/filter.go @@ -57,19 +57,19 @@ type Filter struct { ExceptReleaseTypes string `json:"except_release_types"` Formats []string `json:"formats"` // MP3, FLAC, Ogg, AAC, AC3, DTS Quality []string `json:"quality"` // 192, 320, APS (VBR), V2 (VBR), V1 (VBR), APX (VBR), V0 (VBR), q8.x (VBR), Lossless, 24bit Lossless, Other - //Media []string `json:"media"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other - PerfectFlac bool `json:"perfect_flac"` - Cue bool `json:"cue"` - Log bool `json:"log"` - LogScore int `json:"log_score"` - MatchCategories string `json:"match_categories"` - ExceptCategories string `json:"except_categories"` - MatchUploaders string `json:"match_uploaders"` - ExceptUploaders string `json:"except_uploaders"` - Tags string `json:"tags"` - ExceptTags string `json:"except_tags"` - TagsAny string `json:"tags_any"` - ExceptTagsAny string `json:"except_tags_any"` - Actions []Action `json:"actions"` - Indexers []Indexer `json:"indexers"` + Media []string `json:"media"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other + PerfectFlac bool `json:"perfect_flac"` + Cue bool `json:"cue"` + Log bool `json:"log"` + LogScore int `json:"log_score"` + MatchCategories string `json:"match_categories"` + ExceptCategories string `json:"except_categories"` + MatchUploaders string `json:"match_uploaders"` + ExceptUploaders string `json:"except_uploaders"` + Tags string `json:"tags"` + ExceptTags string `json:"except_tags"` + TagsAny string `json:"tags_any"` + ExceptTagsAny string `json:"except_tags_any"` + Actions []Action `json:"actions"` + Indexers []Indexer `json:"indexers"` } diff --git a/internal/domain/release.go b/internal/domain/release.go index 8a1bfb0..5923482 100644 --- a/internal/domain/release.go +++ b/internal/domain/release.go @@ -847,6 +847,11 @@ func (r *Release) CheckFilter(filter Filter) bool { return false } + if len(filter.Media) > 0 && !checkFilterSource(r.Source, filter.Media) { + r.addRejection("source not matching") + return false + } + if filter.Log && r.HasLog != filter.Log { r.addRejection("wanted: log") return false diff --git a/internal/domain/release_test.go b/internal/domain/release_test.go index 472a182..68dde72 100644 --- a/internal/domain/release_test.go +++ b/internal/domain/release_test.go @@ -1069,7 +1069,7 @@ func TestRelease_CheckFilter(t *testing.T) { Enabled: true, MatchCategories: "Album", Artists: "Artist", - Sources: []string{"CD"}, + Media: []string{"CD"}, Formats: []string{"FLAC"}, Quality: []string{"24bit Lossless"}, Log: true, @@ -1137,7 +1137,7 @@ func TestRelease_CheckFilter(t *testing.T) { Enabled: true, MatchCategories: "Album", Artists: "Artist", - Sources: []string{"CD"}, + Media: []string{"CD"}, Formats: []string{"FLAC"}, Quality: []string{"24bit Lossless"}, //PerfectFlac: true, @@ -1162,7 +1162,7 @@ func TestRelease_CheckFilter(t *testing.T) { MatchReleaseTypes: []string{"Album"}, Years: "2020-2022", Artists: "Artist", - Sources: []string{"CD"}, + Media: []string{"CD"}, Formats: []string{"FLAC"}, Quality: []string{"24bit Lossless", "Lossless"}, PerfectFlac: true, @@ -1185,7 +1185,7 @@ func TestRelease_CheckFilter(t *testing.T) { Enabled: true, MatchReleaseTypes: []string{"Single"}, Artists: "Artist", - Sources: []string{"CD"}, + Media: []string{"CD"}, Formats: []string{"FLAC"}, Quality: []string{"24bit Lossless", "Lossless"}, PerfectFlac: true, @@ -1208,7 +1208,7 @@ func TestRelease_CheckFilter(t *testing.T) { Enabled: true, MatchReleaseTypes: []string{"Album"}, Artists: "Artiiiist", - Sources: []string{"CD"}, + Media: []string{"CD"}, Formats: []string{"FLAC"}, Quality: []string{"24bit Lossless", "Lossless"}, PerfectFlac: true, @@ -1232,7 +1232,7 @@ func TestRelease_CheckFilter(t *testing.T) { MatchReleaseTypes: []string{"Album"}, Artists: "Artist", Albums: "Albumname", - Sources: []string{"CD"}, + Media: []string{"CD"}, Formats: []string{"FLAC"}, Quality: []string{"24bit Lossless", "Lossless"}, PerfectFlac: true, diff --git a/web/src/domain/constants.ts b/web/src/domain/constants.ts index 193ae1a..b4180a9 100644 --- a/web/src/domain/constants.ts +++ b/web/src/domain/constants.ts @@ -52,15 +52,6 @@ export const sources = [ "Mixed", "SiteRip", "Webrip", - "CD", - "WEB", - "DVD", - "Vinyl", - "Soundboard", - "DAT", - "Cassette", - "Blu-Ray", - "SACD", ]; export const SOURCES_OPTIONS = sources.map(v => ({ value: v, label: v, key: v})); diff --git a/web/src/screens/filters/details.tsx b/web/src/screens/filters/details.tsx index 9373f3b..463497d 100644 --- a/web/src/screens/filters/details.tsx +++ b/web/src/screens/filters/details.tsx @@ -252,6 +252,7 @@ export default function FilterDetails() { actions: data.actions || [], formats: data.formats || [], quality: data.quality || [], + media: data.media || [], match_release_types: data.match_release_types || [], log_score: data.log_score, log: data.log, @@ -406,7 +407,7 @@ function Music() {
- +
diff --git a/web/src/types/Action.d.ts b/web/src/types/Action.d.ts index 1c92a04..d41afdf 100644 --- a/web/src/types/Action.d.ts +++ b/web/src/types/Action.d.ts @@ -36,6 +36,7 @@ interface Filter { match_release_types: string[]; quality: string[]; formats: string[]; + media: string[]; match_hdr: string[]; except_hdr: string[]; log_score: number;