refactor(filters): music sources to media (#104)

This commit is contained in:
Ludvig Lundgren 2022-01-31 19:16:47 +01:00 committed by GitHub
parent 373c85f060
commit 9b85f512e5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 145 additions and 35 deletions

View file

@ -58,7 +58,7 @@ func (r *FilterRepo) FindByID(filterID int) (*domain.Filter, error) {
//r.db.lock.RLock()
//defer r.db.lock.RUnlock()
row := r.db.handler.QueryRow("SELECT id, enabled, name, min_size, max_size, delay, match_releases, except_releases, use_regex, match_release_groups, except_release_groups, scene, freeleech, freeleech_percent, shows, seasons, episodes, resolutions, codecs, sources, containers, match_hdr, except_hdr, years, artists, albums, release_types_match, formats, quality, log_score, has_log, has_cue, perfect_flac, match_categories, except_categories, match_uploaders, except_uploaders, tags, except_tags, created_at, updated_at FROM filter WHERE id = ?", filterID)
row := r.db.handler.QueryRow("SELECT id, enabled, name, min_size, max_size, delay, match_releases, except_releases, use_regex, match_release_groups, except_release_groups, scene, freeleech, freeleech_percent, shows, seasons, episodes, resolutions, codecs, sources, containers, match_hdr, except_hdr, years, artists, albums, release_types_match, formats, quality, media, log_score, has_log, has_cue, perfect_flac, match_categories, except_categories, match_uploaders, except_uploaders, tags, except_tags, created_at, updated_at FROM filter WHERE id = ?", filterID)
var f domain.Filter
@ -70,7 +70,7 @@ func (r *FilterRepo) FindByID(filterID int) (*domain.Filter, error) {
var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac sql.NullBool
var delay, logScore sql.NullInt32
if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil {
if err := row.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil {
log.Error().Stack().Err(err).Msgf("filter: %v : error scanning data to struct", filterID)
return nil, err
}
@ -142,6 +142,7 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e
f.release_types_match,
f.formats,
f.quality,
f.media,
f.log_score,
f.has_log,
f.has_cue,
@ -174,7 +175,7 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e
var useRegex, scene, freeleech, hasLog, hasCue, perfectFlac sql.NullBool
var delay, logScore sql.NullInt32
if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil {
if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &minSize, &maxSize, &delay, &matchReleases, &exceptReleases, &useRegex, &matchReleaseGroups, &exceptReleaseGroups, &scene, &freeleech, &freeleechPercent, &shows, &seasons, &episodes, pq.Array(&f.Resolutions), pq.Array(&f.Codecs), pq.Array(&f.Sources), pq.Array(&f.Containers), pq.Array(&f.MatchHDR), pq.Array(&f.ExceptHDR), &years, &artists, &albums, pq.Array(&f.MatchReleaseTypes), pq.Array(&f.Formats), pq.Array(&f.Quality), pq.Array(&f.Media), &logScore, &hasLog, &hasCue, &perfectFlac, &matchCategories, &exceptCategories, &matchUploaders, &exceptUploaders, &tags, &exceptTags, &f.CreatedAt, &f.UpdatedAt); err != nil {
log.Error().Stack().Err(err).Msg("error scanning data to struct")
return nil, err
}
@ -261,12 +262,13 @@ func (r *FilterRepo) Store(filter domain.Filter) (*domain.Filter, error) {
release_types_match,
formats,
quality,
media,
log_score,
has_log,
has_cue,
perfect_flac
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38) ON CONFLICT DO NOTHING`,
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39) ON CONFLICT DO NOTHING`,
filter.Name,
filter.Enabled,
filter.MinSize,
@ -301,6 +303,7 @@ func (r *FilterRepo) Store(filter domain.Filter) (*domain.Filter, error) {
pq.Array(filter.MatchReleaseTypes),
pq.Array(filter.Formats),
pq.Array(filter.Quality),
pq.Array(filter.Media),
filter.LogScore,
filter.Log,
filter.Cue,
@ -361,6 +364,7 @@ func (r *FilterRepo) Update(ctx context.Context, filter domain.Filter) (*domain.
release_types_match = ?,
formats = ?,
quality = ?,
media = ?,
log_score = ?,
has_log = ?,
has_cue = ?,
@ -401,6 +405,7 @@ func (r *FilterRepo) Update(ctx context.Context, filter domain.Filter) (*domain.
pq.Array(filter.MatchReleaseTypes),
pq.Array(filter.Formats),
pq.Array(filter.Quality),
pq.Array(filter.Media),
filter.LogScore,
filter.Log,
filter.Cue,

View file

@ -1,7 +1,9 @@
package database
import (
"database/sql"
"fmt"
"github.com/lib/pq"
)
const schema = `
@ -90,6 +92,7 @@ CREATE TABLE filter
release_types_ignore TEXT [] DEFAULT '{}',
formats TEXT [] DEFAULT '{}',
quality TEXT [] DEFAULT '{}',
media TEXT [] DEFAULT '{}',
log_score INTEGER,
has_log BOOLEAN,
has_cue BOOLEAN,
@ -333,6 +336,10 @@ var migrations = []string{
ALTER TABLE "filter"
ADD COLUMN perfect_flac BOOLEAN;
`,
`
ALTER TABLE "filter"
ADD COLUMN media TEXT [] DEFAULT '{}';
`,
}
func (db *SqliteDB) migrate() error {
@ -368,6 +375,16 @@ func (db *SqliteDB) migrate() error {
}
}
// temp custom data migration
// get data from filter.sources, check if specific types, move to new table and clear
// if migration 6
// TODO 2022-01-30 remove this in future version
if version == 5 && len(migrations) == 6 {
if err := customMigrateCopySourcesToMedia(tx); err != nil {
return fmt.Errorf("could not run custom data migration: %v", err)
}
}
_, err = tx.Exec(fmt.Sprintf("PRAGMA user_version = %d", len(migrations)))
if err != nil {
return fmt.Errorf("failed to bump schema version: %v", err)
@ -375,3 +392,93 @@ func (db *SqliteDB) migrate() error {
return tx.Commit()
}
// customMigrateCopySourcesToMedia move music specific sources to media
func customMigrateCopySourcesToMedia(tx *sql.Tx) error {
rows, err := tx.Query(`
SELECT id, sources
FROM filter
WHERE sources LIKE '%"CD"%'
OR sources LIKE '%"WEB"%'
OR sources LIKE '%"DVD"%'
OR sources LIKE '%"Vinyl"%'
OR sources LIKE '%"Soundboard"%'
OR sources LIKE '%"DAT"%'
OR sources LIKE '%"Cassette"%'
OR sources LIKE '%"Blu-Ray"%'
OR sources LIKE '%"SACD"%'
;`)
if err != nil {
return fmt.Errorf("could not run custom data migration: %v", err)
}
defer rows.Close()
type tmpDataStruct struct {
id int
sources []string
}
var tmpData []tmpDataStruct
// scan data
for rows.Next() {
var t tmpDataStruct
if err := rows.Scan(&t.id, pq.Array(&t.sources)); err != nil {
return err
}
tmpData = append(tmpData, t)
}
if err := rows.Err(); err != nil {
return err
}
// manipulate data
for _, d := range tmpData {
// create new slice with only music source if they exist in d.sources
mediaSources := []string{}
for _, source := range d.sources {
switch source {
case "CD":
mediaSources = append(mediaSources, source)
case "DVD":
mediaSources = append(mediaSources, source)
case "Vinyl":
mediaSources = append(mediaSources, source)
case "Soundboard":
mediaSources = append(mediaSources, source)
case "DAT":
mediaSources = append(mediaSources, source)
case "Cassette":
mediaSources = append(mediaSources, source)
case "Blu-Ray":
mediaSources = append(mediaSources, source)
case "SACD":
mediaSources = append(mediaSources, source)
}
}
_, err = tx.Exec(`UPDATE filter SET media = ? WHERE id = ?`, pq.Array(mediaSources), d.id)
if err != nil {
return err
}
// remove all music specific sources
cleanSources := []string{}
for _, source := range d.sources {
switch source {
case "CD", "WEB", "DVD", "Vinyl", "Soundboard", "DAT", "Cassette", "Blu-Ray", "SACD":
continue
}
cleanSources = append(cleanSources, source)
}
_, err := tx.Exec(`UPDATE filter SET sources = ? WHERE id = ?`, pq.Array(cleanSources), d.id)
if err != nil {
return err
}
}
return nil
}

View file

@ -57,19 +57,19 @@ type Filter struct {
ExceptReleaseTypes string `json:"except_release_types"`
Formats []string `json:"formats"` // MP3, FLAC, Ogg, AAC, AC3, DTS
Quality []string `json:"quality"` // 192, 320, APS (VBR), V2 (VBR), V1 (VBR), APX (VBR), V0 (VBR), q8.x (VBR), Lossless, 24bit Lossless, Other
//Media []string `json:"media"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other
PerfectFlac bool `json:"perfect_flac"`
Cue bool `json:"cue"`
Log bool `json:"log"`
LogScore int `json:"log_score"`
MatchCategories string `json:"match_categories"`
ExceptCategories string `json:"except_categories"`
MatchUploaders string `json:"match_uploaders"`
ExceptUploaders string `json:"except_uploaders"`
Tags string `json:"tags"`
ExceptTags string `json:"except_tags"`
TagsAny string `json:"tags_any"`
ExceptTagsAny string `json:"except_tags_any"`
Actions []Action `json:"actions"`
Indexers []Indexer `json:"indexers"`
Media []string `json:"media"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other
PerfectFlac bool `json:"perfect_flac"`
Cue bool `json:"cue"`
Log bool `json:"log"`
LogScore int `json:"log_score"`
MatchCategories string `json:"match_categories"`
ExceptCategories string `json:"except_categories"`
MatchUploaders string `json:"match_uploaders"`
ExceptUploaders string `json:"except_uploaders"`
Tags string `json:"tags"`
ExceptTags string `json:"except_tags"`
TagsAny string `json:"tags_any"`
ExceptTagsAny string `json:"except_tags_any"`
Actions []Action `json:"actions"`
Indexers []Indexer `json:"indexers"`
}

View file

@ -847,6 +847,11 @@ func (r *Release) CheckFilter(filter Filter) bool {
return false
}
if len(filter.Media) > 0 && !checkFilterSource(r.Source, filter.Media) {
r.addRejection("source not matching")
return false
}
if filter.Log && r.HasLog != filter.Log {
r.addRejection("wanted: log")
return false

View file

@ -1069,7 +1069,7 @@ func TestRelease_CheckFilter(t *testing.T) {
Enabled: true,
MatchCategories: "Album",
Artists: "Artist",
Sources: []string{"CD"},
Media: []string{"CD"},
Formats: []string{"FLAC"},
Quality: []string{"24bit Lossless"},
Log: true,
@ -1137,7 +1137,7 @@ func TestRelease_CheckFilter(t *testing.T) {
Enabled: true,
MatchCategories: "Album",
Artists: "Artist",
Sources: []string{"CD"},
Media: []string{"CD"},
Formats: []string{"FLAC"},
Quality: []string{"24bit Lossless"},
//PerfectFlac: true,
@ -1162,7 +1162,7 @@ func TestRelease_CheckFilter(t *testing.T) {
MatchReleaseTypes: []string{"Album"},
Years: "2020-2022",
Artists: "Artist",
Sources: []string{"CD"},
Media: []string{"CD"},
Formats: []string{"FLAC"},
Quality: []string{"24bit Lossless", "Lossless"},
PerfectFlac: true,
@ -1185,7 +1185,7 @@ func TestRelease_CheckFilter(t *testing.T) {
Enabled: true,
MatchReleaseTypes: []string{"Single"},
Artists: "Artist",
Sources: []string{"CD"},
Media: []string{"CD"},
Formats: []string{"FLAC"},
Quality: []string{"24bit Lossless", "Lossless"},
PerfectFlac: true,
@ -1208,7 +1208,7 @@ func TestRelease_CheckFilter(t *testing.T) {
Enabled: true,
MatchReleaseTypes: []string{"Album"},
Artists: "Artiiiist",
Sources: []string{"CD"},
Media: []string{"CD"},
Formats: []string{"FLAC"},
Quality: []string{"24bit Lossless", "Lossless"},
PerfectFlac: true,
@ -1232,7 +1232,7 @@ func TestRelease_CheckFilter(t *testing.T) {
MatchReleaseTypes: []string{"Album"},
Artists: "Artist",
Albums: "Albumname",
Sources: []string{"CD"},
Media: []string{"CD"},
Formats: []string{"FLAC"},
Quality: []string{"24bit Lossless", "Lossless"},
PerfectFlac: true,

View file

@ -52,15 +52,6 @@ export const sources = [
"Mixed",
"SiteRip",
"Webrip",
"CD",
"WEB",
"DVD",
"Vinyl",
"Soundboard",
"DAT",
"Cassette",
"Blu-Ray",
"SACD",
];
export const SOURCES_OPTIONS = sources.map(v => ({ value: v, label: v, key: v}));

View file

@ -252,6 +252,7 @@ export default function FilterDetails() {
actions: data.actions || [],
formats: data.formats || [],
quality: data.quality || [],
media: data.media || [],
match_release_types: data.match_release_types || [],
log_score: data.log_score,
log: data.log,
@ -406,7 +407,7 @@ function Music() {
</div>
<div className="mt-6 grid grid-cols-12 gap-6">
<MultiSelect name="sources" options={SOURCES_MUSIC_OPTIONS} label="sources" columns={6} />
<MultiSelect name="media" options={SOURCES_MUSIC_OPTIONS} label="Media" columns={6} />
<MultiSelect name="match_release_types" options={RELEASE_TYPE_MUSIC_OPTIONS} label="Type" columns={6} />
</div>

View file

@ -36,6 +36,7 @@ interface Filter {
match_release_types: string[];
quality: string[];
formats: string[];
media: string[];
match_hdr: string[];
except_hdr: string[];
log_score: number;