diff --git a/internal/announce/announce.go b/internal/announce/announce.go index a9ad284..c3ba485 100644 --- a/internal/announce/announce.go +++ b/internal/announce/announce.go @@ -218,23 +218,22 @@ func (a *announceProcessor) onLinesMatched(def domain.IndexerDefinition, vars ma var err error err = release.MapVars(vars) - - // FIXME is this even needed anymore? - // canonicalize name - //canonReleaseName := cleanReleaseName(release.TorrentName) - //log.Trace().Msgf("canonicalize release name: %v", canonReleaseName) + if err != nil { + log.Error().Stack().Err(err).Msg("announce: could not map vars for release") + return err + } // parse fields err = release.Parse() if err != nil { - log.Error().Err(err).Msg("announce: could not parse release") + log.Error().Stack().Err(err).Msg("announce: could not parse release") return err } // generate torrent url torrentUrl, err := a.processTorrentUrl(def.Parse.Match.TorrentURL, vars, def.SettingsMap, def.Parse.Match.Encode) if err != nil { - log.Error().Err(err).Msg("announce: could not process torrent url") + log.Error().Stack().Err(err).Msg("announce: could not process torrent url") return err } diff --git a/internal/database/filter.go b/internal/database/filter.go index 65354a4..6545577 100644 --- a/internal/database/filter.go +++ b/internal/database/filter.go @@ -142,6 +142,7 @@ func (r *FilterRepo) FindFiltersForSite(site string) ([]domain.Filter, error) { return filters, nil } +// FindByIndexerIdentifier find active filters only func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, error) { rows, err := r.db.Query(` @@ -179,7 +180,8 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e FROM filter f JOIN filter_indexer fi on f.id = fi.filter_id JOIN indexer i on i.id = fi.indexer_id - WHERE i.identifier = ?`, indexer) + WHERE i.identifier = ? + AND f.enabled = true`, indexer) if err != nil { log.Fatal().Err(err) } diff --git a/internal/domain/release.go b/internal/domain/release.go index c791ec8..0144c3a 100644 --- a/internal/domain/release.go +++ b/internal/domain/release.go @@ -40,6 +40,7 @@ type Release struct { TorrentName string `json:"torrent_name"` // full release name Size uint64 `json:"size"` Raw string `json:"raw"` // Raw release + Clean string `json:"clean"` // cleaned release name Title string `json:"title"` // Parsed title Category string `json:"category"` Season int `json:"season"` @@ -71,6 +72,7 @@ type Release struct { IsScene bool `json:"is_scene"` Origin string `json:"origin"` // P2P, Internal Tags []string `json:"tags"` + ReleaseTags string `json:"-"` Freeleech bool `json:"freeleech"` FreeleechPercent int `json:"freeleech_percent"` Uploader string `json:"uploader"` @@ -118,6 +120,9 @@ func (r *Release) Parse() error { err = r.extractProper() err = r.extractRepack() err = r.extractWebsite() + err = r.extractReleaseTags() + + r.Clean = cleanReleaseName(r.TorrentName) if err != nil { log.Trace().Msgf("could not parse release: %v", r.TorrentName) @@ -177,6 +182,19 @@ func (r *Release) extractSource() error { return nil } +func (r *Release) extractSourceFromTags(tag string) error { + if r.Source != "" { + return nil + } + v, err := findLast(tag, `(?i)\b(((?:PPV\.)?[HP]DTV|(?:HD)?CAM|B[DR]Rip|(?:HD-?)?TS|(?:PPV )?WEB-?DL(?: DVDRip)?|HDRip|DVDRip|DVDRIP|CamRip|WEB|W[EB]BRip|Blu-?Ray|DvDScr|telesync|CD|DVD|Vinyl|DAT|Cassette))\b`) + if err != nil { + return err + } + r.Source = v + + return nil +} + func (r *Release) extractCodec() error { v, err := findLast(r.TorrentName, `(?i)\b(HEVC|[hx]\.?26[45]|xvid|divx|AVC|MPEG-?2|AV1|VC-?1|VP9|WebP)\b`) if err != nil { @@ -197,6 +215,20 @@ func (r *Release) extractContainer() error { return nil } +func (r *Release) extractContainerFromTags(tag string) error { + if r.Container != "" { + return nil + } + + v, err := findLast(tag, `(?i)\b(AVI|MPG|MKV|MP4|VOB|m2ts|ISO|IMG)\b`) + if err != nil { + return err + } + r.Container = v + + return nil +} + func (r *Release) extractHDR() error { v, err := findLast(r.TorrentName, `(?i)(HDR10\+|HDR10|DoVi HDR|DV HDR|HDR|DV|DoVi|Dolby Vision \+ HDR10|Dolby Vision)`) if err != nil { @@ -217,22 +249,37 @@ func (r *Release) extractAudio() error { return nil } -func (r *Release) extractGroup() error { - // try first for wierd anime group names [group] show name, or in brackets at the end - group := "" +func (r *Release) extractAudioFromTags(tag string) error { + if r.Audio != "" { + return nil + } - g, err := findLast(r.TorrentName, `\[(.*?)\]`) + v, err := findLast(tag, `(?i)(MP3|Ogg Vorbis|FLAC[\. ][1-7][\. ][0-2]|FLAC|Opus|DD-EX|DDP[\. ]?[124567][\. ][012] Atmos|DDP[\. ]?[124567][\. ][012]|DDP|DD[1-7][\. ][0-2]|Dual[\- ]Audio|LiNE|PCM|Dolby TrueHD [0-9][\. ][0-4]|TrueHD [0-9][\. ][0-4] Atmos|TrueHD [0-9][\. ][0-4]|DTS X|DTS-HD MA [0-9][\. ][0-4]|DTS-HD MA|DTS-ES|DTS [1-7][\. ][0-2]|DTS|DD|DD[12][\. ]0|Dolby Atmos|TrueHD ATMOS|TrueHD|Atmos|Dolby Digital Plus|Dolby Digital Audio|Dolby Digital|AAC[.-]LC|AAC (?:\.?[1-7]\.[0-2])?|AAC|eac3|AC3(?:\.5\.1)?)`) if err != nil { return err } - group = g + r.Audio = v - if group == "" { - g2, err := findLast(r.TorrentName, `(- ?([^-]+(?:-={[^-]+-?$)?))$`) - if err != nil { - return err - } - group = g2 + return nil +} + +//func (r *Release) extractCueFromTags(tag string) error { +// v, err := findLast(tag, `Cue`) +// if err != nil { +// return err +// } +// r.HasCue = v +// +// return nil +//} + +func (r *Release) extractGroup() error { + // try first for wierd anime group names [group] show name, or in brackets at the end + + //g, err := findLast(r.Clean, `\[(.*?)\]`) + group, err := findLast(r.TorrentName, `\-([a-zA-Z0-9_\.]+)$`) + if err != nil { + return err } r.Group = group @@ -321,6 +368,116 @@ func (r *Release) extractWebsite() error { return nil } +func (r *Release) extractFreeleechFromTags(tag string) error { + if r.Freeleech == true { + return nil + } + + // Start with the basic most common ones + v, err := findLast(tag, `Freeleech!`) + if err != nil { + return err + } + if v != "" { + r.Freeleech = true + return nil + } + + r.Freeleech = false + + return nil +} + +func (r *Release) extractLogScoreFromTags(tag string) error { + if r.LogScore > 0 { + return nil + } + + // Start with the basic most common ones + + rxp, err := regexp.Compile(`([\d\.]+)%`) + if err != nil { + return err + //return errors.Wrapf(err, "invalid regex: %s", value) + } + + matches := rxp.FindStringSubmatch(tag) + if matches != nil { + // first value is the match, second value is the text + if len(matches) >= 1 { + last := matches[len(matches)-1] + score, err := strconv.ParseInt(last, 10, 32) + if err != nil { + return err + } + + r.LogScore = int(score) + return nil + } + } + + return nil +} + +func (r *Release) extractBitrateFromTags(tag string) error { + if r.Bitrate != "" { + return nil + } + + // Start with the basic most common ones + + rxp, err := regexp.Compile(`^(?:vbr|aps|apx|v\d|\d{2,4}|\d+\.\d+|q\d+\.[\dx]+|Other)?(?:\s*kbps|\s*kbits?|\s*k)?(?:\s*\(?(?:vbr|cbr)\)?)?$`) + if err != nil { + return err + //return errors.Wrapf(err, "invalid regex: %s", value) + } + + matches := rxp.FindStringSubmatch(tag) + if matches != nil { + // first value is the match, second value is the text + if len(matches) >= 1 { + last := matches[len(matches)-1] + + r.Bitrate = last + return nil + } + } + + return nil +} + +func (r *Release) extractReleaseTags() error { + if r.ReleaseTags == "" { + return nil + } + + tags := SplitAny(r.ReleaseTags, ",|/ ") + + for _, t := range tags { + var err error + err = r.extractAudioFromTags(t) + err = r.extractContainerFromTags(t) + err = r.extractSourceFromTags(t) + err = r.extractFreeleechFromTags(t) + err = r.extractLogScoreFromTags(t) + err = r.extractBitrateFromTags(t) + + if err != nil { + continue + } + + switch t { + case "Cue": + r.HasCue = true + case "Log": + r.HasLog = true + // check percent + } + } + + return nil +} + func (r *Release) addRejection(reason string) { r.Rejections = append(r.Rejections, reason) } @@ -354,8 +511,8 @@ func (r *Release) CheckFilter(filter Filter) bool { return false } - // check against title when parsed correctly - if filter.Shows != "" && !checkFilterStrings(r.TorrentName, filter.Shows) { + // check against TorrentName and Clean which is a cleaned name without (. _ -) + if filter.Shows != "" && !checkMultipleFilterStrings(filter.Shows, r.TorrentName, r.Clean) { r.addRejection("shows not matching") return false } @@ -372,22 +529,22 @@ func (r *Release) CheckFilter(filter Filter) bool { // matchRelease // TODO allow to match against regex - if filter.MatchReleases != "" && !checkFilterStrings(r.TorrentName, filter.MatchReleases) { + if filter.MatchReleases != "" && !checkMultipleFilterStrings(filter.MatchReleases, r.TorrentName, r.Clean) { r.addRejection("match release not matching") return false } - if filter.ExceptReleases != "" && checkFilterStrings(r.TorrentName, filter.ExceptReleases) { + if filter.ExceptReleases != "" && !checkMultipleFilterStrings(filter.ExceptReleases, r.TorrentName, r.Clean) { r.addRejection("except_releases: unwanted release") return false } - if filter.MatchReleaseGroups != "" && !checkFilterStrings(r.Group, filter.MatchReleaseGroups) { + if filter.MatchReleaseGroups != "" && !checkMultipleFilterGroups(filter.MatchReleaseGroups, r.Group, r.Clean) { r.addRejection("release groups not matching") return false } - if filter.ExceptReleaseGroups != "" && checkFilterStrings(r.Group, filter.ExceptReleaseGroups) { + if filter.ExceptReleaseGroups != "" && checkMultipleFilterGroups(filter.ExceptReleaseGroups, r.Group, r.Clean) { r.addRejection("unwanted release group") return false } @@ -412,7 +569,7 @@ func (r *Release) CheckFilter(filter Filter) bool { return false } - if len(filter.Sources) > 0 && !checkFilterSlice(r.Source, filter.Sources) { + if len(filter.Sources) > 0 && !checkFilterSource(r.Source, filter.Sources) { r.addRejection("source not matching") return false } @@ -441,15 +598,15 @@ func (r *Release) CheckFilter(filter Filter) bool { return false } - //if filter.Tags != "" && !checkFilterStrings(r.Tags, filter.Tags) { - // r.addRejection("tags not matching") - // return false - //} - // - //if filter.ExceptTags != "" && checkFilterStrings(r.Tags, filter.ExceptTags) { - // r.addRejection("unwanted tags") - // return false - //} + if filter.Tags != "" && !checkFilterTags(r.Tags, filter.Tags) { + r.addRejection("tags not matching") + return false + } + + if filter.ExceptTags != "" && checkFilterTags(r.Tags, filter.ExceptTags) { + r.addRejection("unwanted tags") + return false + } return true } @@ -504,21 +661,21 @@ func (r *Release) CheckSizeFilter(minSize string, maxSize string) bool { // MapVars better name func (r *Release) MapVars(varMap map[string]string) error { - if torrentName, err := getFirstStringMapValue(varMap, []string{"torrentName"}); err != nil { + if torrentName, err := getStringMapValue(varMap, "torrentName"); err != nil { return errors.Wrap(err, "failed parsing required field") } else { r.TorrentName = html.UnescapeString(torrentName) } - if category, err := getFirstStringMapValue(varMap, []string{"category"}); err == nil { + if category, err := getStringMapValue(varMap, "category"); err == nil { r.Category = category } - if freeleech, err := getFirstStringMapValue(varMap, []string{"freeleech"}); err == nil { + if freeleech, err := getStringMapValue(varMap, "freeleech"); err == nil { r.Freeleech = strings.EqualFold(freeleech, "freeleech") || strings.EqualFold(freeleech, "yes") } - if freeleechPercent, err := getFirstStringMapValue(varMap, []string{"freeleechPercent"}); err == nil { + if freeleechPercent, err := getStringMapValue(varMap, "freeleechPercent"); err == nil { // remove % and trim spaces freeleechPercent = strings.Replace(freeleechPercent, "%", "", -1) freeleechPercent = strings.Trim(freeleechPercent, " ") @@ -531,11 +688,11 @@ func (r *Release) MapVars(varMap map[string]string) error { r.FreeleechPercent = freeleechPercentInt } - if uploader, err := getFirstStringMapValue(varMap, []string{"uploader"}); err == nil { + if uploader, err := getStringMapValue(varMap, "uploader"); err == nil { r.Uploader = uploader } - if torrentSize, err := getFirstStringMapValue(varMap, []string{"torrentSize"}); err == nil { + if torrentSize, err := getStringMapValue(varMap, "torrentSize"); err == nil { size, err := humanize.ParseBytes(torrentSize) if err != nil { // log could not parse into bytes @@ -544,47 +701,27 @@ func (r *Release) MapVars(varMap map[string]string) error { // TODO implement other size checks in filter } - if scene, err := getFirstStringMapValue(varMap, []string{"scene"}); err == nil { + if scene, err := getStringMapValue(varMap, "scene"); err == nil { r.IsScene = strings.EqualFold(scene, "true") || strings.EqualFold(scene, "yes") } - //if year, err := getFirstStringMapValue(varMap, []string{"year"}); err == nil { - // yearI, err := strconv.Atoi(year) - // if err != nil { - // //log.Debug().Msgf("bad year var: %v", year) - // } - // r.Year = yearI - //} - - // TODO split this into two - if tags, err := getFirstStringMapValue(varMap, []string{"releaseTags", "tags"}); err == nil { - r.Tags = []string{tags} + if yearVal, err := getStringMapValue(varMap, "year"); err == nil { + year, err := strconv.Atoi(yearVal) + if err != nil { + //log.Debug().Msgf("bad year var: %v", year) + } + r.Year = year } - // TODO parse releaseType - //if releaseType, err := getFirstStringMapValue(varMap, []string{"releaseType", "$releaseType"}); err == nil { - // r.Type = releaseType - //} + if tags, err := getStringMapValue(varMap, "tags"); err == nil { + tagArr := strings.Split(strings.ReplaceAll(tags, " ", ""), ",") + r.Tags = tagArr + } - //if cue, err := getFirstStringMapValue(varMap, []string{"cue", "$cue"}); err == nil { - // r.Cue = strings.EqualFold(cue, "true") - //} - - //if logVar, err := getFirstStringMapValue(varMap, []string{"log", "$log"}); err == nil { - // r.Log = logVar - //} - - //if media, err := getFirstStringMapValue(varMap, []string{"media", "$media"}); err == nil { - // r.Media = media - //} - - //if format, err := getFirstStringMapValue(varMap, []string{"format", "$format"}); err == nil { - // r.Format = format - //} - - //if bitRate, err := getFirstStringMapValue(varMap, []string{"bitrate", "$bitrate"}); err == nil { - // r.Bitrate = bitRate - //} + // handle releaseTags. Most of them are redundant but some are useful + if releaseTags, err := getStringMapValue(varMap, "releaseTags"); err == nil { + r.ReleaseTags = releaseTags + } return nil } @@ -639,6 +776,35 @@ func checkFilterStrings(name string, filterList string) bool { return false } +// checkMultipleFilterStrings check against multiple vars of unknown length +func checkMultipleFilterStrings(filterList string, vars ...string) bool { + filterSplit := strings.Split(filterList, ",") + + for _, name := range vars { + name = strings.ToLower(name) + + for _, s := range filterSplit { + s = strings.ToLower(s) + s = strings.Trim(s, " ") + // check if line contains * or ?, if so try wildcard match, otherwise try substring match + a := strings.ContainsAny(s, "?|*") + if a { + match := wildcard.Match(s, name) + if match { + return true + } + } else { + b := strings.Contains(name, s) + if b { + return true + } + } + } + } + + return false +} + // checkFilterIntStrings "1,2,3-20" func checkFilterIntStrings(value int, filterList string) bool { filters := strings.Split(filterList, ",") @@ -685,6 +851,81 @@ func checkFilterIntStrings(value int, filterList string) bool { return false } +func checkMultipleFilterGroups(filterList string, vars ...string) bool { + filterSplit := strings.Split(filterList, ",") + + for _, name := range vars { + name = strings.ToLower(name) + + for _, s := range filterSplit { + s = strings.ToLower(strings.Trim(s, " ")) + // check if line contains * or ?, if so try wildcard match, otherwise try substring match + a := strings.ContainsAny(s, "?|*") + if a { + match := wildcard.Match(s, name) + if match { + return true + } + } else { + split := SplitAny(name, " .-") + for _, c := range split { + if c == s { + return true + } + } + continue + } + } + } + + return false +} + +func checkFilterSource(name string, filterList []string) bool { + // remove dash (-) in blu-ray web-dl and make lowercase + name = strings.ToLower(strings.ReplaceAll(name, "-", "")) + + for _, filter := range filterList { + // remove dash (-) in blu-ray web-dl, trim spaces and make lowercase + filter = strings.ToLower(strings.Trim(strings.ReplaceAll(filter, "-", ""), " ")) + + b := strings.Contains(name, filter) + if b { + return true + } + } + + return false +} + +func checkFilterTags(tags []string, filter string) bool { + filterTags := strings.Split(filter, ",") + + for _, tag := range tags { + tag = strings.ToLower(tag) + + for _, filter := range filterTags { + filter = strings.ToLower(filter) + filter = strings.Trim(filter, " ") + // check if line contains * or ?, if so try wildcard match, otherwise try substring match + a := strings.ContainsAny(filter, "?|*") + if a { + match := wildcard.Match(filter, tag) + if match { + return true + } + } else { + b := strings.Contains(tag, filter) + if b { + return true + } + } + } + } + + return false +} + func checkFreeleechPercent(announcePercent int, filterPercent string) bool { filters := strings.Split(filterPercent, ",") @@ -874,6 +1115,44 @@ func findLastInt(input string, pattern string) (int, error) { return 0, nil } +func SplitAny(s string, seps string) []string { + splitter := func(r rune) bool { + return strings.ContainsRune(seps, r) + } + return strings.FieldsFunc(s, splitter) +} + +//func Splitter(s string, splits string) []string { +// m := make(map[rune]int) +// for _, r := range splits { +// m[r] = 1 +// } +// +// splitter := func(r rune) bool { +// return m[r] == 1 +// } +// +// return strings.FieldsFunc(s, splitter) +//} +// +//func canonicalizeString(s string) []string { +// //a := strings.FieldsFunc(s, split) +// a := Splitter(s, " .") +// +// return a +//} + +func cleanReleaseName(input string) string { + // Make a Regex to say we only want letters and numbers + reg, err := regexp.Compile(`[\x00-\x1F\x2D\x2E\x5F\x7F]`) + if err != nil { + return "" + } + processedString := reg.ReplaceAllString(input, " ") + + return processedString +} + type ReleaseStats struct { TotalCount int64 `json:"total_count"` FilteredCount int64 `json:"filtered_count"` diff --git a/internal/domain/release_test.go b/internal/domain/release_test.go index 9477410..2b51634 100644 --- a/internal/domain/release_test.go +++ b/internal/domain/release_test.go @@ -1,68 +1,224 @@ package domain import ( - "github.com/stretchr/testify/assert" "testing" - "time" + + "github.com/stretchr/testify/assert" ) func TestRelease_Parse(t *testing.T) { tests := []struct { name string fields Release + want Release wantErr bool }{ - {name: "parse_1", fields: Release{ - ID: 0, - Rejections: nil, - Indexer: "", - FilterName: "", - Protocol: "", - Implementation: "", - Timestamp: time.Time{}, - TorrentID: "", - GroupID: "", - TorrentName: "Servant S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-FLUX", - Raw: "", - Title: "", - Category: "", - Season: 0, - Episode: 0, - Year: 0, - Resolution: "", - Source: "", - Codec: "", - Container: "", - HDR: "", - Audio: "", - Group: "", - Region: "", - Edition: "", - Proper: false, - Repack: false, - Website: "", - Language: "", - Unrated: false, - Hybrid: false, - Size: 0, - ThreeD: false, - Artists: nil, - Type: "", - Format: "", - Bitrate: "", - LogScore: 0, - HasLog: false, - HasCue: false, - IsScene: false, - Origin: "", - Tags: nil, - Freeleech: false, - FreeleechPercent: 0, - Uploader: "", - PreTime: "", - TorrentURL: "", - Filter: nil, - }, wantErr: false}, + { + name: "parse_1", + fields: Release{ + TorrentName: "Servant S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-FLUX", + }, + want: Release{ + TorrentName: "Servant S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-FLUX", + Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX", + Season: 1, + Episode: 0, + Resolution: "2160p", + Source: "WEB-DL", + Codec: "HEVC", + HDR: "DV", + Audio: "DDP 5.1 Atmos", + Group: "FLUX", + Website: "ATVP", + }, + wantErr: false, + }, + { + name: "parse_2", + fields: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + }, + want: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX", + Season: 1, + Episode: 0, + Resolution: "2160p", + Source: "WEB-DL", + Codec: "HEVC", + HDR: "DV", + Audio: "DDP.5.1", // need to fix audio parsing + Group: "FLUX", + Website: "ATVP", + }, + wantErr: false, + }, + { + name: "parse_3", + fields: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + ReleaseTags: "MKV / 2160p / WEB-DL", + }, + want: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX", + ReleaseTags: "MKV / 2160p / WEB-DL", + Container: "MKV", + Season: 1, + Episode: 0, + Resolution: "2160p", + Source: "WEB-DL", + Codec: "HEVC", + HDR: "DV", + Audio: "DDP.5.1", // need to fix audio parsing + Group: "FLUX", + Website: "ATVP", + }, + wantErr: false, + }, + { + name: "parse_4", + fields: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + ReleaseTags: "MKV | 2160p | WEB-DL", + }, + want: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX", + ReleaseTags: "MKV | 2160p | WEB-DL", + Container: "MKV", + Season: 1, + Episode: 0, + Resolution: "2160p", + Source: "WEB-DL", + Codec: "HEVC", + HDR: "DV", + Audio: "DDP.5.1", // need to fix audio parsing + Group: "FLUX", + Website: "ATVP", + }, + wantErr: false, + }, + { + name: "parse_5", + fields: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + ReleaseTags: "MP4 | 2160p | WEB-DL", + }, + want: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX", + ReleaseTags: "MP4 | 2160p | WEB-DL", + Container: "MP4", + Season: 1, + Episode: 0, + Resolution: "2160p", + Source: "WEB-DL", + Codec: "HEVC", + HDR: "DV", + Audio: "DDP.5.1", // need to fix audio parsing + Group: "FLUX", + Website: "ATVP", + }, + wantErr: false, + }, + { + name: "parse_6", + fields: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + ReleaseTags: "MP4 | 2160p | WEB-DL | Freeleech!", + }, + want: Release{ + TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX", + Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX", + ReleaseTags: "MP4 | 2160p | WEB-DL | Freeleech!", + Container: "MP4", + Season: 1, + Episode: 0, + Resolution: "2160p", + Source: "WEB-DL", + Codec: "HEVC", + HDR: "DV", + Audio: "DDP.5.1", // need to fix audio parsing + Group: "FLUX", + Website: "ATVP", + Freeleech: true, + }, + wantErr: false, + }, + { + name: "parse_music_1", + fields: Release{ + TorrentName: "Artist - Albumname", + ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD", + }, + want: Release{ + TorrentName: "Artist - Albumname", + Clean: "Artist Albumname", + ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD", + Group: "", + Audio: "FLAC", + Source: "CD", + HasCue: true, + HasLog: true, + LogScore: 100, + }, + wantErr: false, + }, + { + name: "parse_music_2", + fields: Release{ + TorrentName: "Various Artists - Music '21", + Tags: []string{"house, techno, tech.house, electro.house, future.house, bass.house, melodic.house"}, + ReleaseTags: "MP3 / 320 / Cassette", + }, + want: Release{ + TorrentName: "Various Artists - Music '21", + Clean: "Various Artists Music '21", + Tags: []string{"house, techno, tech.house, electro.house, future.house, bass.house, melodic.house"}, + ReleaseTags: "MP3 / 320 / Cassette", + Group: "", + Audio: "MP3", + Source: "Cassette", + Bitrate: "320", + }, + wantErr: false, + }, + { + name: "parse_music_3", + fields: Release{ + TorrentName: "The artist (ザ・フリーダムユニティ) - Long album name", + ReleaseTags: "MP3 / V0 (VBR) / CD", + }, + want: Release{ + TorrentName: "The artist (ザ・フリーダムユニティ) - Long album name", + Clean: "The artist (ザ・フリーダムユニティ) Long album name", + ReleaseTags: "MP3 / V0 (VBR) / CD", + Group: "", + Audio: "MP3", + Source: "CD", + }, + wantErr: false, + }, + { + name: "parse_music_4", + fields: Release{ + TorrentName: "Artist - Albumname", + ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD", + }, + want: Release{ + TorrentName: "Artist - Albumname", + Clean: "Artist Albumname", + ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD", + Group: "", + Audio: "FLAC", + Source: "CD", + HasCue: true, + HasLog: true, + LogScore: 100, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -70,6 +226,8 @@ func TestRelease_Parse(t *testing.T) { if err := r.Parse(); (err != nil) != tt.wantErr { t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) } + + assert.Equal(t, tt.want, r) }) } } @@ -143,6 +301,55 @@ func TestRelease_CheckFilter(t *testing.T) { }, want: true, }, + + { + name: "movie_parse_2", + fields: &Release{ + TorrentName: "That Movie 2020 2160p Blu-Ray DD5.1 x264-GROUP1", + Category: "Movies", + Freeleech: true, + Size: uint64(30000000001), + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "Movies", + Freeleech: true, + MinSize: "10 GB", + MaxSize: "40GB", + Resolutions: []string{"2160p"}, + Sources: []string{"BluRay"}, + Codecs: []string{"x264"}, + Years: "2020", + MatchReleaseGroups: "GROUP1", + }, + }, + want: true, + }, + { + name: "movie_parse_3", + fields: &Release{ + TorrentName: "That Movie 2020 2160p WEBDL DD5.1 x264-GROUP1", + Category: "Movies", + Freeleech: true, + Size: uint64(30000000001), + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "Movies", + Freeleech: true, + MinSize: "10 GB", + MaxSize: "40GB", + Resolutions: []string{"2160p"}, + Sources: []string{"WEB-DL"}, + Codecs: []string{"x264"}, + Years: "2020", + MatchReleaseGroups: "GROUP1", + }, + }, + want: true, + }, { name: "movie_parse_shows", fields: &Release{ @@ -168,6 +375,31 @@ func TestRelease_CheckFilter(t *testing.T) { }, want: true, }, + { + name: "movie_parse_shows_1", + fields: &Release{ + TorrentName: "That.Movie.2020.2160p.BluRay.DD5.1.x264-GROUP1", + Category: "Movies", + Freeleech: true, + Size: uint64(30000000001), + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "Movies", + Freeleech: true, + MinSize: "10 GB", + MaxSize: "40GB", + Resolutions: []string{"2160p"}, + Sources: []string{"BluRay"}, + Codecs: []string{"x264"}, + Years: "2020", + MatchReleaseGroups: "GROUP1", + Shows: "That Movie", + }, + }, + want: true, + }, { name: "movie_parse_multiple_shows", fields: &Release{ @@ -193,6 +425,31 @@ func TestRelease_CheckFilter(t *testing.T) { }, want: true, }, + { + name: "movie_parse_multiple_shows_1", + fields: &Release{ + TorrentName: "That.Movie.2020.2160p.BluRay.DD5.1.x264-GROUP1", + Category: "Movies", + Freeleech: true, + Size: uint64(30000000001), + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "Movies", + Freeleech: true, + MinSize: "10 GB", + MaxSize: "40GB", + Resolutions: []string{"2160p"}, + Sources: []string{"BluRay"}, + Codecs: []string{"x264"}, + Years: "2020", + MatchReleaseGroups: "GROUP1", + Shows: "That Movie, good story, bad movie", + }, + }, + want: true, + }, { name: "movie_parse_wildcard_shows", fields: &Release{ @@ -346,6 +603,182 @@ func TestRelease_CheckFilter(t *testing.T) { MatchCategories: "*tv*", MatchUploaders: "Uploader1,Uploader2", ExceptUploaders: "Anonymous", + Shows: "Good show", + }, + }, + want: true, + }, + { + name: "match_tags", + fields: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "TV", + Uploader: "Uploader1", + Tags: []string{"tv"}, + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show", + Tags: "tv", + }, + }, + want: true, + }, + { + name: "match_tags_bad", + fields: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "TV", + Uploader: "Uploader1", + Tags: []string{"foreign"}, + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show", + Tags: "tv", + }, + }, + want: false, + }, + { + name: "match_except_tags", + fields: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "TV", + Uploader: "Uploader1", + Tags: []string{"foreign"}, + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show", + ExceptTags: "tv", + }, + }, + want: true, + }, + { + name: "match_except_tags_2", + fields: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "TV", + Uploader: "Uploader1", + Tags: []string{"foreign"}, + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show", + ExceptTags: "foreign", + }, + }, + want: false, + }, + { + name: "match_group_1", + fields: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show", + MatchReleaseGroups: "GROUP", + }, + }, + want: true, + }, + { + name: "match_group_potential_partial_1", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-ift", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "ift", + }, + }, + want: true, + }, + { + name: "match_group_potential_partial_2", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "ift", + }, + }, + want: false, + }, + { + name: "match_group_potential_partial_3", + fields: &Release{ + TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-de[42]", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "de[42]", + }, + }, + want: true, + }, + { + name: "match_group_potential_partial_3", + fields: &Release{ + TorrentName: "[AnimeGroup] Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC", + Category: "TV", + Uploader: "Uploader1", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchCategories: "*tv*", + MatchUploaders: "Uploader1,Uploader2", + ExceptUploaders: "Anonymous", + Shows: "Good show shift", + MatchReleaseGroups: "[AnimeGroup]", }, }, want: true, @@ -362,3 +795,194 @@ func TestRelease_CheckFilter(t *testing.T) { }) } } + +func TestRelease_MapVars(t *testing.T) { + type args struct { + varMap map[string]string + } + tests := []struct { + name string + fields *Release + want *Release + args args + }{ + { + name: "1", + fields: &Release{}, + want: &Release{TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2"}, + args: args{varMap: map[string]string{ + "torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + }}, + }, + { + name: "2", + fields: &Release{}, + want: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "tv", + Freeleech: true, + Uploader: "Anon", + Size: uint64(10000000000), + }, + args: args{varMap: map[string]string{ + "torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + "category": "tv", + "freeleech": "freeleech", + "uploader": "Anon", + "torrentSize": "10GB", + }}, + }, + { + name: "3", + fields: &Release{}, + want: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "tv", + FreeleechPercent: 100, + Uploader: "Anon", + Size: uint64(10000000000), + }, + args: args{varMap: map[string]string{ + "torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + "category": "tv", + "freeleechPercent": "100%", + "uploader": "Anon", + "torrentSize": "10GB", + }}, + }, + { + name: "4", + fields: &Release{}, + want: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "tv", + FreeleechPercent: 100, + Uploader: "Anon", + Size: uint64(10000000000), + Tags: []string{"foreign", "tv"}, + }, + args: args{varMap: map[string]string{ + "torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + "category": "tv", + "freeleechPercent": "100%", + "uploader": "Anon", + "torrentSize": "10GB", + "tags": "foreign,tv", + }}, + }, + { + name: "5", + fields: &Release{}, + want: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "tv", + FreeleechPercent: 100, + Uploader: "Anon", + Size: uint64(10000000000), + Tags: []string{"foreign", "tv"}, + }, + args: args{varMap: map[string]string{ + "torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + "category": "tv", + "freeleechPercent": "100%", + "uploader": "Anon", + "torrentSize": "10GB", + "tags": "foreign,tv", + }}, + }, + { + name: "6", + fields: &Release{}, + want: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "tv", + Year: 2020, + FreeleechPercent: 100, + Uploader: "Anon", + Size: uint64(10000000000), + Tags: []string{"foreign", "tv"}, + }, + args: args{varMap: map[string]string{ + "torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + "category": "tv", + "year": "2020", + "freeleechPercent": "100%", + "uploader": "Anon", + "torrentSize": "10GB", + "tags": "foreign, tv", + }}, + }, + { + name: "7", + fields: &Release{}, + want: &Release{ + TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + Category: "tv", + Year: 2020, + FreeleechPercent: 100, + Uploader: "Anon", + Size: uint64(10000000000), + Tags: []string{"hip.hop", "rhythm.and.blues", "2000s"}, + }, + args: args{varMap: map[string]string{ + "torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2", + "category": "tv", + "year": "2020", + "freeleechPercent": "100%", + "uploader": "Anon", + "torrentSize": "10GB", + "tags": "hip.hop,rhythm.and.blues, 2000s", + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := tt.fields + _ = r.MapVars(tt.args.varMap) + + assert.Equal(t, tt.want, r) + }) + } +} + +func TestSplitAny(t *testing.T) { + type args struct { + s string + seps string + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "test_1", + args: args{ + s: "Tag1 / Tag2 / Tag3", + seps: "/ ", + }, + want: []string{"Tag1", "Tag2", "Tag3"}, + }, + { + name: "test_2", + args: args{ + s: "Tag1 | Tag2 | Tag3", + seps: "| ", + }, + want: []string{"Tag1", "Tag2", "Tag3"}, + }, + { + name: "test_3", + args: args{ + s: "Tag1 | Tag2 / Tag3", + seps: "| /", + }, + want: []string{"Tag1", "Tag2", "Tag3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, SplitAny(tt.args.s, tt.args.seps), "SplitAny(%v, %v)", tt.args.s, tt.args.seps) + }) + } +} diff --git a/internal/filter/service.go b/internal/filter/service.go index 127e47d..1cc3d96 100644 --- a/internal/filter/service.go +++ b/internal/filter/service.go @@ -182,6 +182,8 @@ func (s *service) FindAndCheckFilters(release *domain.Release) (bool, *domain.Fi return false, nil, err } + log.Trace().Msgf("filter-service.find_and_check_filters: found (%d) active filters to check for indexer '%v'", len(filters), release.Indexer) + // loop and check release to filter until match for _, f := range filters { log.Trace().Msgf("checking filter: %+v", f.Name) diff --git a/internal/indexer/definitions/milkie.yaml b/internal/indexer/definitions/milkie.yaml index f36ba42..9a3a628 100644 --- a/internal/indexer/definitions/milkie.yaml +++ b/internal/indexer/definitions/milkie.yaml @@ -49,7 +49,7 @@ parse: vars: - category - torrentName - - tags + - releaseTags - baseUrl - torrentId