idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-22 16:29:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Feature: Improve filtering and release parsing (#56)

Browse source 
* feat: match against orig and cleaned rel

* feat: add more release parse tests

* feat: filter check tags

* feat: improve filter tag parsing

* refactor: simplify tag split and trim

* fix(indexers): use releasetags for milkie

* fix: properly replace spaces in string

* feat: better source check

* feat: extract releasetags
This commit is contained in:
Ludvig Lundgren 2022-01-01 21:50:38 +01:00 • committed by GitHub
parent 8f53becbb3
commit ae1f14d0a4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 1036 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

13
internal/announce/announce.go
View file

@ -218,23 +218,22 @@ func (a *announceProcessor) onLinesMatched(def domain.IndexerDefinition, vars ma
var err error
err = release.MapVars(vars)
// FIXME is this even needed anymore?
// canonicalize name
//canonReleaseName := cleanReleaseName(release.TorrentName)
//log.Trace().Msgf("canonicalize release name: %v", canonReleaseName)
if err != nil {
log.Error().Stack().Err(err).Msg("announce: could not map vars for release")
return err
}
// parse fields
err = release.Parse()
if err != nil {
log.Error().Err(err).Msg("announce: could not parse release")
log.Error().Stack().Err(err).Msg("announce: could not parse release")
return err
}
// generate torrent url
torrentUrl, err := a.processTorrentUrl(def.Parse.Match.TorrentURL, vars, def.SettingsMap, def.Parse.Match.Encode)
if err != nil {
log.Error().Err(err).Msg("announce: could not process torrent url")
log.Error().Stack().Err(err).Msg("announce: could not process torrent url")
return err
}

4
internal/database/filter.go
View file

@ -142,6 +142,7 @@ func (r *FilterRepo) FindFiltersForSite(site string) ([]domain.Filter, error) {
return filters, nil
}
// FindByIndexerIdentifier find active filters only
func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, error) {
rows, err := r.db.Query(`
@ -179,7 +180,8 @@ func (r *FilterRepo) FindByIndexerIdentifier(indexer string) ([]domain.Filter, e
FROM filter f
JOIN filter_indexer fi on f.id = fi.filter_id
JOIN indexer i on i.id = fi.indexer_id
WHERE i.identifier = ?`, indexer)
WHERE i.identifier = ?
AND f.enabled = true`, indexer)
if err != nil {
log.Fatal().Err(err)
}

415
internal/domain/release.go
View file

@ -40,6 +40,7 @@ type Release struct {
TorrentName string `json:"torrent_name"` // full release name
Size uint64 `json:"size"`
Raw string `json:"raw"` // Raw release
Clean string `json:"clean"` // cleaned release name
Title string `json:"title"` // Parsed title
Category string `json:"category"`
Season int `json:"season"`
@ -71,6 +72,7 @@ type Release struct {
IsScene bool `json:"is_scene"`
Origin string `json:"origin"` // P2P, Internal
Tags []string `json:"tags"`
ReleaseTags string `json:"-"`
Freeleech bool `json:"freeleech"`
FreeleechPercent int `json:"freeleech_percent"`
Uploader string `json:"uploader"`
@ -118,6 +120,9 @@ func (r *Release) Parse() error {
err = r.extractProper()
err = r.extractRepack()
err = r.extractWebsite()
err = r.extractReleaseTags()
r.Clean = cleanReleaseName(r.TorrentName)
if err != nil {
log.Trace().Msgf("could not parse release: %v", r.TorrentName)
@ -177,6 +182,19 @@ func (r *Release) extractSource() error {
return nil
}
func (r *Release) extractSourceFromTags(tag string) error {
if r.Source != "" {
return nil
}
v, err := findLast(tag, `(?i)\b(((?:PPV\.)?[HP]DTV|(?:HD)?CAM|B[DR]Rip|(?:HD-?)?TS|(?:PPV )?WEB-?DL(?: DVDRip)?|HDRip|DVDRip|DVDRIP|CamRip|WEB|W[EB]BRip|Blu-?Ray|DvDScr|telesync|CD|DVD|Vinyl|DAT|Cassette))\b`)
if err != nil {
return err
}
r.Source = v
return nil
}
func (r *Release) extractCodec() error {
v, err := findLast(r.TorrentName, `(?i)\b(HEVC|[hx]\.?26[45]|xvid|divx|AVC|MPEG-?2|AV1|VC-?1|VP9|WebP)\b`)
if err != nil {
@ -197,6 +215,20 @@ func (r *Release) extractContainer() error {
return nil
}
func (r *Release) extractContainerFromTags(tag string) error {
if r.Container != "" {
return nil
}
v, err := findLast(tag, `(?i)\b(AVI|MPG|MKV|MP4|VOB|m2ts|ISO|IMG)\b`)
if err != nil {
return err
}
r.Container = v
return nil
}
func (r *Release) extractHDR() error {
v, err := findLast(r.TorrentName, `(?i)(HDR10\+|HDR10|DoVi HDR|DV HDR|HDR|DV|DoVi|Dolby Vision \+ HDR10|Dolby Vision)`)
if err != nil {
@ -217,22 +249,37 @@ func (r *Release) extractAudio() error {
return nil
}
func (r *Release) extractGroup() error {
// try first for wierd anime group names [group] show name, or in brackets at the end
group := ""
func (r *Release) extractAudioFromTags(tag string) error {
if r.Audio != "" {
return nil
}
g, err := findLast(r.TorrentName, `\[(.*?)\]`)
v, err := findLast(tag, `(?i)(MP3|Ogg Vorbis|FLAC[\. ][1-7][\. ][0-2]|FLAC|Opus|DD-EX|DDP[\. ]?[124567][\. ][012] Atmos|DDP[\. ]?[124567][\. ][012]|DDP|DD[1-7][\. ][0-2]|Dual[\- ]Audio|LiNE|PCM|Dolby TrueHD [0-9][\. ][0-4]|TrueHD [0-9][\. ][0-4] Atmos|TrueHD [0-9][\. ][0-4]|DTS X|DTS-HD MA [0-9][\. ][0-4]|DTS-HD MA|DTS-ES|DTS [1-7][\. ][0-2]|DTS|DD|DD[12][\. ]0|Dolby Atmos|TrueHD ATMOS|TrueHD|Atmos|Dolby Digital Plus|Dolby Digital Audio|Dolby Digital|AAC[.-]LC|AAC (?:\.?[1-7]\.[0-2])?|AAC|eac3|AC3(?:\.5\.1)?)`)
if err != nil {
return err
}
group = g
r.Audio = v
if group == "" {
g2, err := findLast(r.TorrentName, `(- ?([^-]+(?:-={[^-]+-?$)?))$`)
if err != nil {
return err
}
group = g2
return nil
}
//func (r *Release) extractCueFromTags(tag string) error {
// v, err := findLast(tag, `Cue`)
// if err != nil {
// return err
// }
// r.HasCue = v
//
// return nil
//}
func (r *Release) extractGroup() error {
// try first for wierd anime group names [group] show name, or in brackets at the end
//g, err := findLast(r.Clean, `\[(.*?)\]`)
group, err := findLast(r.TorrentName, `\-([a-zA-Z0-9_\.]+)$`)
if err != nil {
return err
}
r.Group = group
@ -321,6 +368,116 @@ func (r *Release) extractWebsite() error {
return nil
}
func (r *Release) extractFreeleechFromTags(tag string) error {
if r.Freeleech == true {
return nil
}
// Start with the basic most common ones
v, err := findLast(tag, `Freeleech!`)
if err != nil {
return err
}
if v != "" {
r.Freeleech = true
return nil
}
r.Freeleech = false
return nil
}
func (r *Release) extractLogScoreFromTags(tag string) error {
if r.LogScore > 0 {
return nil
}
// Start with the basic most common ones
rxp, err := regexp.Compile(`([\d\.]+)%`)
if err != nil {
return err
//return errors.Wrapf(err, "invalid regex: %s", value)
}
matches := rxp.FindStringSubmatch(tag)
if matches != nil {
// first value is the match, second value is the text
if len(matches) >= 1 {
last := matches[len(matches)-1]
score, err := strconv.ParseInt(last, 10, 32)
if err != nil {
return err
}
r.LogScore = int(score)
return nil
}
}
return nil
}
func (r *Release) extractBitrateFromTags(tag string) error {
if r.Bitrate != "" {
return nil
}
// Start with the basic most common ones
rxp, err := regexp.Compile(`^(?:vbr|aps|apx|v\d|\d{2,4}|\d+\.\d+|q\d+\.[\dx]+|Other)?(?:\s*kbps|\s*kbits?|\s*k)?(?:\s*\(?(?:vbr|cbr)\)?)?$`)
if err != nil {
return err
//return errors.Wrapf(err, "invalid regex: %s", value)
}
matches := rxp.FindStringSubmatch(tag)
if matches != nil {
// first value is the match, second value is the text
if len(matches) >= 1 {
last := matches[len(matches)-1]
r.Bitrate = last
return nil
}
}
return nil
}
func (r *Release) extractReleaseTags() error {
if r.ReleaseTags == "" {
return nil
}
tags := SplitAny(r.ReleaseTags, ",|/ ")
for _, t := range tags {
var err error
err = r.extractAudioFromTags(t)
err = r.extractContainerFromTags(t)
err = r.extractSourceFromTags(t)
err = r.extractFreeleechFromTags(t)
err = r.extractLogScoreFromTags(t)
err = r.extractBitrateFromTags(t)
if err != nil {
continue
}
switch t {
case "Cue":
r.HasCue = true
case "Log":
r.HasLog = true
// check percent
}
}
return nil
}
func (r *Release) addRejection(reason string) {
r.Rejections = append(r.Rejections, reason)
}
@ -354,8 +511,8 @@ func (r *Release) CheckFilter(filter Filter) bool {
return false
}
// check against title when parsed correctly
if filter.Shows != "" && !checkFilterStrings(r.TorrentName, filter.Shows) {
// check against TorrentName and Clean which is a cleaned name without (. _ -)
if filter.Shows != "" && !checkMultipleFilterStrings(filter.Shows, r.TorrentName, r.Clean) {
r.addRejection("shows not matching")
return false
}
@ -372,22 +529,22 @@ func (r *Release) CheckFilter(filter Filter) bool {
// matchRelease
// TODO allow to match against regex
if filter.MatchReleases != "" && !checkFilterStrings(r.TorrentName, filter.MatchReleases) {
if filter.MatchReleases != "" && !checkMultipleFilterStrings(filter.MatchReleases, r.TorrentName, r.Clean) {
r.addRejection("match release not matching")
return false
}
if filter.ExceptReleases != "" && checkFilterStrings(r.TorrentName, filter.ExceptReleases) {
if filter.ExceptReleases != "" && !checkMultipleFilterStrings(filter.ExceptReleases, r.TorrentName, r.Clean) {
r.addRejection("except_releases: unwanted release")
return false
}
if filter.MatchReleaseGroups != "" && !checkFilterStrings(r.Group, filter.MatchReleaseGroups) {
if filter.MatchReleaseGroups != "" && !checkMultipleFilterGroups(filter.MatchReleaseGroups, r.Group, r.Clean) {
r.addRejection("release groups not matching")
return false
}
if filter.ExceptReleaseGroups != "" && checkFilterStrings(r.Group, filter.ExceptReleaseGroups) {
if filter.ExceptReleaseGroups != "" && checkMultipleFilterGroups(filter.ExceptReleaseGroups, r.Group, r.Clean) {
r.addRejection("unwanted release group")
return false
}
@ -412,7 +569,7 @@ func (r *Release) CheckFilter(filter Filter) bool {
return false
}
if len(filter.Sources) > 0 && !checkFilterSlice(r.Source, filter.Sources) {
if len(filter.Sources) > 0 && !checkFilterSource(r.Source, filter.Sources) {
r.addRejection("source not matching")
return false
}
@ -441,15 +598,15 @@ func (r *Release) CheckFilter(filter Filter) bool {
return false
}
//if filter.Tags != "" && !checkFilterStrings(r.Tags, filter.Tags) {
// r.addRejection("tags not matching")
// return false
//}
//
//if filter.ExceptTags != "" && checkFilterStrings(r.Tags, filter.ExceptTags) {
// r.addRejection("unwanted tags")
// return false
//}
if filter.Tags != "" && !checkFilterTags(r.Tags, filter.Tags) {
r.addRejection("tags not matching")
return false
}
if filter.ExceptTags != "" && checkFilterTags(r.Tags, filter.ExceptTags) {
r.addRejection("unwanted tags")
return false
}
return true
}
@ -504,21 +661,21 @@ func (r *Release) CheckSizeFilter(minSize string, maxSize string) bool {
// MapVars better name
func (r *Release) MapVars(varMap map[string]string) error {
if torrentName, err := getFirstStringMapValue(varMap, []string{"torrentName"}); err != nil {
if torrentName, err := getStringMapValue(varMap, "torrentName"); err != nil {
return errors.Wrap(err, "failed parsing required field")
} else {
r.TorrentName = html.UnescapeString(torrentName)
}
if category, err := getFirstStringMapValue(varMap, []string{"category"}); err == nil {
if category, err := getStringMapValue(varMap, "category"); err == nil {
r.Category = category
}
if freeleech, err := getFirstStringMapValue(varMap, []string{"freeleech"}); err == nil {
if freeleech, err := getStringMapValue(varMap, "freeleech"); err == nil {
r.Freeleech = strings.EqualFold(freeleech, "freeleech") || strings.EqualFold(freeleech, "yes")
}
if freeleechPercent, err := getFirstStringMapValue(varMap, []string{"freeleechPercent"}); err == nil {
if freeleechPercent, err := getStringMapValue(varMap, "freeleechPercent"); err == nil {
// remove % and trim spaces
freeleechPercent = strings.Replace(freeleechPercent, "%", "", -1)
freeleechPercent = strings.Trim(freeleechPercent, " ")
@ -531,11 +688,11 @@ func (r *Release) MapVars(varMap map[string]string) error {
r.FreeleechPercent = freeleechPercentInt
}
if uploader, err := getFirstStringMapValue(varMap, []string{"uploader"}); err == nil {
if uploader, err := getStringMapValue(varMap, "uploader"); err == nil {
r.Uploader = uploader
}
if torrentSize, err := getFirstStringMapValue(varMap, []string{"torrentSize"}); err == nil {
if torrentSize, err := getStringMapValue(varMap, "torrentSize"); err == nil {
size, err := humanize.ParseBytes(torrentSize)
if err != nil {
// log could not parse into bytes
@ -544,47 +701,27 @@ func (r *Release) MapVars(varMap map[string]string) error {
// TODO implement other size checks in filter
}
if scene, err := getFirstStringMapValue(varMap, []string{"scene"}); err == nil {
if scene, err := getStringMapValue(varMap, "scene"); err == nil {
r.IsScene = strings.EqualFold(scene, "true") || strings.EqualFold(scene, "yes")
}
//if year, err := getFirstStringMapValue(varMap, []string{"year"}); err == nil {
// yearI, err := strconv.Atoi(year)
// if err != nil {
// //log.Debug().Msgf("bad year var: %v", year)
// }
// r.Year = yearI
//}
// TODO split this into two
if tags, err := getFirstStringMapValue(varMap, []string{"releaseTags", "tags"}); err == nil {
r.Tags = []string{tags}
if yearVal, err := getStringMapValue(varMap, "year"); err == nil {
year, err := strconv.Atoi(yearVal)
if err != nil {
//log.Debug().Msgf("bad year var: %v", year)
}
r.Year = year
}
// TODO parse releaseType
//if releaseType, err := getFirstStringMapValue(varMap, []string{"releaseType", "$releaseType"}); err == nil {
// r.Type = releaseType
//}
if tags, err := getStringMapValue(varMap, "tags"); err == nil {
tagArr := strings.Split(strings.ReplaceAll(tags, " ", ""), ",")
r.Tags = tagArr
}
//if cue, err := getFirstStringMapValue(varMap, []string{"cue", "$cue"}); err == nil {
// r.Cue = strings.EqualFold(cue, "true")
//}
//if logVar, err := getFirstStringMapValue(varMap, []string{"log", "$log"}); err == nil {
// r.Log = logVar
//}
//if media, err := getFirstStringMapValue(varMap, []string{"media", "$media"}); err == nil {
// r.Media = media
//}
//if format, err := getFirstStringMapValue(varMap, []string{"format", "$format"}); err == nil {
// r.Format = format
//}
//if bitRate, err := getFirstStringMapValue(varMap, []string{"bitrate", "$bitrate"}); err == nil {
// r.Bitrate = bitRate
//}
// handle releaseTags. Most of them are redundant but some are useful
if releaseTags, err := getStringMapValue(varMap, "releaseTags"); err == nil {
r.ReleaseTags = releaseTags
}
return nil
}
@ -639,6 +776,35 @@ func checkFilterStrings(name string, filterList string) bool {
return false
}
// checkMultipleFilterStrings check against multiple vars of unknown length
func checkMultipleFilterStrings(filterList string, vars ...string) bool {
filterSplit := strings.Split(filterList, ",")
for _, name := range vars {
name = strings.ToLower(name)
for _, s := range filterSplit {
s = strings.ToLower(s)
s = strings.Trim(s, " ")
// check if line contains * or ?, if so try wildcard match, otherwise try substring match
a := strings.ContainsAny(s, "?|*")
if a {
match := wildcard.Match(s, name)
if match {
return true
}
} else {
b := strings.Contains(name, s)
if b {
return true
}
}
}
}
return false
}
// checkFilterIntStrings "1,2,3-20"
func checkFilterIntStrings(value int, filterList string) bool {
filters := strings.Split(filterList, ",")
@ -685,6 +851,81 @@ func checkFilterIntStrings(value int, filterList string) bool {
return false
}
func checkMultipleFilterGroups(filterList string, vars ...string) bool {
filterSplit := strings.Split(filterList, ",")
for _, name := range vars {
name = strings.ToLower(name)
for _, s := range filterSplit {
s = strings.ToLower(strings.Trim(s, " "))
// check if line contains * or ?, if so try wildcard match, otherwise try substring match
a := strings.ContainsAny(s, "?|*")
if a {
match := wildcard.Match(s, name)
if match {
return true
}
} else {
split := SplitAny(name, " .-")
for _, c := range split {
if c == s {
return true
}
}
continue
}
}
}
return false
}
func checkFilterSource(name string, filterList []string) bool {
// remove dash (-) in blu-ray web-dl and make lowercase
name = strings.ToLower(strings.ReplaceAll(name, "-", ""))
for _, filter := range filterList {
// remove dash (-) in blu-ray web-dl, trim spaces and make lowercase
filter = strings.ToLower(strings.Trim(strings.ReplaceAll(filter, "-", ""), " "))
b := strings.Contains(name, filter)
if b {
return true
}
}
return false
}
func checkFilterTags(tags []string, filter string) bool {
filterTags := strings.Split(filter, ",")
for _, tag := range tags {
tag = strings.ToLower(tag)
for _, filter := range filterTags {
filter = strings.ToLower(filter)
filter = strings.Trim(filter, " ")
// check if line contains * or ?, if so try wildcard match, otherwise try substring match
a := strings.ContainsAny(filter, "?|*")
if a {
match := wildcard.Match(filter, tag)
if match {
return true
}
} else {
b := strings.Contains(tag, filter)
if b {
return true
}
}
}
}
return false
}
func checkFreeleechPercent(announcePercent int, filterPercent string) bool {
filters := strings.Split(filterPercent, ",")
@ -874,6 +1115,44 @@ func findLastInt(input string, pattern string) (int, error) {
return 0, nil
}
func SplitAny(s string, seps string) []string {
splitter := func(r rune) bool {
return strings.ContainsRune(seps, r)
}
return strings.FieldsFunc(s, splitter)
}
//func Splitter(s string, splits string) []string {
// m := make(map[rune]int)
// for _, r := range splits {
// m[r] = 1
// }
//
// splitter := func(r rune) bool {
// return m[r] == 1
// }
//
// return strings.FieldsFunc(s, splitter)
//}
//
//func canonicalizeString(s string) []string {
// //a := strings.FieldsFunc(s, split)
// a := Splitter(s, " .")
//
// return a
//}
func cleanReleaseName(input string) string {
// Make a Regex to say we only want letters and numbers
reg, err := regexp.Compile(`[\x00-\x1F\x2D\x2E\x5F\x7F]`)
if err != nil {
return ""
}
processedString := reg.ReplaceAllString(input, " ")
return processedString
}
type ReleaseStats struct {
TotalCount int64 `json:"total_count"`
FilteredCount int64 `json:"filtered_count"`

730
internal/domain/release_test.go
View file

@ -1,68 +1,224 @@
package domain
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestRelease_Parse(t *testing.T) {
tests := []struct {
name string
fields Release
want Release
wantErr bool
}{
{name: "parse_1", fields: Release{
ID: 0,
Rejections: nil,
Indexer: "",
FilterName: "",
Protocol: "",
Implementation: "",
Timestamp: time.Time{},
TorrentID: "",
GroupID: "",
TorrentName: "Servant S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-FLUX",
Raw: "",
Title: "",
Category: "",
Season: 0,
Episode: 0,
Year: 0,
Resolution: "",
Source: "",
Codec: "",
Container: "",
HDR: "",
Audio: "",
Group: "",
Region: "",
Edition: "",
Proper: false,
Repack: false,
Website: "",
Language: "",
Unrated: false,
Hybrid: false,
Size: 0,
ThreeD: false,
Artists: nil,
Type: "",
Format: "",
Bitrate: "",
LogScore: 0,
HasLog: false,
HasCue: false,
IsScene: false,
Origin: "",
Tags: nil,
Freeleech: false,
FreeleechPercent: 0,
Uploader: "",
PreTime: "",
TorrentURL: "",
Filter: nil,
}, wantErr: false},
{
name: "parse_1",
fields: Release{
TorrentName: "Servant S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-FLUX",
},
want: Release{
TorrentName: "Servant S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-FLUX",
Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX",
Season: 1,
Episode: 0,
Resolution: "2160p",
Source: "WEB-DL",
Codec: "HEVC",
HDR: "DV",
Audio: "DDP 5.1 Atmos",
Group: "FLUX",
Website: "ATVP",
},
wantErr: false,
},
{
name: "parse_2",
fields: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
},
want: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX",
Season: 1,
Episode: 0,
Resolution: "2160p",
Source: "WEB-DL",
Codec: "HEVC",
HDR: "DV",
Audio: "DDP.5.1", // need to fix audio parsing
Group: "FLUX",
Website: "ATVP",
},
wantErr: false,
},
{
name: "parse_3",
fields: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
ReleaseTags: "MKV / 2160p / WEB-DL",
},
want: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX",
ReleaseTags: "MKV / 2160p / WEB-DL",
Container: "MKV",
Season: 1,
Episode: 0,
Resolution: "2160p",
Source: "WEB-DL",
Codec: "HEVC",
HDR: "DV",
Audio: "DDP.5.1", // need to fix audio parsing
Group: "FLUX",
Website: "ATVP",
},
wantErr: false,
},
{
name: "parse_4",
fields: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
ReleaseTags: "MKV | 2160p | WEB-DL",
},
want: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX",
ReleaseTags: "MKV | 2160p | WEB-DL",
Container: "MKV",
Season: 1,
Episode: 0,
Resolution: "2160p",
Source: "WEB-DL",
Codec: "HEVC",
HDR: "DV",
Audio: "DDP.5.1", // need to fix audio parsing
Group: "FLUX",
Website: "ATVP",
},
wantErr: false,
},
{
name: "parse_5",
fields: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
ReleaseTags: "MP4 | 2160p | WEB-DL",
},
want: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX",
ReleaseTags: "MP4 | 2160p | WEB-DL",
Container: "MP4",
Season: 1,
Episode: 0,
Resolution: "2160p",
Source: "WEB-DL",
Codec: "HEVC",
HDR: "DV",
Audio: "DDP.5.1", // need to fix audio parsing
Group: "FLUX",
Website: "ATVP",
},
wantErr: false,
},
{
name: "parse_6",
fields: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
ReleaseTags: "MP4 | 2160p | WEB-DL | Freeleech!",
},
want: Release{
TorrentName: "Servant.S01.2160p.ATVP.WEB-DL.DDP.5.1.Atmos.DV.HEVC-FLUX",
Clean: "Servant S01 2160p ATVP WEB DL DDP 5 1 Atmos DV HEVC FLUX",
ReleaseTags: "MP4 | 2160p | WEB-DL | Freeleech!",
Container: "MP4",
Season: 1,
Episode: 0,
Resolution: "2160p",
Source: "WEB-DL",
Codec: "HEVC",
HDR: "DV",
Audio: "DDP.5.1", // need to fix audio parsing
Group: "FLUX",
Website: "ATVP",
Freeleech: true,
},
wantErr: false,
},
{
name: "parse_music_1",
fields: Release{
TorrentName: "Artist - Albumname",
ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD",
},
want: Release{
TorrentName: "Artist - Albumname",
Clean: "Artist Albumname",
ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD",
Group: "",
Audio: "FLAC",
Source: "CD",
HasCue: true,
HasLog: true,
LogScore: 100,
},
wantErr: false,
},
{
name: "parse_music_2",
fields: Release{
TorrentName: "Various Artists - Music '21",
Tags: []string{"house, techno, tech.house, electro.house, future.house, bass.house, melodic.house"},
ReleaseTags: "MP3 / 320 / Cassette",
},
want: Release{
TorrentName: "Various Artists - Music '21",
Clean: "Various Artists Music '21",
Tags: []string{"house, techno, tech.house, electro.house, future.house, bass.house, melodic.house"},
ReleaseTags: "MP3 / 320 / Cassette",
Group: "",
Audio: "MP3",
Source: "Cassette",
Bitrate: "320",
},
wantErr: false,
},
{
name: "parse_music_3",
fields: Release{
TorrentName: "The artist (ザ・フリーダムユニティ) - Long album name",
ReleaseTags: "MP3 / V0 (VBR) / CD",
},
want: Release{
TorrentName: "The artist (ザ・フリーダムユニティ) - Long album name",
Clean: "The artist (ザ・フリーダムユニティ) Long album name",
ReleaseTags: "MP3 / V0 (VBR) / CD",
Group: "",
Audio: "MP3",
Source: "CD",
},
wantErr: false,
},
{
name: "parse_music_4",
fields: Release{
TorrentName: "Artist - Albumname",
ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD",
},
want: Release{
TorrentName: "Artist - Albumname",
Clean: "Artist Albumname",
ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD",
Group: "",
Audio: "FLAC",
Source: "CD",
HasCue: true,
HasLog: true,
LogScore: 100,
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -70,6 +226,8 @@ func TestRelease_Parse(t *testing.T) {
if err := r.Parse(); (err != nil) != tt.wantErr {
t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr)
}
assert.Equal(t, tt.want, r)
})
}
}
@ -143,6 +301,55 @@ func TestRelease_CheckFilter(t *testing.T) {
},
want: true,
},
{
name: "movie_parse_2",
fields: &Release{
TorrentName: "That Movie 2020 2160p Blu-Ray DD5.1 x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001),
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2020",
MatchReleaseGroups: "GROUP1",
},
},
want: true,
},
{
name: "movie_parse_3",
fields: &Release{
TorrentName: "That Movie 2020 2160p WEBDL DD5.1 x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001),
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"2160p"},
Sources: []string{"WEB-DL"},
Codecs: []string{"x264"},
Years: "2020",
MatchReleaseGroups: "GROUP1",
},
},
want: true,
},
{
name: "movie_parse_shows",
fields: &Release{
@ -168,6 +375,31 @@ func TestRelease_CheckFilter(t *testing.T) {
},
want: true,
},
{
name: "movie_parse_shows_1",
fields: &Release{
TorrentName: "That.Movie.2020.2160p.BluRay.DD5.1.x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001),
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2020",
MatchReleaseGroups: "GROUP1",
Shows: "That Movie",
},
},
want: true,
},
{
name: "movie_parse_multiple_shows",
fields: &Release{
@ -193,6 +425,31 @@ func TestRelease_CheckFilter(t *testing.T) {
},
want: true,
},
{
name: "movie_parse_multiple_shows_1",
fields: &Release{
TorrentName: "That.Movie.2020.2160p.BluRay.DD5.1.x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001),
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2020",
MatchReleaseGroups: "GROUP1",
Shows: "That Movie, good story, bad movie",
},
},
want: true,
},
{
name: "movie_parse_wildcard_shows",
fields: &Release{
@ -346,6 +603,182 @@ func TestRelease_CheckFilter(t *testing.T) {
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show",
},
},
want: true,
},
{
name: "match_tags",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
Uploader: "Uploader1",
Tags: []string{"tv"},
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show",
Tags: "tv",
},
},
want: true,
},
{
name: "match_tags_bad",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
Uploader: "Uploader1",
Tags: []string{"foreign"},
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show",
Tags: "tv",
},
},
want: false,
},
{
name: "match_except_tags",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
Uploader: "Uploader1",
Tags: []string{"foreign"},
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show",
ExceptTags: "tv",
},
},
want: true,
},
{
name: "match_except_tags_2",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
Uploader: "Uploader1",
Tags: []string{"foreign"},
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show",
ExceptTags: "foreign",
},
},
want: false,
},
{
name: "match_group_1",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP",
Category: "TV",
Uploader: "Uploader1",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show",
MatchReleaseGroups: "GROUP",
},
},
want: true,
},
{
name: "match_group_potential_partial_1",
fields: &Release{
TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-ift",
Category: "TV",
Uploader: "Uploader1",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show shift",
MatchReleaseGroups: "ift",
},
},
want: true,
},
{
name: "match_group_potential_partial_2",
fields: &Release{
TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP",
Category: "TV",
Uploader: "Uploader1",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show shift",
MatchReleaseGroups: "ift",
},
},
want: false,
},
{
name: "match_group_potential_partial_3",
fields: &Release{
TorrentName: "Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-de[42]",
Category: "TV",
Uploader: "Uploader1",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show shift",
MatchReleaseGroups: "de[42]",
},
},
want: true,
},
{
name: "match_group_potential_partial_3",
fields: &Release{
TorrentName: "[AnimeGroup] Good show shift S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC",
Category: "TV",
Uploader: "Uploader1",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
Shows: "Good show shift",
MatchReleaseGroups: "[AnimeGroup]",
},
},
want: true,
@ -362,3 +795,194 @@ func TestRelease_CheckFilter(t *testing.T) {
})
}
}
func TestRelease_MapVars(t *testing.T) {
type args struct {
varMap map[string]string
}
tests := []struct {
name string
fields *Release
want *Release
args args
}{
{
name: "1",
fields: &Release{},
want: &Release{TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2"},
args: args{varMap: map[string]string{
"torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
}},
},
{
name: "2",
fields: &Release{},
want: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "tv",
Freeleech: true,
Uploader: "Anon",
Size: uint64(10000000000),
},
args: args{varMap: map[string]string{
"torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
"category": "tv",
"freeleech": "freeleech",
"uploader": "Anon",
"torrentSize": "10GB",
}},
},
{
name: "3",
fields: &Release{},
want: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "tv",
FreeleechPercent: 100,
Uploader: "Anon",
Size: uint64(10000000000),
},
args: args{varMap: map[string]string{
"torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
"category": "tv",
"freeleechPercent": "100%",
"uploader": "Anon",
"torrentSize": "10GB",
}},
},
{
name: "4",
fields: &Release{},
want: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "tv",
FreeleechPercent: 100,
Uploader: "Anon",
Size: uint64(10000000000),
Tags: []string{"foreign", "tv"},
},
args: args{varMap: map[string]string{
"torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
"category": "tv",
"freeleechPercent": "100%",
"uploader": "Anon",
"torrentSize": "10GB",
"tags": "foreign,tv",
}},
},
{
name: "5",
fields: &Release{},
want: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "tv",
FreeleechPercent: 100,
Uploader: "Anon",
Size: uint64(10000000000),
Tags: []string{"foreign", "tv"},
},
args: args{varMap: map[string]string{
"torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
"category": "tv",
"freeleechPercent": "100%",
"uploader": "Anon",
"torrentSize": "10GB",
"tags": "foreign,tv",
}},
},
{
name: "6",
fields: &Release{},
want: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "tv",
Year: 2020,
FreeleechPercent: 100,
Uploader: "Anon",
Size: uint64(10000000000),
Tags: []string{"foreign", "tv"},
},
args: args{varMap: map[string]string{
"torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
"category": "tv",
"year": "2020",
"freeleechPercent": "100%",
"uploader": "Anon",
"torrentSize": "10GB",
"tags": "foreign, tv",
}},
},
{
name: "7",
fields: &Release{},
want: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "tv",
Year: 2020,
FreeleechPercent: 100,
Uploader: "Anon",
Size: uint64(10000000000),
Tags: []string{"hip.hop", "rhythm.and.blues", "2000s"},
},
args: args{varMap: map[string]string{
"torrentName": "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
"category": "tv",
"year": "2020",
"freeleechPercent": "100%",
"uploader": "Anon",
"torrentSize": "10GB",
"tags": "hip.hop,rhythm.and.blues, 2000s",
}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := tt.fields
_ = r.MapVars(tt.args.varMap)
assert.Equal(t, tt.want, r)
})
}
}
func TestSplitAny(t *testing.T) {
type args struct {
s string
seps string
}
tests := []struct {
name string
args args
want []string
}{
{
name: "test_1",
args: args{
s: "Tag1 / Tag2 / Tag3",
seps: "/ ",
},
want: []string{"Tag1", "Tag2", "Tag3"},
},
{
name: "test_2",
args: args{
s: "Tag1 | Tag2 | Tag3",
seps: "| ",
},
want: []string{"Tag1", "Tag2", "Tag3"},
},
{
name: "test_3",
args: args{
s: "Tag1 | Tag2 / Tag3",
seps: "| /",
},
want: []string{"Tag1", "Tag2", "Tag3"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, SplitAny(tt.args.s, tt.args.seps), "SplitAny(%v, %v)", tt.args.s, tt.args.seps)
})
}
}

2
internal/filter/service.go
View file

@ -182,6 +182,8 @@ func (s *service) FindAndCheckFilters(release *domain.Release) (bool, *domain.Fi
return false, nil, err
}
log.Trace().Msgf("filter-service.find_and_check_filters: found (%d) active filters to check for indexer '%v'", len(filters), release.Indexer)
// loop and check release to filter until match
for _, f := range filters {
log.Trace().Msgf("checking filter: %+v", f.Name)

2
internal/indexer/definitions/milkie.yaml
View file

@ -49,7 +49,7 @@ parse:
vars:
- category
- torrentName
- tags
- releaseTags
- baseUrl
- torrentId