diff --git a/internal/domain/filter.go b/internal/domain/filter.go index 069a636..c095807 100644 --- a/internal/domain/filter.go +++ b/internal/domain/filter.go @@ -475,7 +475,7 @@ func (f *Filter) CheckFilter(r *Release) ([]string, bool) { f.addRejectionF("formats not matching. got: %v want: %v", r.Audio, f.Formats) } - if len(f.Quality) > 0 && !sliceContainsSlice(r.Audio, f.Quality) { + if len(f.Quality) > 0 && !containsMatchBasic(r.Audio, f.Quality) { f.addRejectionF("quality not matching. got: %v want: %v", r.Audio, f.Quality) } @@ -547,7 +547,7 @@ func (f *Filter) CheckFilter(r *Release) ([]string, bool) { return nil, true } -func (f Filter) checkMaxDownloads() bool { +func (f *Filter) checkMaxDownloads() bool { if f.Downloads == nil { return false } @@ -580,7 +580,7 @@ func (f *Filter) isPerfectFLAC(r *Release) bool { if !containsAny(r.Audio, "Log") { return false } - if !containsAny(r.Audio, "Log100") { + if !containsAny(r.Audio, "Log100") || r.LogScore != 100 { return false } if !containsAny(r.Audio, "FLAC") { @@ -616,6 +616,32 @@ func (f *Filter) checkSizeFilter(r *Release) bool { return true } +// IsPerfectFLAC Perfect is "CD FLAC Cue Log 100% Lossless or 24bit Lossless" +func (f *Filter) IsPerfectFLAC(r *Release) ([]string, bool) { + rejections := []string{} + + if r.Source != "CD" { + rejections = append(rejections, fmt.Sprintf("wanted Source CD, got %s", r.Source)) + } + if r.AudioFormat != "FLAC" { + rejections = append(rejections, fmt.Sprintf("wanted Format FLAC, got %s", r.AudioFormat)) + } + if !r.HasCue { + rejections = append(rejections, fmt.Sprintf("wanted Cue, got %t", r.HasCue)) + } + if !r.HasLog { + rejections = append(rejections, fmt.Sprintf("wanted Log, got %t", r.HasLog)) + } + if r.LogScore != 100 { + rejections = append(rejections, fmt.Sprintf("wanted Log Score 100, got %d", r.LogScore)) + } + if !containsSlice(r.Bitrate, []string{"Lossless", "24bit Lossless"}) { + rejections = append(rejections, fmt.Sprintf("wanted Bitrate Lossless / 24bit Lossless, got %s", r.Bitrate)) + } + + return rejections, len(rejections) == 0 +} + func (f *Filter) addRejection(reason string) { f.Rejections = append(f.Rejections, reason) } diff --git a/internal/domain/filter_test.go b/internal/domain/filter_test.go index 91392b1..a530b82 100644 --- a/internal/domain/filter_test.go +++ b/internal/domain/filter_test.go @@ -1165,16 +1165,17 @@ func TestFilter_CheckFilter(t *testing.T) { LogScore: 100, Cue: true, }, - rejections: []string{"quality not matching. got: [FLAC Lossless Log100 Log] want: [24bit Lossless]", "wanted: cue", "log score. got: 0 want: 100"}, + rejections: []string{"quality not matching. got: [FLAC Lossless Log100 Log] want: [24bit Lossless]", "wanted: cue"}, }, want: false, }, { name: "match_music_5", fields: &Release{ - TorrentName: "Artist - Albumname FLAC CD", + //TorrentName: "Artist - Albumname FLAC CD", + TorrentName: "Artist - Albumname [2022] [Album] (FLAC 24bit Lossless CD)", Year: 2022, - ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD", + ReleaseTags: "FLAC / 24bit Lossless / Log / 100% / Cue / CD", Category: "Album", }, args: args{ @@ -1185,11 +1186,12 @@ func TestFilter_CheckFilter(t *testing.T) { Artists: "Artist", Media: []string{"CD"}, Formats: []string{"FLAC"}, - Quality: []string{"24bit Lossless", "Lossless"}, - PerfectFlac: true, - Log: true, + Quality: []string{"24bit Lossless"}, + //PerfectFlac: true, + //Log: true, //LogScore: 100, Cue: true, + //Cue: true, }, }, want: true, @@ -1214,7 +1216,7 @@ func TestFilter_CheckFilter(t *testing.T) { LogScore: 100, Cue: true, }, - rejections: []string{"release type not matching. got: Album want: [Single]", "log score. got: 0 want: 100"}, + rejections: []string{"release type not matching. got: Album want: [Single]"}, }, want: false, }, @@ -1238,7 +1240,7 @@ func TestFilter_CheckFilter(t *testing.T) { LogScore: 100, Cue: true, }, - rejections: []string{"artists not matching. got: Artist want: Artiiiist", "log score. got: 0 want: 100"}, + rejections: []string{"artists not matching. got: Artist want: Artiiiist"}, }, want: false, }, @@ -1266,6 +1268,28 @@ func TestFilter_CheckFilter(t *testing.T) { }, want: true, }, + { + name: "match_music_9", + fields: &Release{ + TorrentName: "Artist - Albumname [2022] [Album] (FLAC 24bit Lossless CD)", + Year: 2022, + ReleaseTags: "FLAC / 24bit Lossless / Log / 100% / Cue / CD", + Category: "Album", + }, + args: args{ + filter: Filter{ + Enabled: true, + MatchReleaseTypes: []string{"Album"}, + Years: "2020-2022", + Artists: "Artist", + Media: []string{"CD"}, + Formats: []string{"FLAC"}, + Quality: []string{"Lossless"}, + }, + rejections: []string{"quality not matching. got: [24BIT Lossless Cue FLAC Log100 Log] want: [Lossless]"}, + }, + want: false, + }, { name: "match_anime_1", fields: &Release{ @@ -2130,8 +2154,8 @@ func Test_matchRegex(t *testing.T) { {name: "test_5", args: args{tag: "Some.show.S01.DV.2160p.ATVP.WEB-DL.DDPA5.1.x265-GROUP2", filter: ".*1080p.+(group1|group3),.*720p.+,"}, want: false}, {name: "test_6", args: args{tag: "[Group] -Name of a Novel Something Good- [2012][Translated (Group)][EPUB]", filter: "(?:.*Something Good.*|.*Something Bad.*)"}, want: true}, {name: "test_7", args: args{tag: "[Group] -Name of a Novel Something Good- [2012][Translated (Group)][EPUB]", filter: "(?:.*Something Funny.*|.*Something Bad.*)"}, want: false}, - {name: "test_8", args: args{tag: ".s10E123.", filter:`\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: true}, - {name: "test_9", args: args{tag: "S1E1", filter:`\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: false}, + {name: "test_8", args: args{tag: ".s10E123.", filter: `\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: true}, + {name: "test_9", args: args{tag: "S1E1", filter: `\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/domain/indexer.go b/internal/domain/indexer.go index 11fbe03..274ac74 100644 --- a/internal/domain/indexer.go +++ b/internal/domain/indexer.go @@ -352,6 +352,8 @@ func (p *IndexerIRCParse) Parse(def *IndexerDefinition, vars map[string]string, parser = IRCParserGazelleGames{} case "ops": parser = IRCParserOrpheus{} + case "redacted": + parser = IRCParserRedacted{} default: parser = IRCParserDefault{} } diff --git a/internal/domain/indexer_test.go b/internal/domain/indexer_test.go index 59c8de1..4f3e47d 100644 --- a/internal/domain/indexer_test.go +++ b/internal/domain/indexer_test.go @@ -335,11 +335,14 @@ func TestIRCParserOrpheus_Parse(t *testing.T) { rls: NewRelease("ops"), vars: map[string]string{ "torrentName": "Busta Rhymes – BEACH BALL (feat. BIA) – [2023] [Single] WEB/FLAC/24bit Lossless", + "title": "Busta Rhymes – BEACH BALL (feat. BIA)", + "year": "2023", + "releaseTags": "WEB/FLAC/24bit Lossless", }, }, want: want{ - title: "BEACH BALL", - release: "Busta Rhymes - BEACH BALL (feat. BIA) - [2023] [Single] WEB/FLAC/24bit Lossless", + title: "Busta Rhymes - BEACH BALL (feat. BIA)", + release: "Busta Rhymes - BEACH BALL (feat. BIA) [2023] (WEB FLAC 24BIT Lossless)", }, }, { @@ -348,11 +351,14 @@ func TestIRCParserOrpheus_Parse(t *testing.T) { rls: NewRelease("ops"), vars: map[string]string{ "torrentName": "Busta Rhymes – BEACH BALL (feat. BIA) – [2023] [Single] CD/FLAC/Lossless", + "title": "Busta Rhymes – BEACH BALL (feat. BIA)", + "year": "2023", + "releaseTags": "CD/FLAC/Lossless", }, }, want: want{ - title: "BEACH BALL", - release: "Busta Rhymes - BEACH BALL (feat. BIA) - [2023] [Single] CD/FLAC/Lossless", + title: "Busta Rhymes - BEACH BALL (feat. BIA)", + release: "Busta Rhymes - BEACH BALL (feat. BIA) [2023] (CD FLAC Lossless)", }, }, } diff --git a/internal/domain/irc.go b/internal/domain/irc.go index fb8c032..0497838 100644 --- a/internal/domain/irc.go +++ b/internal/domain/irc.go @@ -6,6 +6,7 @@ package domain import ( "context" "encoding/json" + "fmt" "strings" "time" ) @@ -185,13 +186,88 @@ func (p IRCParserGazelleGames) Parse(rls *Release, vars map[string]string) error type IRCParserOrpheus struct{} +func (p IRCParserOrpheus) replaceSeparator(s string) string { + return strings.ReplaceAll(s, "–", "-") +} + func (p IRCParserOrpheus) Parse(rls *Release, vars map[string]string) error { // OPS uses en-dashes as separators, which causes moistari/rls to not parse the torrentName properly, // we replace the en-dashes with hyphens here - torrentName := vars["torrentName"] - rls.TorrentName = strings.ReplaceAll(torrentName, "–", "-") + torrentName := p.replaceSeparator(vars["torrentName"]) + title := p.replaceSeparator(vars["title"]) - rls.ParseString(rls.TorrentName) + year := vars["year"] + releaseTagsString := vars["releaseTags"] + + //cleanTags := strings.ReplaceAll(releaseTagsString, "/", " ") + cleanTags := CleanReleaseTags(releaseTagsString) + + tags := ParseReleaseTagString(cleanTags) + rls.ReleaseTags = cleanTags + + audio := []string{} + if tags.Source != "" { + audio = append(audio, tags.Source) + } + if tags.AudioFormat != "" { + audio = append(audio, tags.AudioFormat) + } + if tags.AudioBitrate != "" { + audio = append(audio, tags.AudioBitrate) + } + rls.Bitrate = tags.AudioBitrate + rls.AudioFormat = tags.AudioFormat + + // set log score + rls.HasLog = tags.HasLog + rls.LogScore = tags.LogScore + rls.HasCue = tags.HasCue + + // Construct new release name so we have full control. We remove category such as EP/Single/Album because EP is being mis-parsed. + //torrentName = fmt.Sprintf("%s [%s] (%s)", title, year, strings.Join(audio, " ")) + torrentName = fmt.Sprintf("%s [%s] (%s)", title, year, strings.Join(audio, " ")) + + rls.ParseString(torrentName) + rls.Title = title + + return nil +} + +// IRCParserRedacted parser for Redacted announces +type IRCParserRedacted struct{} + +func (p IRCParserRedacted) Parse(rls *Release, vars map[string]string) error { + title := vars["title"] + year := vars["year"] + releaseTagsString := vars["releaseTags"] + + cleanTags := CleanReleaseTags(releaseTagsString) + + tags := ParseReleaseTagString(cleanTags) + + audio := []string{} + if tags.Source != "" { + audio = append(audio, tags.Source) + } + if tags.AudioFormat != "" { + audio = append(audio, tags.AudioFormat) + } + if tags.AudioBitrate != "" { + audio = append(audio, tags.AudioBitrate) + } + rls.Bitrate = tags.AudioBitrate + rls.AudioFormat = tags.AudioFormat + + // set log score + rls.HasLog = tags.HasLog + rls.LogScore = tags.LogScore + rls.HasCue = tags.HasCue + + // Construct new release name so we have full control. We remove category such as EP/Single/Album because EP is being mis-parsed. + name := fmt.Sprintf("%s [%s] (%s)", title, year, strings.Join(audio, " ")) + + rls.ParseString(name) + rls.Title = title return nil } diff --git a/internal/domain/release.go b/internal/domain/release.go index 12e478c..f9727fe 100644 --- a/internal/domain/release.go +++ b/internal/domain/release.go @@ -12,7 +12,6 @@ import ( "net/http" "net/http/cookiejar" "os" - "regexp" "strconv" "strings" "time" @@ -75,6 +74,8 @@ type Release struct { HDR []string `json:"hdr"` Audio []string `json:"-"` AudioChannels string `json:"-"` + AudioFormat string `json:"-"` + Bitrate string `json:"-"` Group string `json:"group"` Region string `json:"-"` Language []string `json:"-"` @@ -84,6 +85,8 @@ type Release struct { Artists string `json:"-"` Type string `json:"type"` // Album,Single,EP LogScore int `json:"-"` + HasCue bool `json:"-"` + HasLog bool `json:"-"` Origin string `json:"origin"` // P2P, Internal Tags []string `json:"-"` ReleaseTags string `json:"-"` @@ -289,6 +292,8 @@ func NewRelease(indexer string) *Release { func (r *Release) ParseString(title string) { rel := rls.ParseString(title) + r.Type = rel.Type.String() + r.TorrentName = title r.Source = rel.Source r.Resolution = rel.Resolution @@ -327,18 +332,40 @@ func (r *Release) ParseString(title string) { var ErrUnrecoverableError = errors.New("unrecoverable error") func (r *Release) ParseReleaseTagsString(tags string) { - // trim delimiters and closest space - re := regexp.MustCompile(`\| |/ |, `) - cleanTags := re.ReplaceAllString(tags, "") - + cleanTags := CleanReleaseTags(tags) t := ParseReleaseTagString(cleanTags) if len(t.Audio) > 0 { - r.Audio = getUniqueTags(r.Audio, t.Audio) + //r.Audio = getUniqueTags(r.Audio, t.Audio) + r.Audio = t.Audio + } + + if t.AudioBitrate != "" { + r.Bitrate = t.AudioBitrate + } + + if t.AudioFormat != "" { + r.AudioFormat = t.AudioFormat + } + + if r.AudioChannels == "" && t.Channels != "" { + r.AudioChannels = t.Channels + } + + if t.HasLog { + r.HasLog = true + + if t.LogScore > 0 { + r.LogScore = t.LogScore + } + } + + if t.HasCue { + r.HasCue = true } if len(t.Bonus) > 0 { - if sliceContainsSlice([]string{"Freeleech"}, t.Bonus) { + if sliceContainsSlice([]string{"Freeleech", "Freeleech!"}, t.Bonus) { r.Freeleech = true } // TODO handle percent and other types @@ -363,9 +390,6 @@ func (r *Release) ParseReleaseTagsString(tags string) { if r.Source == "" && t.Source != "" { r.Source = t.Source } - if r.AudioChannels == "" && t.Channels != "" { - r.AudioChannels = t.Channels - } } // ParseSizeBytesString If there are parsing errors, then it keeps the original (or default size 0) diff --git a/internal/domain/release_download_test.go b/internal/domain/release_download_test.go new file mode 100644 index 0000000..a99aad0 --- /dev/null +++ b/internal/domain/release_download_test.go @@ -0,0 +1,276 @@ +// Copyright (c) 2021 - 2024, Ludvig Lundgren and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +//go:build integration + +package domain + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/rs/zerolog" +) + +var trackerLessTestTorrent = `d7:comment19:This is just a test10:created by12:Johnny Bravo13:creation datei1430648794e8:encoding5:UTF-84:infod6:lengthi1128e4:name12:testfile.bin12:piece lengthi32768e6:pieces20:Õˆë =‘UŒäiÎ^æ °Eâ?ÇÒe5:nodesl35:udp://tracker.openbittorrent.com:8035:udp://tracker.openbittorrent.com:80ee` + +func TestRelease_DownloadTorrentFile(t *testing.T) { + // disable logger + zerolog.SetGlobalLevel(zerolog.Disabled) + + mux := http.NewServeMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + mux.HandleFunc("/files/valid_torrent_as_html", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + payload, _ := os.ReadFile("testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent") + w.Write(payload) + }) + + mux.HandleFunc("/files/invalid_torrent_as_html", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + payload := []byte("This is not the torrent you are looking for") + w.Write(payload) + }) + + mux.HandleFunc("/index.html", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + payload := []byte("This is not the torrent you are looking for") + w.Write(payload) + }) + + mux.HandleFunc("/plaintext", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/plain") + payload := []byte("This is not a valid torrent file.") + w.Write(payload) + }) + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.RequestURI, "401") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("unauthorized")) + return + } + if strings.Contains(r.RequestURI, "403") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("forbidden")) + return + } + if strings.Contains(r.RequestURI, "404") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("not found")) + return + } + if strings.Contains(r.RequestURI, "405") { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("method not allowed")) + return + } + + if strings.Contains(r.RequestURI, "file.torrent") { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/x-bittorrent") + payload, _ := os.ReadFile("testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent") + w.Write(payload) + return + } + + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("internal error")) + }) + + type fields struct { + ID int64 + FilterStatus ReleaseFilterStatus + Rejections []string + Indexer string + FilterName string + Protocol ReleaseProtocol + Implementation ReleaseImplementation + Timestamp time.Time + GroupID string + TorrentID string + DownloadURL string + TorrentTmpFile string + TorrentDataRawBytes []byte + TorrentHash string + TorrentName string + Size uint64 + Title string + Category string + Categories []string + Season int + Episode int + Year int + Resolution string + Source string + Codec []string + Container string + HDR []string + Audio []string + AudioChannels string + Group string + Region string + Language []string + Proper bool + Repack bool + Website string + Artists string + Type string + LogScore int + Origin string + Tags []string + ReleaseTags string + Freeleech bool + FreeleechPercent int + Bonus []string + Uploader string + PreTime string + Other []string + RawCookie string + AdditionalSizeCheckRequired bool + FilterID int + Filter *Filter + ActionStatus []ReleaseActionStatus + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "401", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + DownloadURL: fmt.Sprintf("%s/%d", ts.URL, 401), + Protocol: ReleaseProtocolTorrent, + }, + wantErr: true, + }, + { + name: "403", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + DownloadURL: fmt.Sprintf("%s/%d", ts.URL, 403), + Protocol: ReleaseProtocolTorrent, + }, + wantErr: true, + }, + { + name: "500", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + DownloadURL: fmt.Sprintf("%s/%d", ts.URL, 500), + Protocol: ReleaseProtocolTorrent, + }, + wantErr: true, + }, + { + name: "ok", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + DownloadURL: fmt.Sprintf("%s/%s", ts.URL, "file.torrent"), + Protocol: ReleaseProtocolTorrent, + }, + wantErr: false, + }, + { + name: "valid_torrent_with_text-html_header", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + DownloadURL: fmt.Sprintf("%s/files/%s", ts.URL, "valid_torrent_as_html"), + Protocol: ReleaseProtocolTorrent, + }, + wantErr: false, + }, + { + name: "invalid_torrent_with_text-html_header", + fields: fields{ + Indexer: "mock-indexer", + TorrentName: "Test.Release-GROUP", + DownloadURL: fmt.Sprintf("%s/files/%s", ts.URL, "invalid_torrent_as_html"), + Protocol: ReleaseProtocolTorrent, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Release{ + ID: tt.fields.ID, + FilterStatus: tt.fields.FilterStatus, + Rejections: tt.fields.Rejections, + Indexer: tt.fields.Indexer, + FilterName: tt.fields.FilterName, + Protocol: tt.fields.Protocol, + Implementation: tt.fields.Implementation, + Timestamp: tt.fields.Timestamp, + GroupID: tt.fields.GroupID, + TorrentID: tt.fields.TorrentID, + DownloadURL: tt.fields.DownloadURL, + TorrentTmpFile: tt.fields.TorrentTmpFile, + TorrentDataRawBytes: tt.fields.TorrentDataRawBytes, + TorrentHash: tt.fields.TorrentHash, + TorrentName: tt.fields.TorrentName, + Size: tt.fields.Size, + Title: tt.fields.Title, + Category: tt.fields.Category, + Categories: tt.fields.Categories, + Season: tt.fields.Season, + Episode: tt.fields.Episode, + Year: tt.fields.Year, + Resolution: tt.fields.Resolution, + Source: tt.fields.Source, + Codec: tt.fields.Codec, + Container: tt.fields.Container, + HDR: tt.fields.HDR, + Audio: tt.fields.Audio, + AudioChannels: tt.fields.AudioChannels, + Group: tt.fields.Group, + Region: tt.fields.Region, + Language: tt.fields.Language, + Proper: tt.fields.Proper, + Repack: tt.fields.Repack, + Website: tt.fields.Website, + Artists: tt.fields.Artists, + Type: tt.fields.Type, + LogScore: tt.fields.LogScore, + Origin: tt.fields.Origin, + Tags: tt.fields.Tags, + ReleaseTags: tt.fields.ReleaseTags, + Freeleech: tt.fields.Freeleech, + FreeleechPercent: tt.fields.FreeleechPercent, + Bonus: tt.fields.Bonus, + Uploader: tt.fields.Uploader, + PreTime: tt.fields.PreTime, + Other: tt.fields.Other, + RawCookie: tt.fields.RawCookie, + AdditionalSizeCheckRequired: tt.fields.AdditionalSizeCheckRequired, + FilterID: tt.fields.FilterID, + Filter: tt.fields.Filter, + ActionStatus: tt.fields.ActionStatus, + } + err := r.DownloadTorrentFile() + if err == nil && tt.wantErr { + fmt.Println("error") + } + + }) + } +} diff --git a/internal/domain/release_test.go b/internal/domain/release_test.go index f49fb68..b45ea39 100644 --- a/internal/domain/release_test.go +++ b/internal/domain/release_test.go @@ -4,15 +4,8 @@ package domain import ( - "fmt" - "net/http" - "net/http/httptest" - "os" - "strings" "testing" - "time" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" ) @@ -40,6 +33,7 @@ func TestRelease_Parse(t *testing.T) { HDR: []string{"DV"}, Group: "FLUX", //Website: "ATVP", + Type: "series", }, }, { @@ -59,6 +53,7 @@ func TestRelease_Parse(t *testing.T) { AudioChannels: "5.1", HDR: []string{"DV"}, Group: "FLUX", + Type: "series", }, }, { @@ -81,6 +76,7 @@ func TestRelease_Parse(t *testing.T) { AudioChannels: "5.1", HDR: []string{"DV"}, Group: "FLUX", + Type: "series", }, }, { @@ -103,6 +99,7 @@ func TestRelease_Parse(t *testing.T) { AudioChannels: "5.1", HDR: []string{"DV"}, Group: "FLUX", + Type: "series", }, }, { @@ -125,6 +122,7 @@ func TestRelease_Parse(t *testing.T) { AudioChannels: "5.1", HDR: []string{"DV"}, Group: "FLUX", + Type: "series", }, }, { @@ -149,6 +147,7 @@ func TestRelease_Parse(t *testing.T) { Group: "FLUX", Freeleech: true, Bonus: []string{"Freeleech"}, + Type: "series", }, }, { @@ -163,7 +162,12 @@ func TestRelease_Parse(t *testing.T) { Title: "Artist", Group: "Albumname", Audio: []string{"Cue", "FLAC", "Lossless", "Log100", "Log"}, + AudioFormat: "FLAC", Source: "CD", + Bitrate: "Lossless", + HasLog: true, + LogScore: 100, + HasCue: true, }, }, { @@ -180,6 +184,8 @@ func TestRelease_Parse(t *testing.T) { Title: "Various Artists - Music '21", Source: "Cassette", Audio: []string{"320", "MP3"}, + AudioFormat: "MP3", + Bitrate: "320", }, }, { @@ -194,7 +200,9 @@ func TestRelease_Parse(t *testing.T) { Title: "The artist", Group: "name", Source: "CD", - Audio: []string{"MP3", "VBR"}, + Audio: []string{"MP3", "VBR", "V0 (VBR)"}, + AudioFormat: "MP3", + Bitrate: "V0 (VBR)", }, }, { @@ -209,7 +217,12 @@ func TestRelease_Parse(t *testing.T) { Title: "Artist", Group: "Albumname", Audio: []string{"Cue", "FLAC", "Lossless", "Log100", "Log"}, + AudioFormat: "FLAC", Source: "CD", + Bitrate: "Lossless", + HasLog: true, + LogScore: 100, + HasCue: true, }, }, { @@ -224,7 +237,32 @@ func TestRelease_Parse(t *testing.T) { Title: "Artist", Group: "Albumname", Audio: []string{"24BIT Lossless", "Cue", "FLAC", "Log100", "Log"}, + AudioFormat: "FLAC", Source: "CD", + Bitrate: "24BIT Lossless", + HasLog: true, + LogScore: 100, + HasCue: true, + }, + }, + { + name: "parse_music_6", + fields: Release{ + TorrentName: "Artist - Albumname", + ReleaseTags: "FLAC / 24bit Lossless / Log / 78% / Cue / CD", + }, + want: Release{ + TorrentName: "Artist - Albumname", + ReleaseTags: "FLAC / 24bit Lossless / Log / 78% / Cue / CD", + Title: "Artist", + Group: "Albumname", + Audio: []string{"24BIT Lossless", "Cue", "FLAC", "Log78", "Log"}, + AudioFormat: "FLAC", + Source: "CD", + Bitrate: "24BIT Lossless", + HasLog: true, + LogScore: 78, + HasCue: true, }, }, { @@ -244,6 +282,7 @@ func TestRelease_Parse(t *testing.T) { Year: 2007, Group: "GROUP1", Other: []string{"HYBRiD", "REMUX"}, + Type: "movie", }, }, { @@ -264,6 +303,7 @@ func TestRelease_Parse(t *testing.T) { Group: "GROUP1", Season: 1, Language: []string{"ENGLiSH"}, + Type: "series", }, }, } @@ -663,264 +703,6 @@ func TestRelease_ParseString(t *testing.T) { } } -var trackerLessTestTorrent = `d7:comment19:This is just a test10:created by12:Johnny Bravo13:creation datei1430648794e8:encoding5:UTF-84:infod6:lengthi1128e4:name12:testfile.bin12:piece lengthi32768e6:pieces20:Õˆë =‘UŒäiÎ^æ °Eâ?ÇÒe5:nodesl35:udp://tracker.openbittorrent.com:8035:udp://tracker.openbittorrent.com:80ee` - -func TestRelease_DownloadTorrentFile(t *testing.T) { - // disable logger - zerolog.SetGlobalLevel(zerolog.Disabled) - - mux := http.NewServeMux() - ts := httptest.NewServer(mux) - defer ts.Close() - - mux.HandleFunc("/files/valid_torrent_as_html", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "text/html") - payload, _ := os.ReadFile("testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent") - w.Write(payload) - }) - - mux.HandleFunc("/files/invalid_torrent_as_html", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "text/html") - payload := []byte("This is not the torrent you are looking for") - w.Write(payload) - }) - - mux.HandleFunc("/index.html", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "text/html") - payload := []byte("This is not the torrent you are looking for") - w.Write(payload) - }) - - mux.HandleFunc("/plaintext", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "text/plain") - payload := []byte("This is not a valid torrent file.") - w.Write(payload) - }) - - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - if strings.Contains(r.RequestURI, "401") { - w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte("unauthorized")) - return - } - if strings.Contains(r.RequestURI, "403") { - w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte("forbidden")) - return - } - if strings.Contains(r.RequestURI, "404") { - w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte("not found")) - return - } - if strings.Contains(r.RequestURI, "405") { - w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte("method not allowed")) - return - } - - if strings.Contains(r.RequestURI, "file.torrent") { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "application/x-bittorrent") - payload, _ := os.ReadFile("testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent") - w.Write(payload) - return - } - - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("internal error")) - }) - - type fields struct { - ID int64 - FilterStatus ReleaseFilterStatus - Rejections []string - Indexer string - FilterName string - Protocol ReleaseProtocol - Implementation ReleaseImplementation - Timestamp time.Time - GroupID string - TorrentID string - DownloadURL string - TorrentTmpFile string - TorrentDataRawBytes []byte - TorrentHash string - TorrentName string - Size uint64 - Title string - Category string - Categories []string - Season int - Episode int - Year int - Resolution string - Source string - Codec []string - Container string - HDR []string - Audio []string - AudioChannels string - Group string - Region string - Language []string - Proper bool - Repack bool - Website string - Artists string - Type string - LogScore int - Origin string - Tags []string - ReleaseTags string - Freeleech bool - FreeleechPercent int - Bonus []string - Uploader string - PreTime string - Other []string - RawCookie string - AdditionalSizeCheckRequired bool - FilterID int - Filter *Filter - ActionStatus []ReleaseActionStatus - } - tests := []struct { - name string - fields fields - wantErr bool - }{ - { - name: "401", - fields: fields{ - Indexer: "mock-indexer", - TorrentName: "Test.Release-GROUP", - DownloadURL: fmt.Sprintf("%s/%d", ts.URL, 401), - Protocol: ReleaseProtocolTorrent, - }, - wantErr: true, - }, - { - name: "403", - fields: fields{ - Indexer: "mock-indexer", - TorrentName: "Test.Release-GROUP", - DownloadURL: fmt.Sprintf("%s/%d", ts.URL, 403), - Protocol: ReleaseProtocolTorrent, - }, - wantErr: true, - }, - { - name: "500", - fields: fields{ - Indexer: "mock-indexer", - TorrentName: "Test.Release-GROUP", - DownloadURL: fmt.Sprintf("%s/%d", ts.URL, 500), - Protocol: ReleaseProtocolTorrent, - }, - wantErr: true, - }, - { - name: "ok", - fields: fields{ - Indexer: "mock-indexer", - TorrentName: "Test.Release-GROUP", - DownloadURL: fmt.Sprintf("%s/%s", ts.URL, "file.torrent"), - Protocol: ReleaseProtocolTorrent, - }, - wantErr: false, - }, - { - name: "valid_torrent_with_text-html_header", - fields: fields{ - Indexer: "mock-indexer", - TorrentName: "Test.Release-GROUP", - DownloadURL: fmt.Sprintf("%s/files/%s", ts.URL, "valid_torrent_as_html"), - Protocol: ReleaseProtocolTorrent, - }, - wantErr: false, - }, - { - name: "invalid_torrent_with_text-html_header", - fields: fields{ - Indexer: "mock-indexer", - TorrentName: "Test.Release-GROUP", - DownloadURL: fmt.Sprintf("%s/files/%s", ts.URL, "invalid_torrent_as_html"), - Protocol: ReleaseProtocolTorrent, - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := &Release{ - ID: tt.fields.ID, - FilterStatus: tt.fields.FilterStatus, - Rejections: tt.fields.Rejections, - Indexer: tt.fields.Indexer, - FilterName: tt.fields.FilterName, - Protocol: tt.fields.Protocol, - Implementation: tt.fields.Implementation, - Timestamp: tt.fields.Timestamp, - GroupID: tt.fields.GroupID, - TorrentID: tt.fields.TorrentID, - DownloadURL: tt.fields.DownloadURL, - TorrentTmpFile: tt.fields.TorrentTmpFile, - TorrentDataRawBytes: tt.fields.TorrentDataRawBytes, - TorrentHash: tt.fields.TorrentHash, - TorrentName: tt.fields.TorrentName, - Size: tt.fields.Size, - Title: tt.fields.Title, - Category: tt.fields.Category, - Categories: tt.fields.Categories, - Season: tt.fields.Season, - Episode: tt.fields.Episode, - Year: tt.fields.Year, - Resolution: tt.fields.Resolution, - Source: tt.fields.Source, - Codec: tt.fields.Codec, - Container: tt.fields.Container, - HDR: tt.fields.HDR, - Audio: tt.fields.Audio, - AudioChannels: tt.fields.AudioChannels, - Group: tt.fields.Group, - Region: tt.fields.Region, - Language: tt.fields.Language, - Proper: tt.fields.Proper, - Repack: tt.fields.Repack, - Website: tt.fields.Website, - Artists: tt.fields.Artists, - Type: tt.fields.Type, - LogScore: tt.fields.LogScore, - Origin: tt.fields.Origin, - Tags: tt.fields.Tags, - ReleaseTags: tt.fields.ReleaseTags, - Freeleech: tt.fields.Freeleech, - FreeleechPercent: tt.fields.FreeleechPercent, - Bonus: tt.fields.Bonus, - Uploader: tt.fields.Uploader, - PreTime: tt.fields.PreTime, - Other: tt.fields.Other, - RawCookie: tt.fields.RawCookie, - AdditionalSizeCheckRequired: tt.fields.AdditionalSizeCheckRequired, - FilterID: tt.fields.FilterID, - Filter: tt.fields.Filter, - ActionStatus: tt.fields.ActionStatus, - } - err := r.DownloadTorrentFile() - if err == nil && tt.wantErr { - fmt.Println("error") - } - - }) - } -} - func Test_getUniqueTags(t *testing.T) { type args struct { target []string diff --git a/internal/domain/releasetags.go b/internal/domain/releasetags.go index 2154c05..6e107e9 100644 --- a/internal/domain/releasetags.go +++ b/internal/domain/releasetags.go @@ -6,6 +6,7 @@ package domain import ( "fmt" "regexp" + "strconv" "github.com/autobrr/autobrr/pkg/errors" ) @@ -27,6 +28,8 @@ func init() { {tag: "AAC", title: "Advanced Audio Coding (LC)", regexp: "", re: nil}, {tag: "AC3D", title: "", regexp: "ac[\\-\\._ ]?3d", re: nil}, {tag: "Atmos", title: "Dolby Atmos", regexp: "", re: nil}, + {tag: "APS (VBR)", title: "APS Variable Bit Rate", regexp: "", re: nil}, + {tag: "APX (VBR)", title: "APX Variable Bit Rate", regexp: "", re: nil}, {tag: "CBR", title: "Constant Bit Rate", regexp: "", re: nil}, {tag: "Cue", title: "Cue File", regexp: "", re: nil}, {tag: "DDPA", title: "Dolby Digital+ Atmos (E-AC-3+Atmos)", regexp: "dd[p\\+]a", re: nil}, @@ -46,7 +49,7 @@ func init() { {tag: "FLAC", title: "Free Lossless Audio Codec", regexp: "", re: nil}, {tag: "LiNE", title: "Line", regexp: "(?-i:L[iI]NE)", re: nil}, {tag: "Lossless", title: "", regexp: "(?i:(?:^|[^t] )Lossless)", re: nil}, - {tag: "Log100", title: "", regexp: "(log 100%|log \\(100%\\))", re: nil}, + {tag: "LogScore", title: "LogScore", regexp: "log\\s?(?:\\(|\\s)(\\d+)%\\)?", re: nil}, {tag: "Log", title: "", regexp: "(?:log)", re: nil}, {tag: "LPCM", title: "Linear Pulse-Code Modulation", regexp: "", re: nil}, {tag: "MP3", title: "", regexp: "", re: nil}, @@ -54,9 +57,66 @@ func init() { {tag: "OPUS", title: "", regexp: "", re: nil}, {tag: "TrueHD", title: "Dolby TrueHD", regexp: "(?:dolby[\\-\\._ ]?)?true[\\-\\._ ]?hd", re: nil}, {tag: "VBR", title: "Variable Bit Rate", regexp: "", re: nil}, + {tag: "V0 (VBR)", title: "V0 Variable Bit Rate", regexp: "", re: nil}, + {tag: "V1 (VBR)", title: "V1 Variable Bit Rate", regexp: "", re: nil}, + {tag: "V2 (VBR)", title: "V2 Variable Bit Rate", regexp: "", re: nil}, } types["audio"] = audio + audioBitrate := []*TagInfo{ + {tag: "24BIT", title: "", regexp: "(?-i:24BIT)", re: nil}, + {tag: "24BIT Lossless", title: "", regexp: "(?:24BIT lossless)", re: nil}, + {tag: "16BIT", title: "", regexp: "(?-i:16BIT)", re: nil}, + {tag: "320", title: "320 Kbps", regexp: "320[\\\\-\\\\._ kbps]?", re: nil}, + {tag: "256", title: "256 Kbps", regexp: "256[\\\\-\\\\._ kbps]?", re: nil}, + {tag: "192", title: "192 Kbps", regexp: "192[\\\\-\\\\._ kbps]?", re: nil}, + {tag: "128", title: "128 Kbps", regexp: "128[\\\\-\\\\._ kbps]?", re: nil}, + {tag: "APS (VBR)", title: "APS Variable Bit Rate", regexp: "", re: nil}, + {tag: "APX (VBR)", title: "APX Variable Bit Rate", regexp: "", re: nil}, + {tag: "CBR", title: "Constant Bit Rate", regexp: "", re: nil}, + {tag: "Lossless", title: "", regexp: "(?i:(?:^|[^t] )Lossless)", re: nil}, + {tag: "VBR", title: "Variable Bit Rate", regexp: "", re: nil}, + {tag: "V0 (VBR)", title: "V0 Variable Bit Rate", regexp: "", re: nil}, + {tag: "V1 (VBR)", title: "V1 Variable Bit Rate", regexp: "", re: nil}, + {tag: "V2 (VBR)", title: "V2 Variable Bit Rate", regexp: "", re: nil}, + } + types["audioBitrate"] = audioBitrate + + audioFormat := []*TagInfo{ + {tag: "AAC-LC", title: "Advanced Audio Coding (LC)", regexp: "aac[\\-\\._ ]?lc", re: nil}, + {tag: "AAC", title: "Advanced Audio Coding (LC)", regexp: "", re: nil}, + {tag: "AC3D", title: "", regexp: "ac[\\-\\._ ]?3d", re: nil}, + {tag: "Atmos", title: "Dolby Atmos", regexp: "", re: nil}, + {tag: "DDPA", title: "Dolby Digital+ Atmos (E-AC-3+Atmos)", regexp: "dd[p\\+]a", re: nil}, + {tag: "DDP", title: "Dolby Digital+ (E-AC-3)", regexp: "dd[p\\+]|e[\\-\\._ ]?ac3", re: nil}, + {tag: "DD", title: "Dolby Digital (AC-3)", regexp: "dd|ac3|dolby[\\-\\._ ]?digital", re: nil}, + {tag: "DTS-HD.HRA", title: "DTS (HD HRA)", regexp: "dts[\\-\\._ ]?hd[\\-\\._ ]?hra", re: nil}, + {tag: "DTS-HD.HR", title: "DTS (HD HR)", regexp: "dts[\\-\\._ ]?hd[\\-\\._ ]?hr", re: nil}, + {tag: "DTS-HD.MA", title: "DTS (HD MA)", regexp: "dts[\\-\\._ ]?hd[\\-\\._ ]?ma", re: nil}, + {tag: "DTS-HD", title: "DTS (HD)", regexp: "dts[\\-\\._ ]?hd[\\-\\._ ]?", re: nil}, + {tag: "DTS-MA", title: "DTS (MA)", regexp: "dts[\\-\\._ ]?ma[\\-\\._ ]?", re: nil}, + {tag: "DTS-X", title: "DTS (X)", regexp: "dts[\\-\\._ ]?x", re: nil}, + {tag: "DTS", title: "", regexp: "", re: nil}, + {tag: "EAC3D", title: "", regexp: "", re: nil}, + {tag: "ES", title: "Dolby Digital (ES)", regexp: "(?-i:ES)", re: nil}, + {tag: "EX", title: "Dolby Digital (EX)", regexp: "(?-i:EX)", re: nil}, + {tag: "FLAC", title: "Free Lossless Audio Codec", regexp: "", re: nil}, + {tag: "LPCM", title: "Linear Pulse-Code Modulation", regexp: "", re: nil}, + {tag: "MP3", title: "", regexp: "", re: nil}, + {tag: "OGG", title: "", regexp: "", re: nil}, + {tag: "OPUS", title: "", regexp: "", re: nil}, + {tag: "TrueHD", title: "Dolby TrueHD", regexp: "(?:dolby[\\-\\._ ]?)?true[\\-\\._ ]?hd", re: nil}, + } + types["audioFormat"] = audioFormat + + audioExtra := []*TagInfo{ + {tag: "Cue", title: "Cue File", regexp: "", re: nil}, + {tag: "Log100", title: "", regexp: "(log 100%|log \\(100%\\))", re: nil}, + {tag: "LogScore", title: "LogScore", regexp: "log\\s?(?:\\(|\\s)(\\d+)%\\)?", re: nil}, + {tag: "Log", title: "", regexp: "(?:log)", re: nil}, + } + types["audioExtra"] = audioExtra + bonus := []*TagInfo{ {tag: "Freeleech", title: "Freeleech", regexp: "freeleech", re: nil}, } @@ -234,6 +294,11 @@ func (info *TagInfo) Regexp() string { return info.regexp } +// FindMatch returns the regexp matches. +func (info *TagInfo) FindMatch(t string) []string { + return info.re.FindStringSubmatch(t) +} + //// Other returns the tag info other. //func (info *TagInfo) Other() string { // return info.other @@ -279,16 +344,21 @@ func Find(infos ...*TagInfo) FindFunc { } type ReleaseTags struct { - Audio []string - Bonus []string - Channels string - Codec string - Container string - HDR []string - Origin string - Other []string - Resolution string - Source string + Audio []string + AudioBitrate string + AudioFormat string + LogScore int + HasLog bool + HasCue bool + Bonus []string + Channels string + Codec string + Container string + HDR []string + Origin string + Other []string + Resolution string + Source string } func ParseReleaseTags(tags []string) ReleaseTags { @@ -299,14 +369,32 @@ func ParseReleaseTags(tags []string) ReleaseTags { for tagType, tagInfos := range types { for _, info := range tagInfos { + + if info.Tag() == "LogScore" { + m := info.FindMatch(tag) + if len(m) == 3 { + score, err := strconv.Atoi(m[2]) + if err != nil { + // handle error + } + releaseTags.LogScore = score + } + continue + } + // check tag match := info.Match(tag) if match { - fmt.Printf("match: %v, info: %v\n", tag, info.Tag()) switch tagType { case "audio": releaseTags.Audio = append(releaseTags.Audio, info.Tag()) continue + case "audioBitrate": + releaseTags.AudioBitrate = info.Tag() + continue + case "audioFormat": + releaseTags.AudioFormat = info.Tag() + continue case "bonus": releaseTags.Bonus = append(releaseTags.Bonus, info.Tag()) continue @@ -347,7 +435,6 @@ func ParseReleaseTagString(tags string) ReleaseTags { releaseTags := ReleaseTags{} for tagType, tagInfos := range types { - //fmt.Printf("tagType: %v\n", tagType) for _, info := range tagInfos { // check tag @@ -356,10 +443,33 @@ func ParseReleaseTagString(tags string) ReleaseTags { continue } - //fmt.Printf("match: info: %v\n", info.Tag()) + if info.Tag() == "LogScore" { + m := info.FindMatch(tags) + if len(m) == 2 { + score, err := strconv.Atoi(m[1]) + if err != nil { + // handle error + } + releaseTags.HasLog = true + releaseTags.LogScore = score + + releaseTags.Audio = append(releaseTags.Audio, fmt.Sprintf("Log%d", score)) + } + continue + } + switch tagType { case "audio": releaseTags.Audio = append(releaseTags.Audio, info.Tag()) + if info.Tag() == "Cue" { + releaseTags.HasCue = true + } + continue + case "audioBitrate": + releaseTags.AudioBitrate = info.Tag() + continue + case "audioFormat": + releaseTags.AudioFormat = info.Tag() continue case "bonus": releaseTags.Bonus = append(releaseTags.Bonus, info.Tag()) @@ -396,3 +506,10 @@ func ParseReleaseTagString(tags string) ReleaseTags { return releaseTags } + +var tagsDelimiterRegexp = regexp.MustCompile(`\s*[|/,]\s*`) + +// CleanReleaseTags trim delimiters and closest space +func CleanReleaseTags(tagString string) string { + return tagsDelimiterRegexp.ReplaceAllString(tagString, " ") +} diff --git a/internal/domain/releasetags_test.go b/internal/domain/releasetags_test.go index 767bdb8..2cfecc1 100644 --- a/internal/domain/releasetags_test.go +++ b/internal/domain/releasetags_test.go @@ -18,7 +18,7 @@ func TestParseReleaseTags(t *testing.T) { args args want ReleaseTags }{ - {name: "test_1", args: args{tags: []string{"CD", "FLAC", "Lossless"}}, want: ReleaseTags{Source: "CD", Audio: []string{"FLAC", "Lossless"}}}, + {name: "test_1", args: args{tags: []string{"CD", "FLAC", "Lossless"}}, want: ReleaseTags{Audio: []string{"FLAC", "Lossless"}, AudioBitrate: "Lossless", AudioFormat: "FLAC", Source: "CD"}}, {name: "test_2", args: args{tags: []string{"MP4", "2160p", "BluRay", "DV"}}, want: ReleaseTags{Source: "BluRay", Resolution: "2160p", Container: "mp4", HDR: []string{"DV"}}}, } for _, tt := range tests { @@ -37,12 +37,12 @@ func TestParseReleaseTagString(t *testing.T) { args args want ReleaseTags }{ - {name: "music_1", args: args{tags: "FLAC / Lossless / Log / 80% / Cue / CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log"}, Source: "CD"}}, - {name: "music_2", args: args{tags: "FLAC Lossless Log 80% Cue CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log"}, Source: "CD"}}, - {name: "music_3", args: args{tags: "FLAC Lossless Log 100% Cue CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log100", "Log"}, Source: "CD"}}, - {name: "music_4", args: args{tags: "FLAC 24bit Lossless Log 100% Cue CD"}, want: ReleaseTags{Audio: []string{"24BIT Lossless", "Cue", "FLAC", "Log100", "Log"}, Source: "CD"}}, - {name: "music_5", args: args{tags: "MP3 320 WEB"}, want: ReleaseTags{Audio: []string{"320", "MP3"}, Source: "WEB"}}, - {name: "music_6", args: args{tags: "FLAC Lossless Log (100%) Cue CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log100", "Log"}, Source: "CD"}}, + {name: "music_1", args: args{tags: "FLAC / Lossless / Log / 80% / Cue / CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log"}, AudioBitrate: "Lossless", AudioFormat: "FLAC", Source: "CD", HasCue: true}}, + {name: "music_2", args: args{tags: "FLAC Lossless Log 80% Cue CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log80", "Log"}, AudioBitrate: "Lossless", AudioFormat: "FLAC", Source: "CD", HasLog: true, LogScore: 80, HasCue: true}}, + {name: "music_3", args: args{tags: "FLAC Lossless Log 100% Cue CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log100", "Log"}, AudioBitrate: "Lossless", AudioFormat: "FLAC", Source: "CD", HasLog: true, LogScore: 100, HasCue: true}}, + {name: "music_4", args: args{tags: "FLAC 24bit Lossless Log 100% Cue CD"}, want: ReleaseTags{Audio: []string{"24BIT Lossless", "Cue", "FLAC", "Log100", "Log"}, AudioBitrate: "24BIT Lossless", AudioFormat: "FLAC", Source: "CD", HasLog: true, LogScore: 100, HasCue: true}}, + {name: "music_5", args: args{tags: "MP3 320 WEB"}, want: ReleaseTags{Audio: []string{"320", "MP3"}, AudioBitrate: "320", AudioFormat: "MP3", Source: "WEB"}}, + {name: "music_6", args: args{tags: "FLAC Lossless Log (100%) Cue CD"}, want: ReleaseTags{Audio: []string{"Cue", "FLAC", "Lossless", "Log100", "Log"}, AudioBitrate: "Lossless", AudioFormat: "FLAC", Source: "CD", HasCue: true, HasLog: true, LogScore: 100}}, {name: "movies_1", args: args{tags: "x264 Blu-ray MKV 1080p"}, want: ReleaseTags{Codec: "x264", Source: "BluRay", Resolution: "1080p", Container: "mkv"}}, {name: "movies_2", args: args{tags: "HEVC HDR Blu-ray mp4 2160p"}, want: ReleaseTags{Codec: "HEVC", Source: "BluRay", Resolution: "2160p", Container: "mp4", HDR: []string{"HDR"}}}, {name: "movies_3", args: args{tags: "HEVC HDR DV Blu-ray mp4 2160p"}, want: ReleaseTags{Codec: "HEVC", Source: "BluRay", Resolution: "2160p", Container: "mp4", HDR: []string{"HDR", "DV"}}}, @@ -51,9 +51,8 @@ func TestParseReleaseTagString(t *testing.T) { {name: "movies_6", args: args{tags: "H.264, DVD"}, want: ReleaseTags{Codec: "H.264", Source: "DVD"}}, {name: "movies_7", args: args{tags: "H.264, DVD, Freeleech"}, want: ReleaseTags{Codec: "H.264", Source: "DVD", Bonus: []string{"Freeleech"}}}, {name: "movies_8", args: args{tags: "H.264, DVD, Freeleech!"}, want: ReleaseTags{Codec: "H.264", Source: "DVD", Bonus: []string{"Freeleech"}}}, - {name: "anime_1", args: args{tags: "Web / MKV / h264 / 1080p / AAC 2.0 / Softsubs (SubsPlease) / Episode 22 / Freeleech"}, want: ReleaseTags{Audio: []string{"AAC"}, Channels: "2.0", Source: "WEB", Resolution: "1080p", Container: "mkv", Codec: "H.264", Bonus: []string{"Freeleech"}}}, - {name: "anime_2", args: args{tags: "Web | ISO | h264 | 1080p | AAC 2.0 | Softsubs (SubsPlease) | Episode 22 | Freeleech"}, want: ReleaseTags{Audio: []string{"AAC"}, Channels: "2.0", Source: "WEB", Resolution: "1080p", Container: "iso", Codec: "H.264", Bonus: []string{"Freeleech"}}}, - {name: "tv_1", args: args{tags: "MKV | H.264 | WEB-DL | 1080p | Internal | FastTorrent"}, want: ReleaseTags{Source: "WEB-DL", Codec: "H.264", Resolution: "1080p", Container: "mkv", Origin: "Internal"}}, + {name: "anime_1", args: args{tags: "Web / MKV / h264 / 1080p / AAC 2.0 / Softsubs (SubsPlease) / Episode 22 / Freeleech"}, want: ReleaseTags{Audio: []string{"AAC"}, AudioBitrate: "", AudioFormat: "AAC", Bonus: []string{"Freeleech"}, Channels: "2.0", Codec: "H.264", Container: "mkv", Resolution: "1080p", Source: "WEB"}}, + {name: "anime_2", args: args{tags: "Web | ISO | h264 | 1080p | AAC 2.0 | Softsubs (SubsPlease) | Episode 22 | Freeleech"}, want: ReleaseTags{Audio: []string{"AAC"}, AudioBitrate: "", AudioFormat: "AAC", Bonus: []string{"Freeleech"}, Channels: "2.0", Codec: "H.264", Container: "iso", Resolution: "1080p", Source: "WEB"}}, {name: "tv_1", args: args{tags: "MKV | H.264 | WEB-DL | 1080p | Internal | FastTorrent"}, want: ReleaseTags{Source: "WEB-DL", Codec: "H.264", Resolution: "1080p", Container: "mkv", Origin: "Internal"}}, {name: "tv_2", args: args{tags: "MKV | H.264 | WEB-DL | 1080p | Scene | FastTorrent"}, want: ReleaseTags{Source: "WEB-DL", Codec: "H.264", Resolution: "1080p", Container: "mkv", Origin: "Scene"}}, {name: "tv_3", args: args{tags: "MKV | H.264 | WEB-DL | 1080p | P2P | FastTorrent"}, want: ReleaseTags{Source: "WEB-DL", Codec: "H.264", Resolution: "1080p", Container: "mkv", Origin: "P2P"}}, } @@ -63,3 +62,23 @@ func TestParseReleaseTagString(t *testing.T) { }) } } + +func Test_cleanReleaseTags(t *testing.T) { + type args struct { + tagString string + } + tests := []struct { + name string + args args + want string + }{ + {name: "1", args: args{tagString: "FLAC / Lossless / Log / 100% / Cue / CD"}, want: "FLAC Lossless Log 100% Cue CD"}, + {name: "2", args: args{tagString: "FLAC/Lossless/Log 100%/Cue/CD"}, want: "FLAC Lossless Log 100% Cue CD"}, + {name: "3", args: args{tagString: "FLAC | Lossless | Log | 100% | Cue | CD"}, want: "FLAC Lossless Log 100% Cue CD"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, CleanReleaseTags(tt.args.tagString), "cleanReleaseTags(%v)", tt.args.tagString) + }) + } +} diff --git a/internal/feed/rss_test.go b/internal/feed/rss_test.go index 73141e4..3b6c358 100644 --- a/internal/feed/rss_test.go +++ b/internal/feed/rss_test.go @@ -68,7 +68,7 @@ func TestRSSJob_processItem(t *testing.T) { Link: "/details.php?id=00000&hit=1", GUID: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", }}, - want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", DownloadURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 1490000000, Title: "Some Release Title", Description: "Category: Example\n Size: 1.49 GB\n Status: 27 seeders and 1 leechers\n Speed: 772.16 kB/s\n Added: 2022-09-29 16:06:08\n", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: nil, Proper: false, Repack: false, Website: "", Artists: "", Type: "", LogScore: 0, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, + want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", DownloadURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 1490000000, Title: "Some Release Title", Description: "Category: Example\n Size: 1.49 GB\n Status: 27 seeders and 1 leechers\n Speed: 772.16 kB/s\n Added: 2022-09-29 16:06:08\n", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: nil, Proper: false, Repack: false, Website: "", Artists: "", Type: "episode", LogScore: 0, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, }, { name: "with_baseurl", @@ -97,7 +97,7 @@ func TestRSSJob_processItem(t *testing.T) { Link: "https://fake-feed.com/details.php?id=00000&hit=1", GUID: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", }}, - want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", DownloadURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 1490000000, Title: "Some Release Title", Description: "Category: Example\n Size: 1.49 GB\n Status: 27 seeders and 1 leechers\n Speed: 772.16 kB/s\n Added: 2022-09-29 16:06:08\n", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: nil, Proper: false, Repack: false, Website: "", Artists: "", Type: "", LogScore: 0, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, + want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", DownloadURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 1490000000, Title: "Some Release Title", Description: "Category: Example\n Size: 1.49 GB\n Status: 27 seeders and 1 leechers\n Speed: 772.16 kB/s\n Added: 2022-09-29 16:06:08\n", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: nil, Proper: false, Repack: false, Website: "", Artists: "", Type: "episode", LogScore: 0, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, }, { name: "time_parse", @@ -127,7 +127,7 @@ func TestRSSJob_processItem(t *testing.T) { GUID: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", //PublishedParsed: &nowMinusTime, }}, - want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", DownloadURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 1490000000, Title: "Some Release Title", Description: "Category: Example\n Size: 1.49 GB\n Status: 27 seeders and 1 leechers\n Speed: 772.16 kB/s\n Added: 2022-09-29 16:06:08\n", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: nil, Proper: false, Repack: false, Website: "", Artists: "", Type: "", LogScore: 0, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, + want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", DownloadURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 1490000000, Title: "Some Release Title", Description: "Category: Example\n Size: 1.49 GB\n Status: 27 seeders and 1 leechers\n Speed: 772.16 kB/s\n Added: 2022-09-29 16:06:08\n", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: nil, Proper: false, Repack: false, Website: "", Artists: "", Type: "episode", LogScore: 0, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, }, { name: "time_parse", diff --git a/internal/indexer/definitions/orpheus.yaml b/internal/indexer/definitions/orpheus.yaml index b62f41f..a2bf243 100644 --- a/internal/indexer/definitions/orpheus.yaml +++ b/internal/indexer/definitions/orpheus.yaml @@ -80,6 +80,7 @@ irc: - line: 'TORRENT: Dirty Dike – Bogies & Alcohol – [2008] [Album] CD/MP3/320 – hip.hop,uk.hip.hop,united.kingdom – https://orpheus.network/torrents.php?id=0000000 – https://orpheus.network/torrents.php?id=0000000&torrentid=0000000&action=download' expect: torrentName: Dirty Dike – Bogies & Alcohol – [2008] [Album] CD/MP3/320 + title: Dirty Dike – Bogies & Alcohol year: "2008" category: Album releaseTags: CD/MP3/320 @@ -89,6 +90,7 @@ irc: - line: 'TORRENT: Various Artists – Bicycle Day: 85 Yrs of LSD Special – [2023] [Compilation] WEB/FLAC/Lossless – ambient,electronic – https://orpheus.network/torrents.php?id=0000000 – https://orpheus.network/torrents.php?id=0000000&torrentid=0000000&action=download' expect: torrentName: 'Various Artists – Bicycle Day: 85 Yrs of LSD Special – [2023] [Compilation] WEB/FLAC/Lossless' + title: 'Various Artists – Bicycle Day: 85 Yrs of LSD Special' year: "2023" category: Compilation releaseTags: WEB/FLAC/Lossless @@ -98,15 +100,17 @@ irc: - line: 'TORRENT: Snoop Dogg – Untitled – [2001] [Sampler] Vinyl/MP3/320 – – https://orpheus.network/torrents.php?id=0000000 – https://orpheus.network/torrents.php?id=0000000&torrentid=0000000&action=download' expect: torrentName: Snoop Dogg – Untitled – [2001] [Sampler] Vinyl/MP3/320 + title: Snoop Dogg – Untitled year: "2001" category: Sampler releaseTags: Vinyl/MP3/320 tags: "" baseUrl: https://orpheus.network/ torrentId: "0000000" - pattern: 'TORRENT: (.* . \[(.*?)\] \[(.*?)\] (.*)) . \s*(.*) . https?:\/\/.* . (https?:\/\/.*\/).*torrentid=(\d+).*' + pattern: 'TORRENT: ((.*) . \[(.*?)\] \[(.*?)\] (.*)) . \s*(.*) . https?:\/\/.* . (https?:\/\/.*\/).*torrentid=(\d+).*' vars: - torrentName + - title - year - category - releaseTags diff --git a/internal/indexer/definitions/red.yaml b/internal/indexer/definitions/red.yaml index f366297..3c29c4f 100644 --- a/internal/indexer/definitions/red.yaml +++ b/internal/indexer/definitions/red.yaml @@ -86,6 +86,7 @@ irc: - line: Artist - Albumname [2008] [Single] - FLAC / Lossless / Log / 100% / Cue / CD - https://redacted.ch/torrents.php?id=0000000 / https://redacted.ch/torrents.php?action=download&id=0000000 - hip.hop,rhythm.and.blues,2000s expect: torrentName: Artist - Albumname [2008] [Single] - FLAC / Lossless / Log / 100% / Cue / CD + title: Artist - Albumname year: "2008" category: Single releaseTags: FLAC / Lossless / Log / 100% / Cue / CD @@ -96,6 +97,7 @@ irc: - line: A really long name here - Concertos 5 and 6, Suite No 2 [1991] [Album] - FLAC / Lossless / Log / 100% / Cue / CD - https://redacted.ch/torrents.php?id=0000000 / https://redacted.ch/torrents.php?action=download&id=0000000 - classical expect: torrentName: A really long name here - Concertos 5 and 6, Suite No 2 [1991] [Album] - FLAC / Lossless / Log / 100% / Cue / CD + title: A really long name here - Concertos 5 and 6, Suite No 2 year: "1991" category: Album releaseTags: FLAC / Lossless / Log / 100% / Cue / CD @@ -103,9 +105,10 @@ irc: baseUrl: https://redacted.ch/ torrentId: "0000000" tags: classical - pattern: '(.* (?:\[(.*)\] \[(.*)\] - (.*))?) - .*id=(.*) \/ (https?://.+/).+id=(\d+)[ -]*(.*)' + pattern: '((.*) (?:\[(.*)\] \[(.*)\] - (.*))?) - .*id=(.*) \/ (https?://.+/).+id=(\d+)[ -]*(.*)' vars: - torrentName + - title - year - category - releaseTags diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go new file mode 100644 index 0000000..a94540b --- /dev/null +++ b/internal/indexer/indexer_test.go @@ -0,0 +1,362 @@ +// Copyright (c) 2021 - 2024, Ludvig Lundgren and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +package indexer + +import ( + "io" + "testing" + + "github.com/autobrr/autobrr/internal/domain" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" +) + +func TestIndexersParseAndFilter(t *testing.T) { + type fields struct { + identifier string + settings map[string]string + } + type filterTest struct { + filter *domain.Filter + match bool + rejections []string + } + type args struct { + announceLines []string + filters []filterTest + } + type subTest struct { + name string + args args + match bool + } + tests := []struct { + name string + fields fields + match bool + subTests []subTest + }{ + { + name: "ops", + fields: fields{ + identifier: "orpheus", + settings: map[string]string{ + "torrent_pass": "pass", + "api_key": "key", + }, + }, + subTests: []subTest{ + { + name: "announce_1", + args: args{ + announceLines: []string{"TORRENT: Dirty Dike – Bogies & Alcohol – [2008] [Album] CD/MP3/320 – hip.hop,uk.hip.hop,united.kingdom – https://orpheus.network/torrents.php?id=0000000 – https://orpheus.network/torrents.php?id=0000000&torrentid=0000000&action=download"}, + filters: []filterTest{ + { + filter: &domain.Filter{ + Name: "filter_1", + MatchCategories: "Album", + Years: "2008", + }, + match: true, + }, + { + filter: &domain.Filter{ + Name: "filter_2", + MatchCategories: "Single", + Years: "2008", + }, + match: false, + rejections: []string{"category not matching. got: Album want: Single"}, + }, + }, + }, + match: false, + }, + { + name: "announce_2", + args: args{ + announceLines: []string{"TORRENT: Dirty Dike – Bogies & Alcohol – [2024] [EP] CD/FLAC/Lossless – hip.hop,uk.hip.hop,united.kingdom – https://orpheus.network/torrents.php?id=0000000 – https://orpheus.network/torrents.php?id=0000000&torrentid=0000000&action=download"}, + filters: []filterTest{ + { + filter: &domain.Filter{ + Name: "filter_1", + MatchCategories: "EP,Album", + Years: "2024", + Quality: []string{"Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + }, + match: true, + }, + { + filter: &domain.Filter{ + Name: "filter_2", + MatchCategories: "EP,Album", + Years: "2024", + Quality: []string{"24bit Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + }, + match: false, + rejections: []string{"quality not matching. got: [FLAC Lossless] want: [24bit Lossless]"}, + }, + }, + }, + match: false, + }, + }, + match: true, + }, + { + name: "redacted", + fields: fields{ + identifier: "red", + settings: map[string]string{ + "authkey": "key", + "torrent_pass": "pass", + "api_key": "key", + }, + }, + subTests: []subTest{ + { + name: "announce_1", + args: args{ + announceLines: []string{"Artist - Albumname [2008] [Single] - FLAC / Lossless / Log / 100% / Cue / CD - https://redacted.ch/torrents.php?id=0000000 / https://redacted.ch/torrents.php?action=download&id=0000000 - hip.hop,rhythm.and.blues,2000s"}, + filters: []filterTest{ + { + filter: &domain.Filter{ + Name: "filter_1", + MatchCategories: "Single", + Years: "2008", + }, + match: true, + }, + { + filter: &domain.Filter{ + Name: "filter_2", + MatchCategories: "Album", + }, + match: false, + rejections: []string{"category not matching. got: Album want: Single"}, + }, + }, + }, + match: false, + }, + { + name: "announce_2", + args: args{ + announceLines: []string{"A really long name here - Concertos 5 and 6, Suite No 2 [1991] [Album] - FLAC / Lossless / Log / 100% / Cue / CD - https://redacted.ch/torrents.php?id=0000000 / https://redacted.ch/torrents.php?action=download&id=0000000 - classical"}, + filters: []filterTest{ + { + filter: &domain.Filter{ + Name: "filter_1", + MatchCategories: "EP,Album", + Years: "1991", + PerfectFlac: true, + //Quality: []string{"Lossless"}, + //Sources: []string{"CD"}, + //Formats: []string{"FLAC"}, + Tags: "classical", + }, + match: true, + }, + { + filter: &domain.Filter{ + Name: "filter_2", + MatchCategories: "EP,Album", + Years: "2024", + Quality: []string{"24bit Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + }, + match: false, + rejections: []string{"year not matching. got: 1991 want: 2024", "quality not matching. got: [Cue FLAC Lossless Log100 Log] want: [24bit Lossless]"}, + }, + }, + }, + match: false, + }, + { + name: "announce_3", + args: args{ + announceLines: []string{"The best artist - Album No 2 [2024] [EP] - FLAC / Lossless / Log / 100% / Cue / CD - https://redacted.ch/torrents.php?id=0000000 / https://redacted.ch/torrents.php?action=download&id=0000000 - classical"}, + filters: []filterTest{ + { + filter: &domain.Filter{ + Name: "filter_1", + MatchCategories: "EP", + Years: "2024", + Quality: []string{"Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + Log: true, + LogScore: 100, + Cue: true, + }, + match: true, + }, + { + filter: &domain.Filter{ + Name: "filter_2", + MatchCategories: "EP,Album", + Years: "2024", + Quality: []string{"24bit Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + }, + match: false, + rejections: []string{"quality not matching. got: [FLAC Lossless] want: [24bit Lossless]"}, + }, + }, + }, + match: false, + }, + { + name: "announce_4", + args: args{ + announceLines: []string{"The best artist - Album No 2 [2024] [EP] - FLAC / Lossless / Log / 100% / Cue / CD - https://redacted.ch/torrents.php?id=0000000 / https://redacted.ch/torrents.php?action=download&id=0000000 - classical"}, + filters: []filterTest{ + { + filter: &domain.Filter{ + Name: "filter_1", + MatchCategories: "EP", + Years: "2024", + Quality: []string{"Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + Log: true, + LogScore: 100, + Cue: true, + }, + match: true, + }, + { + filter: &domain.Filter{ + Name: "filter_2", + MatchCategories: "EP,Album", + Years: "2024", + Quality: []string{"24bit Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + }, + match: false, + rejections: []string{"quality not matching. got: [FLAC Lossless] want: [24bit Lossless]"}, + }, + }, + }, + match: false, + }, + { + name: "announce_5", + args: args{ + announceLines: []string{"The best artist - Album No 1 [2024] [EP] - FLAC / Lossless / Log / 87% / Cue / CD - https://redacted.ch/torrents.php?id=0000000 / https://redacted.ch/torrents.php?action=download&id=0000000 - classical"}, + filters: []filterTest{ + { + filter: &domain.Filter{ + Name: "filter_1", + MatchCategories: "EP", + Years: "2024", + Quality: []string{"Lossless"}, + Sources: []string{"CD"}, + Formats: []string{"FLAC"}, + Log: true, + LogScore: 100, + Cue: true, + }, + match: false, + rejections: []string{"log score. got: 87 want: 100"}, + }, + { + filter: &domain.Filter{ + Name: "filter_2", + MatchCategories: "EP", + PerfectFlac: true, + }, + match: false, + rejections: []string{"wanted: perfect flac. got: [Cue FLAC Lossless Log87 Log]"}, + }, + }, + }, + match: false, + }, + }, + match: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + //l := zerolog.New(io.Discard) + //l := logger.Mock() + + i, err := OpenAndProcessDefinition("./definitions/" + tt.fields.identifier + ".yaml") + assert.NoError(t, err) + + i.SettingsMap = tt.fields.settings + + ll := zerolog.New(io.Discard) + + // indexer subtests + for _, subT := range tt.subTests { + t.Run(subT.name, func(t *testing.T) { + + // from announce/announce.go + tmpVars := map[string]string{} + parseFailed := false + + for idx, parseLine := range i.IRC.Parse.Lines { + match, err := ParseLine(&ll, parseLine.Pattern, parseLine.Vars, tmpVars, subT.args.announceLines[idx], parseLine.Ignore) + if err != nil { + parseFailed = true + break + } + + if !match { + parseFailed = true + break + } + } + + if parseFailed { + return + } + + rls := domain.NewRelease(i.Identifier) + rls.Protocol = domain.ReleaseProtocol(i.Protocol) + + // on lines matched + err = i.IRC.Parse.Parse(i, tmpVars, rls) + assert.NoError(t, err) + + // release/service.go + + //ctx := context.Background() + //filterSvc := filter.NewService(l, nil, nil, nil, nil, nil) + + for _, filterT := range subT.args.filters { + t.Run(filterT.filter.Name, func(t *testing.T) { + filter := filterT.filter + + //l := s.log.With().Str("indexer", release.Indexer).Str("filter", filter.Name).Str("release", release.TorrentName).Logger() + + // save filter on release + rls.Filter = filter + rls.FilterName = filter.Name + rls.FilterID = filter.ID + + // test filter + //match, err := filterSvc.CheckFilter(ctx, filter, rls) + + rejections, matchedFilter := filter.CheckFilter(rls) + assert.Len(t, rejections, len(filterT.rejections)) + assert.Equal(t, filterT.match, matchedFilter) + }) + } + }) + } + }) + } +} diff --git a/internal/indexer/service.go b/internal/indexer/service.go index feedd90..1faa752 100644 --- a/internal/indexer/service.go +++ b/internal/indexer/service.go @@ -516,6 +516,46 @@ func (s *service) LoadIndexerDefinitions() error { return nil } +var ErrIndexerDefinitionDeprecated = errors.New("DEPRECATED: indexer definition version") + +func isValidExtension(ext string) bool { + return ext == ".yaml" || ext == ".yml" +} + +func OpenAndProcessDefinition(file string) (*domain.IndexerDefinition, error) { + f, err := os.Open(file) + if err != nil { + return nil, errors.Wrap(err, "could not open file: %s", file) + } + defer f.Close() + + var d *domain.IndexerDefinitionCustom + + dec := yaml.NewDecoder(f) + dec.KnownFields(false) + + if err = dec.Decode(&d); err != nil { + return nil, errors.Wrap(err, "could not decode definition file: %s", file) + } + + if d == nil { + //s.log.Warn().Msgf("skipping empty file: %s", file) + return nil, errors.New("empty definition file") + } + + if d.Implementation == "" { + d.Implementation = "irc" + } + + //if d.Implementation == "irc" && d.IRC != nil { + // if d.IRC.Parse == nil { + // s.log.Warn().Msgf("DEPRECATED: indexer definition version: %s", file) + // } + //} + + return d.ToIndexerDefinition(), nil +} + // LoadCustomIndexerDefinitions load definitions from custom path func (s *service) LoadCustomIndexerDefinitions() error { if s.config.CustomDefinitions == "" { @@ -539,51 +579,23 @@ func (s *service) LoadCustomIndexerDefinitions() error { customCount := 0 for _, f := range entries { - fileExtension := filepath.Ext(f.Name()) - if fileExtension != ".yaml" && fileExtension != ".yml" { - s.log.Warn().Stack().Msgf("skipping unknown extension definition file: %s", f.Name()) + ext := filepath.Ext(f.Name()) + if !isValidExtension(ext) { + s.log.Warn().Msgf("unsupported extension %s, definition file: %s", ext, f.Name()) continue } file := filepath.Join(s.config.CustomDefinitions, f.Name()) - s.log.Trace().Msgf("parsing custom: %s", file) + s.log.Trace().Msgf("parsing custom definition: %s", file) - data, err := os.ReadFile(file) + definition, err := OpenAndProcessDefinition(file) if err != nil { - s.log.Error().Stack().Err(err).Msgf("failed reading file: %s", file) - return errors.Wrap(err, "could not read file: %s", file) - } - - var d *domain.IndexerDefinitionCustom - dec := yaml.NewDecoder(bytes.NewReader(data)) - // Do _not_ fail on unknown fields while parsing custom indexer - // definitions for better backwards compatibility. See discussion: - // https://github.com/autobrr/autobrr/pull/1257#issuecomment-1813821391 - dec.KnownFields(false) - - if err = dec.Decode(&d); err != nil { - s.log.Error().Stack().Err(err).Msgf("failed unmarshal file: %s", file) - return errors.Wrap(err, "could not unmarshal file: %s", file) - } - - if d == nil { - s.log.Warn().Msgf("skipping empty file: %s", file) + s.log.Error().Err(err).Msgf("could not open definition file: %s", file) continue } - if d.Implementation == "" { - d.Implementation = "irc" - } - - // to prevent crashing from non-updated definitions lets skip - if d.Implementation == "irc" && d.IRC != nil { - if d.IRC.Parse == nil { - s.log.Warn().Msgf("DEPRECATED: indexer definition version: %s", file) - } - } - - s.definitions[d.Identifier] = *d.ToIndexerDefinition() + s.definitions[definition.Identifier] = *definition customCount++ }