idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 08:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(indexers): add support for optional baseurl override (#551)

Browse source 
* feat(indexers): optional baseurl override

* feat(indexers): update baseUrl parsing

* refactor(indexers): BREAKING move parse to IRC struct

* Move Parse as part of IRC struct from Indexer
* Updated definitions
* Build torrentUrl in stages
* Use new url.JoinPath to build torrentUrl
* Update tests

* refactor(indexers): select option obj

* refactor(indexers): make backwards compatible
This commit is contained in:
ze0s 2022-12-03 15:40:45 +01:00 • committed by GitHub
parent 301180e55b
commit 25a165b764
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
66 changed files with 1533 additions and 1211 deletions
Download patch file Download diff file Expand all files Collapse all files

139
internal/domain/indexer.go
View file

@ -3,10 +3,11 @@ package domain
import (
"bytes"
"context"
"errors"
"net/url"
"text/template"
"github.com/autobrr/autobrr/pkg/errors"
"github.com/Masterminds/sprig/v3"
"github.com/dustin/go-humanize"
)
@ -26,6 +27,7 @@ type Indexer struct {
Identifier string `json:"identifier"`
Enabled bool `json:"enabled"`
Implementation string `json:"implementation"`
BaseURL string `json:"base_url,omitempty"`
Settings map[string]string `json:"settings,omitempty"`
}
@ -34,6 +36,7 @@ type IndexerDefinition struct {
Name string `json:"name"`
Identifier string `json:"identifier"`
Implementation string `json:"implementation"`
BaseURL string `json:"base_url,omitempty"`
Enabled bool `json:"enabled,omitempty"`
Description string `json:"description"`
Language string `json:"language"`
@ -46,7 +49,6 @@ type IndexerDefinition struct {
IRC *IndexerIRC `json:"irc,omitempty"`
Torznab *Torznab `json:"torznab,omitempty"`
RSS *FeedSettings `json:"rss,omitempty"`
Parse *IndexerParse `json:"parse,omitempty"`
}
func (i IndexerDefinition) HasApi() bool {
@ -58,6 +60,55 @@ func (i IndexerDefinition) HasApi() bool {
return false
}
type IndexerDefinitionCustom struct {
ID int `json:"id,omitempty"`
Name string `json:"name"`
Identifier string `json:"identifier"`
Implementation string `json:"implementation"`
BaseURL string `json:"base_url,omitempty"`
Enabled bool `json:"enabled,omitempty"`
Description string `json:"description"`
Language string `json:"language"`
Privacy string `json:"privacy"`
Protocol string `json:"protocol"`
URLS []string `json:"urls"`
Supports []string `json:"supports"`
Settings []IndexerSetting `json:"settings,omitempty"`
SettingsMap map[string]string `json:"-"`
IRC *IndexerIRC `json:"irc,omitempty"`
Torznab *Torznab `json:"torznab,omitempty"`
RSS *FeedSettings `json:"rss,omitempty"`
Parse *IndexerIRCParse `json:"parse,omitempty"`
}
func (i *IndexerDefinitionCustom) ToIndexerDefinition() *IndexerDefinition {
d := &IndexerDefinition{
ID: i.ID,
Name: i.Name,
Identifier: i.Identifier,
Implementation: i.Implementation,
BaseURL: i.BaseURL,
Enabled: i.Enabled,
Description: i.Description,
Language: i.Language,
Privacy: i.Privacy,
Protocol: i.Protocol,
URLS: i.URLS,
Supports: i.Supports,
Settings: i.Settings,
SettingsMap: i.SettingsMap,
IRC: i.IRC,
Torznab: i.Torznab,
RSS: i.RSS,
}
if i.IRC != nil && i.Parse != nil {
i.IRC.Parse = i.Parse
}
return d
}
type IndexerSetting struct {
Name string `json:"name"`
Required bool `json:"required,omitempty"`
@ -89,6 +140,7 @@ type IndexerIRC struct {
Announcers []string `json:"announcers"`
SettingsMap map[string]string `json:"-"`
Settings []IndexerSetting `json:"settings"`
Parse *IndexerIRCParse `json:"parse,omitempty"`
}
func (i IndexerIRC) ValidAnnouncer(announcer string) bool {
@ -109,48 +161,39 @@ func (i IndexerIRC) ValidChannel(channel string) bool {
return false
}
type IndexerParse struct {
type IndexerIRCParse struct {
Type string `json:"type"`
ForceSizeUnit string `json:"forcesizeunit"`
Lines []IndexerParseExtract `json:"lines"`
Match IndexerParseMatch `json:"match"`
Lines []IndexerIRCParseLine `json:"lines"`
Match IndexerIRCParseMatch `json:"match"`
}
type IndexerParseExtract struct {
type IndexerIRCParseLine struct {
Test []string `json:"test"`
Pattern string `json:"pattern"`
Vars []string `json:"vars"`
}
type IndexerParseMatch struct {
type IndexerIRCParseMatch struct {
TorrentURL string `json:"torrenturl"`
TorrentName string `json:"torrentname"`
Encode []string `json:"encode"`
}
func (p *IndexerParse) ParseMatch(vars map[string]string, extraVars map[string]string, release *Release) error {
tmpVars := map[string]string{}
type IndexerIRCParseMatched struct {
TorrentURL string
TorrentName string
}
// copy vars to new tmp map
for k, v := range vars {
tmpVars[k] = v
}
// merge extra vars with vars
if extraVars != nil {
for k, v := range extraVars {
tmpVars[k] = v
}
}
func (p *IndexerIRCParse) ParseMatch(baseURL string, vars map[string]string) (*IndexerIRCParseMatched, error) {
matched := &IndexerIRCParseMatched{}
// handle url encode of values
if p.Match.Encode != nil {
for _, e := range p.Match.Encode {
if v, ok := tmpVars[e]; ok {
// url encode value
t := url.QueryEscape(v)
tmpVars[e] = t
}
for _, e := range p.Match.Encode {
if v, ok := vars[e]; ok {
// url encode value
t := url.QueryEscape(v)
vars[e] = t
}
}
@ -158,38 +201,52 @@ func (p *IndexerParse) ParseMatch(vars map[string]string, extraVars map[string]s
// setup text template to inject variables into
tmpl, err := template.New("torrenturl").Funcs(sprig.TxtFuncMap()).Parse(p.Match.TorrentURL)
if err != nil {
return errors.New("could not create torrent url template")
return nil, errors.New("could not create torrent url template")
}
var urlBytes bytes.Buffer
if err := tmpl.Execute(&urlBytes, &tmpVars); err != nil {
return errors.New("could not write torrent url template output")
if err := tmpl.Execute(&urlBytes, &vars); err != nil {
return nil, errors.New("could not write torrent url template output")
}
release.TorrentURL = urlBytes.String()
parsedUrl, err := url.Parse(urlBytes.String())
if err != nil {
return nil, err
}
// for backwards compatibility remove Host and Scheme to rebuild url
if parsedUrl.Host != "" {
parsedUrl.Host = ""
}
if parsedUrl.Scheme != "" {
parsedUrl.Scheme = ""
}
// join baseURL with query
torrentURL, err := url.JoinPath(baseURL, parsedUrl.Path)
if err != nil {
return nil, errors.Wrap(err, "could not join torrent url")
}
matched.TorrentURL = torrentURL
}
if p.Match.TorrentName != "" {
// setup text template to inject variables into
tmplName, err := template.New("torrentname").Funcs(sprig.TxtFuncMap()).Parse(p.Match.TorrentName)
if err != nil {
return err
return nil, err
}
var nameBytes bytes.Buffer
if err := tmplName.Execute(&nameBytes, &tmpVars); err != nil {
return errors.New("could not write torrent name template output")
if err := tmplName.Execute(&nameBytes, &vars); err != nil {
return nil, errors.New("could not write torrent name template output")
}
release.TorrentName = nameBytes.String()
matched.TorrentName = nameBytes.String()
}
// handle cookies
if v, ok := extraVars["cookie"]; ok {
release.RawCookie = v
}
return nil
return matched, nil
}
type TorrentBasic struct {

164
internal/domain/indexer_test.go
View file

@ -6,23 +6,22 @@ import (
"github.com/stretchr/testify/assert"
)
func TestIndexerParse_ParseMatch(t *testing.T) {
func TestIndexerIRCParse_ParseMatch(t *testing.T) {
type fields struct {
Type string
ForceSizeUnit string
Lines []IndexerParseExtract
Match IndexerParseMatch
Lines []IndexerIRCParseLine
Match IndexerIRCParseMatch
}
type args struct {
vars map[string]string
extraVars map[string]string
release *Release
baseURL string
vars map[string]string
}
tests := []struct {
name string
fields fields
args args
expect *Release
want *IndexerIRCParseMatched
wantErr bool
}{
{
@ -30,7 +29,7 @@ func TestIndexerParse_ParseMatch(t *testing.T) {
fields: fields{
Type: "",
ForceSizeUnit: "",
Lines: []IndexerParseExtract{
Lines: []IndexerIRCParseLine{
{
Test: nil,
Pattern: "New Torrent Announcement:\\s*<([^>]*)>\\s*Name:'(.*)' uploaded by '([^']*)'\\s*(freeleech)*\\s*-\\s*(https?\\:\\/\\/[^\\/]+\\/)torrent\\/(\\d+)",
@ -44,39 +43,25 @@ func TestIndexerParse_ParseMatch(t *testing.T) {
},
},
},
Match: IndexerParseMatch{
TorrentURL: "{{ .baseUrl }}rss/download/{{ .torrentId }}/{{ .rsskey }}/{{ .torrentName }}.torrent",
TorrentName: "",
Encode: []string{"torrentName"},
Match: IndexerIRCParseMatch{
TorrentURL: "rss/download/{{ .torrentId }}/{{ .rsskey }}/{{ .torrentName }}.torrent",
Encode: []string{"torrentName"},
},
},
args: args{
baseURL: "https://mock.local/",
vars: map[string]string{
"category": "TV :: Episodes HD",
"torrentName": "The Show 2019 S03E08 2160p DV WEBRip 6CH x265 HEVC-GROUP",
"uploader": "Anonymous",
"freeleech": "",
"baseUrl": "https://mock.org/",
"baseUrl": "https://mock.local/",
"torrentId": "240860011",
},
extraVars: map[string]string{
"rsskey": "00000000000000000000",
},
release: &Release{
Indexer: "mock",
FilterStatus: ReleaseStatusFilterPending,
Rejections: []string{},
Protocol: ReleaseProtocolTorrent,
Implementation: ReleaseImplementationIRC,
"rsskey": "00000000000000000000",
},
},
expect: &Release{
Indexer: "mock",
FilterStatus: ReleaseStatusFilterPending,
Rejections: []string{},
Protocol: ReleaseProtocolTorrent,
Implementation: ReleaseImplementationIRC,
TorrentURL: "https://mock.org/rss/download/240860011/00000000000000000000/The+Show+2019+S03E08+2160p+DV+WEBRip+6CH+x265+HEVC-GROUP.torrent",
want: &IndexerIRCParseMatched{
TorrentURL: "https://mock.local/rss/download/240860011/00000000000000000000/The+Show+2019+S03E08+2160p+DV+WEBRip+6CH+x265+HEVC-GROUP.torrent",
},
wantErr: false,
},
@ -85,7 +70,7 @@ func TestIndexerParse_ParseMatch(t *testing.T) {
fields: fields{
Type: "",
ForceSizeUnit: "",
Lines: []IndexerParseExtract{
Lines: []IndexerIRCParseLine{
{
Test: nil,
Pattern: `(.*?)(?: - )?(Visual Novel|Light Novel|TV.*|Movie|Manga|OVA|ONA|DVD Special|BD Special|Oneshot|Anthology|Manhwa|Manhua|Artbook|Game|Live Action.*|)[\s\p{Zs}]{2,}\[(\d+)\] :: (.*?)(?: \/ (?:RAW|Softsubs|Hardsubs|Translated)\s\((.+)\)(?:.*Episode\s(\d+))?(?:.*(Freeleech))?.*)? \|\| (https.*)\/torrents.*\?id=\d+&torrentid=(\d+) \|\| (.+?(?:(?:\|\| Uploaded by|$))?) (?:\|\| Uploaded by: (.*))?$`,
@ -104,13 +89,14 @@ func TestIndexerParse_ParseMatch(t *testing.T) {
},
},
},
Match: IndexerParseMatch{
TorrentURL: "{{ .baseUrl }}/torrent/{{ .torrentId }}/download/{{ .passkey }}",
Match: IndexerIRCParseMatch{
TorrentURL: "/torrent/{{ .torrentId }}/download/{{ .passkey }}",
TorrentName: `{{ if .releaseGroup }}[{{ .releaseGroup }}] {{ end }}{{ .torrentName }} [{{ .year }}] {{ if .releaseEpisode }}{{ printf "- %02s " .releaseEpisode }}{{ end }}{{ print "[" .releaseTags "]" | replace " / " "][" }}`,
Encode: nil,
},
},
args: args{
baseURL: "https://mock.local/",
vars: map[string]string{
"torrentName": "Great BluRay SoftSubbed Anime",
"category": "TV Series",
@ -119,45 +105,113 @@ func TestIndexerParse_ParseMatch(t *testing.T) {
"releaseGroup": "Softsubs",
"releaseEpisode": "",
"freeleech": "freeleech",
"baseUrl": "https://mock.org",
"baseUrl": "https://mock.local",
"torrentId": "240860011",
"tags": "comedy, drama, school.life, sports",
"uploader": "Uploader",
},
extraVars: map[string]string{
"passkey": "00000000000000000000",
},
release: &Release{
Indexer: "mock",
FilterStatus: ReleaseStatusFilterPending,
Rejections: []string{},
Protocol: ReleaseProtocolTorrent,
Implementation: ReleaseImplementationIRC,
"passkey": "00000000000000000000",
},
},
expect: &Release{
Indexer: "mock",
FilterStatus: ReleaseStatusFilterPending,
Rejections: []string{},
Protocol: ReleaseProtocolTorrent,
Implementation: ReleaseImplementationIRC,
TorrentURL: "https://mock.org/torrent/240860011/download/00000000000000000000",
TorrentName: "[Softsubs] Great BluRay SoftSubbed Anime [2020] [Blu-ray][MKV][h264 10-bit][1080p][FLAC 2.0][Dual Audio][Softsubs (Sub Group)][Freeleech]",
want: &IndexerIRCParseMatched{
TorrentURL: "https://mock.local/torrent/240860011/download/00000000000000000000",
TorrentName: "[Softsubs] Great BluRay SoftSubbed Anime [2020] [Blu-ray][MKV][h264 10-bit][1080p][FLAC 2.0][Dual Audio][Softsubs (Sub Group)][Freeleech]",
},
wantErr: false,
},
{
name: "test_03",
fields: fields{
Type: "",
ForceSizeUnit: "",
Lines: []IndexerIRCParseLine{
{
Test: nil,
Pattern: "New Torrent Announcement:\\s*<([^>]*)>\\s*Name:'(.*)' uploaded by '([^']*)'\\s*(freeleech)*\\s*-\\s*(https?\\:\\/\\/[^\\/]+\\/)torrent\\/(\\d+)",
Vars: []string{
"category",
"torrentName",
"uploader",
"freeleech",
"baseUrl",
"torrentId",
},
},
},
Match: IndexerIRCParseMatch{
TorrentURL: "{{ .baseUrl }}rss/download/{{ .torrentId }}/{{ .rsskey }}/{{ .torrentName }}.torrent",
Encode: []string{"torrentName"},
},
},
args: args{
baseURL: "https://mock.local/",
vars: map[string]string{
"category": "TV :: Episodes HD",
"torrentName": "The Show 2019 S03E08 2160p DV WEBRip 6CH x265 HEVC-GROUP",
"uploader": "Anonymous",
"freeleech": "",
"baseUrl": "https://mock.local/",
"torrentId": "240860011",
"rsskey": "00000000000000000000",
},
},
want: &IndexerIRCParseMatched{
TorrentURL: "https://mock.local/rss/download/240860011/00000000000000000000/The+Show+2019+S03E08+2160p+DV+WEBRip+6CH+x265+HEVC-GROUP.torrent",
},
wantErr: false,
},
{
name: "test_04",
fields: fields{
Type: "",
ForceSizeUnit: "",
Lines: []IndexerIRCParseLine{
{
Test: nil,
Pattern: "New Torrent Announcement:\\s*<([^>]*)>\\s*Name:'(.*)' uploaded by '([^']*)'\\s*(freeleech)*\\s*-\\s*(https?\\:\\/\\/[^\\/]+\\/)torrent\\/(\\d+)",
Vars: []string{
"category",
"torrentName",
"uploader",
"freeleech",
"baseUrl",
"torrentId",
},
},
},
Match: IndexerIRCParseMatch{
TorrentURL: "https://mock.local/rss/download/{{ .torrentId }}/{{ .rsskey }}/{{ .torrentName }}.torrent",
Encode: []string{"torrentName"},
},
},
args: args{
baseURL: "https://mock.local/",
vars: map[string]string{
"category": "TV :: Episodes HD",
"torrentName": "The Show 2019 S03E08 2160p DV WEBRip 6CH x265 HEVC-GROUP",
"uploader": "Anonymous",
"freeleech": "",
"baseUrl": "https://mock.local/",
"torrentId": "240860011",
"rsskey": "00000000000000000000",
},
},
want: &IndexerIRCParseMatched{
TorrentURL: "https://mock.local/rss/download/240860011/00000000000000000000/The+Show+2019+S03E08+2160p+DV+WEBRip+6CH+x265+HEVC-GROUP.torrent",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &IndexerParse{
p := &IndexerIRCParse{
Type: tt.fields.Type,
ForceSizeUnit: tt.fields.ForceSizeUnit,
Lines: tt.fields.Lines,
Match: tt.fields.Match,
}
p.ParseMatch(tt.args.vars, tt.args.extraVars, tt.args.release)
assert.Equal(t, tt.expect, tt.args.release)
got, _ := p.ParseMatch(tt.args.baseURL, tt.args.vars)
assert.Equal(t, tt.want, got)
})
}
}

14
internal/domain/release.go
View file

@ -219,8 +219,6 @@ func (r *Release) ParseString(title string) {
}
r.ParseReleaseTagsString(r.ReleaseTags)
return
}
var ErrUnrecoverableError = errors.New("unrecoverable error")
@ -264,8 +262,6 @@ func (r *Release) ParseReleaseTagsString(tags string) {
if r.AudioChannels == "" && t.Channels != "" {
r.AudioChannels = t.Channels
}
return
}
func (r *Release) ParseSizeBytesString(size string) {
@ -401,7 +397,7 @@ func (r *Release) RejectionsString() string {
return ""
}
// MapVars better name
// MapVars map vars from regex captures to fields on release
func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) error {
if torrentName, err := getStringMapValue(varMap, "torrentName"); err != nil {
@ -444,16 +440,12 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
switch freeleechPercentInt {
case 25:
r.Bonus = append(r.Bonus, "Freeleech25")
break
case 50:
r.Bonus = append(r.Bonus, "Freeleech50")
break
case 75:
r.Bonus = append(r.Bonus, "Freeleech75")
break
case 100:
r.Bonus = append(r.Bonus, "Freeleech100")
break
}
}
@ -464,8 +456,8 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
if torrentSize, err := getStringMapValue(varMap, "torrentSize"); err == nil {
// handling for indexer who doesn't explicitly set which size unit is used like (AR)
if def.Parse != nil && def.Parse.ForceSizeUnit != "" {
torrentSize = fmt.Sprintf("%v %v", torrentSize, def.Parse.ForceSizeUnit)
if def.IRC != nil && def.IRC.Parse != nil && def.IRC.Parse.ForceSizeUnit != "" {
torrentSize = fmt.Sprintf("%v %v", torrentSize, def.IRC.Parse.ForceSizeUnit)
}
size, err := humanize.ParseBytes(torrentSize)

20
internal/domain/release_test.go
View file

@ -449,7 +449,7 @@ func TestRelease_MapVars(t *testing.T) {
"torrentSize": "10000",
"tags": "hip.hop,rhythm.and.blues, 2000s",
},
definition: IndexerDefinition{Parse: &IndexerParse{ForceSizeUnit: "MB"}},
definition: IndexerDefinition{IRC: &IndexerIRC{Parse: &IndexerIRCParse{ForceSizeUnit: "MB"}}},
},
},
{
@ -769,10 +769,7 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
TorrentURL: fmt.Sprintf("%v/%v", ts.URL, 401),
},
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
if err != nil {
return true
}
return false
return err != nil
},
},
{
@ -783,10 +780,7 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
TorrentURL: fmt.Sprintf("%v/%v", ts.URL, 403),
},
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
if err != nil {
return true
}
return false
return err != nil
},
},
{
@ -797,13 +791,11 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
TorrentURL: fmt.Sprintf("%v/%v", ts.URL, "file.torrent"),
},
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
if err != nil {
return true
}
return false
return err != nil
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := &Release{
@ -861,7 +853,7 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
Filter: tt.fields.Filter,
ActionStatus: tt.fields.ActionStatus,
}
tt.wantErr(t, r.DownloadTorrentFile(), fmt.Sprintf("DownloadTorrentFile()"))
tt.wantErr(t, r.DownloadTorrentFile(), "DownloadTorrentFile()")
})
}
}