feat(feeds): torznab add test button (#347)

This commit is contained in:
Ludvig Lundgren 2022-07-10 15:54:56 +02:00 committed by GitHub
parent c1df9c817f
commit ebba72ec1f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 804 additions and 186 deletions

98
pkg/torznab/caps.go Normal file
View file

@ -0,0 +1,98 @@
package torznab
import "encoding/xml"
type Server struct {
Version string `xml:"version,attr"`
Title string `xml:"title,attr"`
Strapline string `xml:"strapline,attr"`
Email string `xml:"email,attr"`
URL string `xml:"url,attr"`
Image string `xml:"image,attr"`
}
type Limits struct {
Max string `xml:"max,attr"`
Default string `xml:"default,attr"`
}
type Retention struct {
Days string `xml:"days,attr"`
}
type Registration struct {
Available string `xml:"available,attr"`
Open string `xml:"open,attr"`
}
type Searching struct {
Search Search `xml:"search"`
TvSearch Search `xml:"tv-search"`
MovieSearch Search `xml:"movie-search"`
AudioSearch Search `xml:"audio-search"`
BookSearch Search `xml:"book-search"`
}
type Search struct {
Available string `xml:"available,attr"`
SupportedParams string `xml:"supportedParams,attr"`
}
type Categories struct {
Category []Category `xml:"category"`
}
type Category struct {
ID string `xml:"id,attr"`
Name string `xml:"name,attr"`
Subcat []SubCategory `xml:"subcat"`
}
type SubCategory struct {
ID string `xml:"id,attr"`
Name string `xml:"name,attr"`
}
type Groups struct {
Group Group `xml:"group"`
}
type Group struct {
ID string `xml:"id,attr"`
Name string `xml:"name,attr"`
Description string `xml:"description,attr"`
Lastupdate string `xml:"lastupdate,attr"`
}
type Genres struct {
Genre Genre `xml:"genre"`
}
type Genre struct {
ID string `xml:"id,attr"`
Categoryid string `xml:"categoryid,attr"`
Name string `xml:"name,attr"`
}
type Tags struct {
Tag []Tag `xml:"tag"`
}
type Tag struct {
Name string `xml:"name,attr"`
Description string `xml:"description,attr"`
}
type CapsResponse struct {
Caps Caps `xml:"caps"`
}
type Caps struct {
XMLName xml.Name `xml:"caps"`
Server Server `xml:"server"`
Limits Limits `xml:"limits"`
Retention Retention `xml:"retention"`
Registration Registration `xml:"registration"`
Searching Searching `xml:"searching"`
Categories Categories `xml:"categories"`
Groups Groups `xml:"groups"`
Genres Genres `xml:"genres"`
Tags Tags `xml:"tags"`
}

View file

@ -1,176 +0,0 @@
package torznab
import (
"bytes"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"time"
"github.com/autobrr/autobrr/pkg/errors"
)
type Response struct {
Channel struct {
Items []FeedItem `xml:"item"`
} `xml:"channel"`
}
type FeedItem struct {
Title string `xml:"title,omitempty"`
GUID string `xml:"guid,omitempty"`
PubDate Time `xml:"pub_date,omitempty"`
Prowlarrindexer struct {
Text string `xml:",chardata"`
ID string `xml:"id,attr"`
} `xml:"prowlarrindexer"`
Comments string `xml:"comments"`
Size string `xml:"size"`
Link string `xml:"link"`
Category []string `xml:"category,omitempty"`
Categories []string
// attributes
TvdbId string `xml:"tvdb,omitempty"`
//TvMazeId string
ImdbId string `xml:"imdb,omitempty"`
TmdbId string `xml:"tmdb,omitempty"`
Attributes []struct {
XMLName xml.Name
Name string `xml:"name,attr"`
Value string `xml:"value,attr"`
} `xml:"attr"`
}
// Time credits: https://github.com/mrobinsn/go-newznab/blob/cd89d9c56447859fa1298dc9a0053c92c45ac7ef/newznab/structs.go#L150
type Time struct {
time.Time
}
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if err := e.EncodeToken(start); err != nil {
return errors.Wrap(err, "failed to encode xml token")
}
if err := e.EncodeToken(xml.CharData([]byte(t.UTC().Format(time.RFC1123Z)))); err != nil {
return errors.Wrap(err, "failed to encode xml token")
}
if err := e.EncodeToken(xml.EndElement{Name: start.Name}); err != nil {
return errors.Wrap(err, "failed to encode xml token")
}
return nil
}
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var raw string
err := d.DecodeElement(&raw, &start)
if err != nil {
return errors.Wrap(err, "could not decode element")
}
date, err := time.Parse(time.RFC1123Z, raw)
if err != nil {
return errors.Wrap(err, "could not parse date")
}
*t = Time{date}
return nil
}
type Client struct {
http *http.Client
Host string
ApiKey string
UseBasicAuth bool
BasicAuth BasicAuth
}
type BasicAuth struct {
Username string
Password string
}
func NewClient(url string, apiKey string) *Client {
httpClient := &http.Client{
Timeout: time.Second * 20,
}
c := &Client{
http: httpClient,
Host: url,
ApiKey: apiKey,
}
return c
}
func (c *Client) get(endpoint string, opts map[string]string) (int, *Response, error) {
reqUrl := fmt.Sprintf("%v%v", c.Host, endpoint)
req, err := http.NewRequest("GET", reqUrl, nil)
if err != nil {
return 0, nil, errors.Wrap(err, "could not build request")
}
if c.UseBasicAuth {
req.SetBasicAuth(c.BasicAuth.Username, c.BasicAuth.Password)
}
if c.ApiKey != "" {
req.Header.Add("X-API-Key", c.ApiKey)
}
resp, err := c.http.Do(req)
if err != nil {
return 0, nil, errors.Wrap(err, "could not make request. %+v", req)
}
defer resp.Body.Close()
var buf bytes.Buffer
if _, err = io.Copy(&buf, resp.Body); err != nil {
return resp.StatusCode, nil, errors.Wrap(err, "torznab.io.Copy")
}
var response Response
if err := xml.Unmarshal(buf.Bytes(), &response); err != nil {
return resp.StatusCode, nil, errors.Wrap(err, "torznab: could not decode feed")
}
return resp.StatusCode, &response, nil
}
func (c *Client) GetFeed() ([]FeedItem, error) {
status, res, err := c.get("?t=search", nil)
if err != nil {
return nil, errors.Wrap(err, "could not get feed")
}
if status != http.StatusOK {
return nil, errors.New("could not get feed")
}
return res.Channel.Items, nil
}
func (c *Client) Search(query string) ([]FeedItem, error) {
v := url.Values{}
v.Add("q", query)
params := v.Encode()
status, res, err := c.get("&t=search&"+params, nil)
if err != nil {
return nil, errors.Wrap(err, "could not search feed")
}
if status != http.StatusOK {
return nil, errors.New("could not search feed")
}
return res.Channel.Items, nil
}

76
pkg/torznab/feed.go Normal file
View file

@ -0,0 +1,76 @@
package torznab
import (
"encoding/xml"
"time"
"github.com/autobrr/autobrr/pkg/errors"
)
type Response struct {
Channel struct {
Items []FeedItem `xml:"item"`
} `xml:"channel"`
}
type FeedItem struct {
Title string `xml:"title,omitempty"`
GUID string `xml:"guid,omitempty"`
PubDate Time `xml:"pub_date,omitempty"`
Prowlarrindexer struct {
Text string `xml:",chardata"`
ID string `xml:"id,attr"`
} `xml:"prowlarrindexer"`
Comments string `xml:"comments"`
Size string `xml:"size"`
Link string `xml:"link"`
Category []string `xml:"category,omitempty"`
Categories []string
// attributes
TvdbId string `xml:"tvdb,omitempty"`
//TvMazeId string
ImdbId string `xml:"imdb,omitempty"`
TmdbId string `xml:"tmdb,omitempty"`
Attributes []struct {
XMLName xml.Name
Name string `xml:"name,attr"`
Value string `xml:"value,attr"`
} `xml:"attr"`
}
// Time credits: https://github.com/mrobinsn/go-newznab/blob/cd89d9c56447859fa1298dc9a0053c92c45ac7ef/newznab/structs.go#L150
type Time struct {
time.Time
}
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if err := e.EncodeToken(start); err != nil {
return errors.Wrap(err, "failed to encode xml token")
}
if err := e.EncodeToken(xml.CharData([]byte(t.UTC().Format(time.RFC1123Z)))); err != nil {
return errors.Wrap(err, "failed to encode xml token")
}
if err := e.EncodeToken(xml.EndElement{Name: start.Name}); err != nil {
return errors.Wrap(err, "failed to encode xml token")
}
return nil
}
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var raw string
err := d.DecodeElement(&raw, &start)
if err != nil {
return errors.Wrap(err, "could not decode element")
}
date, err := time.Parse(time.RFC1123Z, raw)
if err != nil {
return errors.Wrap(err, "could not parse date")
}
*t = Time{date}
return nil
}

41
pkg/torznab/testdata/caps_response.xml vendored Normal file
View file

@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<caps>
<server version="1.1" title="..." strapline="..."
email="..." url="http://indexer.local/"
image="http://indexer.local/content/banner.jpg" />
<limits max="100" default="50" />
<retention days="400" />
<registration available="yes" open="yes" />
<searching>
<search available="yes" supportedParams="q" />
<tv-search available="yes" supportedParams="q,rid,tvdbid,season,ep" />
<movie-search available="no" supportedParams="q,imdbid,genre" />
<audio-search available="no" supportedParams="q" />
<book-search available="no" supportedParams="q" />
</searching>
<categories>
<category id="2000" name="Movies">
<subcat id="2010" name="Foreign" />
</category>
<category id="5000" name="TV">
<subcat id="5040" name="HD" />
<subcat id="5070" name="Anime" />
</category>
</categories>
<groups>
<group id="1" name="alt.binaries...." description="..." lastupdate="..." />
</groups>
<genres>
<genre id="1" categoryid="5000" name="Kids" />
</genres>
<tags>
<tag name="anonymous" description="Uploader is anonymous" />
<tag name="trusted" description="Uploader has high reputation" />
<tag name="internal" description="Uploader is an internal release group" />
</tags>
</caps>

209
pkg/torznab/torznab.go Normal file
View file

@ -0,0 +1,209 @@
package torznab
import (
"bytes"
"encoding/xml"
"io"
"log"
"net/http"
"net/http/httputil"
"net/url"
"strings"
"time"
"github.com/autobrr/autobrr/pkg/errors"
)
type Client interface {
GetFeed() ([]FeedItem, error)
GetCaps() (*Caps, error)
}
type client struct {
http *http.Client
Host string
ApiKey string
UseBasicAuth bool
BasicAuth BasicAuth
Log *log.Logger
}
type BasicAuth struct {
Username string
Password string
}
type Config struct {
Host string
ApiKey string
UseBasicAuth bool
BasicAuth BasicAuth
Log *log.Logger
}
func NewClient(config Config) Client {
httpClient := &http.Client{
Timeout: time.Second * 20,
}
c := &client{
http: httpClient,
Host: config.Host,
ApiKey: config.ApiKey,
Log: log.New(io.Discard, "", log.LstdFlags),
}
if config.Log != nil {
c.Log = config.Log
}
return c
}
func (c *client) get(endpoint string, opts map[string]string) (int, *Response, error) {
params := url.Values{
"t": {"search"},
}
u, err := url.Parse(c.Host)
u.Path = strings.TrimSuffix(u.Path, "/")
u.RawQuery = params.Encode()
reqUrl := u.String()
req, err := http.NewRequest("GET", reqUrl, nil)
if err != nil {
return 0, nil, errors.Wrap(err, "could not build request")
}
if c.UseBasicAuth {
req.SetBasicAuth(c.BasicAuth.Username, c.BasicAuth.Password)
}
if c.ApiKey != "" {
req.Header.Add("X-API-Key", c.ApiKey)
}
resp, err := c.http.Do(req)
if err != nil {
return 0, nil, errors.Wrap(err, "could not make request. %+v", req)
}
defer resp.Body.Close()
var buf bytes.Buffer
if _, err = io.Copy(&buf, resp.Body); err != nil {
return resp.StatusCode, nil, errors.Wrap(err, "torznab.io.Copy")
}
var response Response
if err := xml.Unmarshal(buf.Bytes(), &response); err != nil {
return resp.StatusCode, nil, errors.Wrap(err, "torznab: could not decode feed")
}
return resp.StatusCode, &response, nil
}
func (c *client) GetFeed() ([]FeedItem, error) {
status, res, err := c.get("", nil)
if err != nil {
return nil, errors.Wrap(err, "could not get feed")
}
if status != http.StatusOK {
return nil, errors.New("could not get feed")
}
return res.Channel.Items, nil
}
func (c *client) getCaps(endpoint string, opts map[string]string) (int, *Caps, error) {
params := url.Values{
"t": {"caps"},
}
u, err := url.Parse(c.Host)
u.Path = strings.TrimSuffix(u.Path, "/")
u.RawQuery = params.Encode()
reqUrl := u.String()
req, err := http.NewRequest("GET", reqUrl, nil)
if err != nil {
return 0, nil, errors.Wrap(err, "could not build request")
}
if c.UseBasicAuth {
req.SetBasicAuth(c.BasicAuth.Username, c.BasicAuth.Password)
}
if c.ApiKey != "" {
req.Header.Add("X-API-Key", c.ApiKey)
}
resp, err := c.http.Do(req)
if err != nil {
return 0, nil, errors.Wrap(err, "could not make request. %+v", req)
}
defer resp.Body.Close()
dump, err := httputil.DumpResponse(resp, true)
if err != nil {
return 0, nil, errors.Wrap(err, "could not dump response")
}
c.Log.Printf("get torrent trackers response dump: %q", dump)
if resp.StatusCode == http.StatusUnauthorized {
return resp.StatusCode, nil, errors.New("unauthorized")
} else if resp.StatusCode != http.StatusOK {
return resp.StatusCode, nil, errors.New("bad status: %d", resp.StatusCode)
}
var buf bytes.Buffer
if _, err = io.Copy(&buf, resp.Body); err != nil {
return resp.StatusCode, nil, errors.Wrap(err, "torznab.io.Copy")
}
var response Caps
if err := xml.Unmarshal(buf.Bytes(), &response); err != nil {
return resp.StatusCode, nil, errors.Wrap(err, "torznab: could not decode feed")
}
return resp.StatusCode, &response, nil
}
func (c *client) GetCaps() (*Caps, error) {
status, res, err := c.getCaps("?t=caps", nil)
if err != nil {
return nil, errors.Wrap(err, "could not get caps for feed")
}
if status != http.StatusOK {
return nil, errors.Wrap(err, "could not get caps for feed")
}
return res, nil
}
func (c *client) Search(query string) ([]FeedItem, error) {
v := url.Values{}
v.Add("q", query)
params := v.Encode()
status, res, err := c.get("&t=search&"+params, nil)
if err != nil {
return nil, errors.Wrap(err, "could not search feed")
}
if status != http.StatusOK {
return nil, errors.New("could not search feed")
}
return res.Channel.Items, nil
}

235
pkg/torznab/torznab_test.go Normal file
View file

@ -0,0 +1,235 @@
package torznab
import (
"encoding/xml"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
)
//func TestClient_GetFeed(t *testing.T) {
// key := "mock-key"
//
// srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// apiKey := r.Header.Get("X-API-Key")
// if apiKey != "" {
// if apiKey != key {
// w.WriteHeader(http.StatusUnauthorized)
// w.Write(nil)
// return
// }
// }
// payload, err := ioutil.ReadFile("testdata/torznab_response.xml")
// if err != nil {
// http.Error(w, err.Error(), http.StatusInternalServerError)
// return
// }
// w.Header().Set("Content-Type", "application/xml")
// w.Write(payload)
// }))
// defer srv.Close()
//
// type fields struct {
// Host string
// ApiKey string
// BasicAuth BasicAuth
// }
// tests := []struct {
// name string
// fields fields
// want []FeedItem
// wantErr bool
// }{
// {
// name: "get feed",
// fields: fields{
// Host: srv.URL + "/api",
// ApiKey: key,
// BasicAuth: BasicAuth{},
// },
// want: nil,
// wantErr: false,
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// c := NewClient(Config{Host: tt.fields.Host, ApiKey: tt.fields.ApiKey})
//
// _, err := c.GetFeed()
// if tt.wantErr && assert.Error(t, err) {
// assert.Equal(t, tt.wantErr, err)
// }
// //assert.Equal(t, tt.want, got)
// })
// }
//}
func TestClient_GetCaps(t *testing.T) {
key := "mock-key"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
apiKey := r.Header.Get("X-API-Key")
if apiKey != key {
w.WriteHeader(http.StatusUnauthorized)
w.Write(nil)
return
}
payload, err := ioutil.ReadFile("testdata/caps_response.xml")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/xml")
w.Write(payload)
}))
defer srv.Close()
type fields struct {
Host string
ApiKey string
BasicAuth BasicAuth
}
tests := []struct {
name string
fields fields
want *Caps
wantErr bool
expectedErr string
}{
{
name: "get caps",
fields: fields{
Host: srv.URL + "/api/",
ApiKey: key,
BasicAuth: BasicAuth{},
},
want: &Caps{
XMLName: xml.Name{
Space: "",
Local: "caps",
},
Server: Server{
Version: "1.1",
Title: "...",
Strapline: "...",
Email: "...",
URL: "http://indexer.local/",
Image: "http://indexer.local/content/banner.jpg",
},
Limits: Limits{
Max: "100",
Default: "50",
},
Retention: Retention{
Days: "400",
},
Registration: Registration{
Available: "yes",
Open: "yes",
},
Searching: Searching{
Search: Search{
Available: "yes",
SupportedParams: "q",
},
TvSearch: Search{
Available: "yes",
SupportedParams: "q,rid,tvdbid,season,ep",
},
MovieSearch: Search{
Available: "no",
SupportedParams: "q,imdbid,genre",
},
AudioSearch: Search{
Available: "no",
SupportedParams: "q",
},
BookSearch: Search{
Available: "no",
SupportedParams: "q",
},
},
Categories: Categories{Category: []Category{
{
ID: "2000",
Name: "Movies",
Subcat: []SubCategory{
{
ID: "2010",
Name: "Foreign",
},
},
},
{
ID: "5000",
Name: "TV",
Subcat: []SubCategory{
{
ID: "5040",
Name: "HD",
},
{
ID: "5070",
Name: "Anime",
},
},
},
}},
Groups: Groups{Group: Group{
ID: "1",
Name: "alt.binaries....",
Description: "...",
Lastupdate: "...",
}},
Genres: Genres{
Genre: Genre{
ID: "1",
Categoryid: "5000",
Name: "Kids",
},
},
Tags: Tags{Tag: []Tag{
{
Name: "anonymous",
Description: "Uploader is anonymous",
},
{
Name: "trusted",
Description: "Uploader has high reputation",
},
{
Name: "internal",
Description: "Uploader is an internal release group",
},
}},
},
wantErr: false,
},
{
name: "bad key",
fields: fields{
Host: srv.URL,
ApiKey: "badkey",
BasicAuth: BasicAuth{},
},
want: nil,
wantErr: true,
expectedErr: "could not get caps for feed: unauthorized",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := NewClient(Config{Host: tt.fields.Host, ApiKey: tt.fields.ApiKey})
got, err := c.GetCaps()
if tt.wantErr && assert.Error(t, err) {
assert.EqualErrorf(t, err, tt.expectedErr, "Error should be: %v, got: %v", tt.wantErr, err)
}
assert.Equal(t, tt.want, got)
})
}
}