feat(indexers): add API support for Orpheus to fetch size (#944)

* feat(indexers): add API support for Orpheus

* feat(filters): add ops to AdditionalSizeCheck
This commit is contained in:
ze0s 2023-05-21 16:55:10 +02:00 committed by GitHub
parent 8bef297841
commit 8925266104
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 481 additions and 3 deletions

View file

@ -339,6 +339,7 @@ type TorrentBasic struct {
TorrentId string `json:"TorrentId,omitempty"`
InfoHash string `json:"InfoHash"`
Size string `json:"Size"`
Uploader string `json:"Uploader"`
}
func (t TorrentBasic) ReleaseSizeBytes() uint64 {

View file

@ -439,9 +439,10 @@ func (s *service) AdditionalSizeCheck(ctx context.Context, f domain.Filter, rele
s.log.Debug().Msgf("filter.Service.AdditionalSizeCheck: (%s) additional size check required", f.Name)
switch release.Indexer {
case "ptp", "btn", "ggn", "redacted", "mock":
case "ptp", "btn", "ggn", "redacted", "ops", "mock":
if release.Size == 0 {
s.log.Trace().Msgf("filter.Service.AdditionalSizeCheck: (%s) preparing to check via api", f.Name)
torrentInfo, err := s.apiService.GetTorrentByID(ctx, release.Indexer, release.TorrentID)
if err != nil || torrentInfo == nil {
s.log.Error().Stack().Err(err).Msgf("filter.Service.AdditionalSizeCheck: (%s) could not get torrent info from api: '%s' from: %s", f.Name, release.TorrentID, release.Indexer)

View file

@ -6,16 +6,17 @@ package indexer
import (
"context"
"github.com/rs/zerolog"
"github.com/autobrr/autobrr/internal/domain"
"github.com/autobrr/autobrr/internal/logger"
"github.com/autobrr/autobrr/internal/mock"
"github.com/autobrr/autobrr/pkg/btn"
"github.com/autobrr/autobrr/pkg/errors"
"github.com/autobrr/autobrr/pkg/ggn"
"github.com/autobrr/autobrr/pkg/ops"
"github.com/autobrr/autobrr/pkg/ptp"
"github.com/autobrr/autobrr/pkg/red"
"github.com/rs/zerolog"
)
type APIService interface {
@ -115,6 +116,13 @@ func (s *apiService) AddClient(indexer string, settings map[string]string) error
}
s.apiClients[indexer] = red.NewClient(key)
case "ops":
key, ok := settings["api_key"]
if !ok || key == "" {
return errors.New("api.Service.AddClient: could not initialize orpheus client: missing var 'api_key'")
}
s.apiClients[indexer] = ops.NewClient(key)
case "mock":
s.apiClients[indexer] = mock.NewMockClient("", "mock")
@ -166,6 +174,12 @@ func (s *apiService) getClientForTest(req domain.IndexerTestApiRequest) (apiClie
}
return red.NewClient(req.ApiKey), nil
case "ops":
if req.ApiKey == "" {
return nil, errors.New("api.Service.AddClient: could not initialize orpheus client: missing var 'api_key'")
}
return ops.NewClient(req.ApiKey), nil
case "mock":
return mock.NewMockClient("", "mock"), nil

View file

@ -9,6 +9,7 @@ urls:
privacy: private
protocol: torrent
supports:
- api
- irc
- rss
source: gazelle
@ -19,6 +20,24 @@ settings:
label: Torrent pass
help: Right click DL on a torrent and get the torrent_pass.
- name: api_key
type: secret
required: true
label: API Key
help: Settings -> Access Settings -> API Keys - Create a new api token.
api:
url: https://orpheus.network/ajax.php
type: json
limits:
max: 5
per: 10 seconds
settings:
- name: api_key
type: secret
label: API Key
help: Settings -> Access Settings -> API Keys - Create a new api token.
irc:
network: Orpheus
server: irc.orpheus.network

227
pkg/ops/ops.go Normal file
View file

@ -0,0 +1,227 @@
package ops
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
"github.com/autobrr/autobrr/internal/domain"
"github.com/autobrr/autobrr/pkg/errors"
"golang.org/x/time/rate"
)
type ApiClient interface {
GetTorrentByID(ctx context.Context, torrentID string) (*domain.TorrentBasic, error)
TestAPI(ctx context.Context) (bool, error)
UseURL(url string)
}
type Client struct {
Url string
client *http.Client
RateLimiter *rate.Limiter
APIKey string
}
func NewClient(apiKey string) ApiClient {
c := &Client{
Url: "https://orpheus.network/ajax.php",
client: &http.Client{
Timeout: time.Second * 30,
},
RateLimiter: rate.NewLimiter(rate.Every(10*time.Second), 5),
APIKey: apiKey,
}
return c
}
func (c *Client) UseURL(url string) {
c.Url = url
}
type ErrorResponse struct {
Status string `json:"status"`
Error string `json:"error,omitempty"`
}
type TorrentDetailsResponse struct {
Status string `json:"status"`
Response struct {
Group Group `json:"group"`
Torrent Torrent `json:"torrent"`
} `json:"response"`
Error string `json:"error,omitempty"`
}
type Group struct {
//WikiBody string `json:"wikiBody"`
//WikiImage string `json:"wikiImage"`
Id int `json:"id"`
Name string `json:"name"`
Year int `json:"year"`
RecordLabel string `json:"recordLabel"`
CatalogueNumber string `json:"catalogueNumber"`
ReleaseType int `json:"releaseType"`
CategoryId int `json:"categoryId"`
CategoryName string `json:"categoryName"`
Time string `json:"time"`
VanityHouse bool `json:"vanityHouse"`
//MusicInfo struct {
// Composers []interface{} `json:"composers"`
// Dj []interface{} `json:"dj"`
// Artists []struct {
// Id int `json:"id"`
// Name string `json:"name"`
// } `json:"artists"`
// With []struct {
// Id int `json:"id"`
// Name string `json:"name"`
// } `json:"with"`
// Conductor []interface{} `json:"conductor"`
// RemixedBy []interface{} `json:"remixedBy"`
// Producer []interface{} `json:"producer"`
//} `json:"musicInfo"`
}
type Torrent struct {
Id int `json:"id"`
InfoHash string `json:"infoHash"`
Media string `json:"media"`
Format string `json:"format"`
Encoding string `json:"encoding"`
Remastered bool `json:"remastered"`
RemasterYear int `json:"remasterYear"`
RemasterTitle string `json:"remasterTitle"`
RemasterRecordLabel string `json:"remasterRecordLabel"`
RemasterCatalogueNumber string `json:"remasterCatalogueNumber"`
Scene bool `json:"scene"`
HasLog bool `json:"hasLog"`
HasCue bool `json:"hasCue"`
LogScore int `json:"logScore"`
FileCount int `json:"fileCount"`
Size int `json:"size"`
Seeders int `json:"seeders"`
Leechers int `json:"leechers"`
Snatched int `json:"snatched"`
FreeTorrent string `json:"freeTorrent"`
Time string `json:"time"`
Description string `json:"description"`
FileList string `json:"fileList"`
FilePath string `json:"filePath"`
UserId int `json:"userId"`
Username string `json:"username"`
}
func (c *Client) Do(req *http.Request) (*http.Response, error) {
//ctx := context.Background()
err := c.RateLimiter.Wait(req.Context()) // This is a blocking call. Honors the rate limit
if err != nil {
return nil, err
}
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
return resp, nil
}
func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
if c.APIKey == "" {
return nil, errors.New("orpheus client missing API key!")
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
if err != nil {
return nil, errors.Wrap(err, "could not build request")
}
req.Header.Add("Authorization", fmt.Sprintf("token %s", c.APIKey))
req.Header.Set("User-Agent", "autobrr")
res, err := c.Do(req)
if err != nil {
return nil, errors.Wrap(err, "could not make request: %+v", req)
}
// return early if not OK
if res.StatusCode != http.StatusOK {
var r ErrorResponse
body, readErr := io.ReadAll(res.Body)
if readErr != nil {
return nil, errors.Wrap(readErr, "could not read body")
}
if err = json.Unmarshal(body, &r); err != nil {
return nil, errors.Wrap(err, "could not unmarshal body")
}
res.Body.Close()
return nil, errors.New("status code: %d status: %s error: %s", res.StatusCode, r.Status, r.Error)
}
return res, nil
}
func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.TorrentBasic, error) {
if torrentID == "" {
return nil, errors.New("orpheus client: must have torrentID")
}
var r TorrentDetailsResponse
v := url.Values{}
v.Add("id", torrentID)
params := v.Encode()
reqUrl := fmt.Sprintf("%s?action=torrent&%s", c.Url, params)
resp, err := c.get(ctx, reqUrl)
if err != nil {
return nil, errors.Wrap(err, "could not get torrent by id: %v", torrentID)
}
defer resp.Body.Close()
body, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return nil, errors.Wrap(readErr, "could not read body")
}
if err := json.Unmarshal(body, &r); err != nil {
return nil, errors.Wrap(err, "could not unmarshal body")
}
res := &domain.TorrentBasic{
Id: strconv.Itoa(r.Response.Torrent.Id),
InfoHash: r.Response.Torrent.InfoHash,
Size: strconv.Itoa(r.Response.Torrent.Size),
Uploader: r.Response.Torrent.Username,
}
return res, nil
}
// TestAPI try api access against torrents page
func (c *Client) TestAPI(ctx context.Context) (bool, error) {
resp, err := c.get(ctx, c.Url+"?action=index")
if err != nil {
return false, errors.Wrap(err, "test api error")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, nil
}
return true, nil
}

110
pkg/ops/ops_test.go Normal file
View file

@ -0,0 +1,110 @@
package ops
import (
"context"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/autobrr/autobrr/internal/domain"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
)
func TestOrpheusClient_GetTorrentByID(t *testing.T) {
// disable logger
zerolog.SetGlobalLevel(zerolog.Disabled)
key := "mock-key"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// request validation logic
apiKey := r.Header.Get("Authorization")
if !strings.Contains(apiKey, key) {
w.WriteHeader(http.StatusUnauthorized)
w.Write(nil)
return
}
if !strings.Contains(r.RequestURI, "2156788") {
jsonPayload, _ := os.ReadFile("testdata/get_torrent_by_id_not_found.json")
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
w.Write(jsonPayload)
return
}
// read json response
jsonPayload, _ := os.ReadFile("testdata/get_torrent_by_id.json")
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(jsonPayload)
}))
defer ts.Close()
type fields struct {
Url string
APIKey string
}
type args struct {
torrentID string
}
tests := []struct {
name string
fields fields
args args
want *domain.TorrentBasic
wantErr string
}{
{
name: "get_by_id_1",
fields: fields{
Url: ts.URL,
APIKey: key,
},
args: args{torrentID: "2156788"},
want: &domain.TorrentBasic{
Id: "2156788",
InfoHash: "",
Size: "255299244",
Uploader: "uploader",
},
wantErr: "",
},
{
name: "get_by_id_2",
fields: fields{
Url: ts.URL,
APIKey: key,
},
args: args{torrentID: "100002"},
want: nil,
wantErr: "could not get torrent by id: 100002: status code: 400 status: failure error: bad id parameter",
},
{
name: "get_by_id_3",
fields: fields{
Url: ts.URL,
APIKey: "",
},
args: args{torrentID: "100002"},
want: nil,
wantErr: "could not get torrent by id: 100002: orpheus client missing API key!",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := NewClient(tt.fields.APIKey)
c.UseURL(tt.fields.Url)
got, err := c.GetTorrentByID(context.Background(), tt.args.torrentID)
if tt.wantErr != "" && assert.Error(t, err) {
assert.EqualErrorf(t, err, tt.wantErr, "Error should be: %v, got: %v", tt.wantErr, err)
}
assert.Equal(t, tt.want, got)
})
}
}

29
pkg/ops/testdata/get_index.json vendored Normal file
View file

@ -0,0 +1,29 @@
{
"status": "success",
"response": {
"username": "username",
"id": 1,
"authkey": "redacted",
"passkey": "redacted",
"notifications": {
"messages": 0,
"notifications": 0,
"newAnnouncement": false,
"newBlog": false,
"newSubscriptions": false
},
"userstats": {
"uploaded": 0,
"downloaded": 0,
"ratio": -1,
"requiredratio": 0,
"bonusPoints": 0,
"bonusPointsPerHour": 0.0,
"class": "User"
}
},
"info": {
"source": "Orpheus",
"version": 1
}
}

76
pkg/ops/testdata/get_torrent_by_id.json vendored Normal file
View file

@ -0,0 +1,76 @@
{
"status": "success",
"response": {
"group": {
"wikiBody": "<span class=\"size4\"><strong>Tracklist<\/strong><\/span><br \/>\r\n<strong>1.<\/strong> Den som gikk for fort <span style=\"font-style: italic;\">(04:32)<\/span><br \/>\r\n<strong>2.<\/strong> Then There <span style=\"font-style: italic;\">(04:53)<\/span><br \/>\r\n<strong>3.<\/strong> Fall Sideways <span style=\"font-style: italic;\">(06:40)<\/span><br \/>\r\n<strong>4.<\/strong> Laugarbakki <span style=\"font-style: italic;\">(05:06)<\/span><br \/>\r\n<strong>5.<\/strong> Seventyfour <span style=\"font-style: italic;\">(06:25)<\/span><br \/>\r\n<strong>6.<\/strong> &Oslash;.T.O. <span style=\"font-style: italic;\">(05:00)<\/span><br \/>\r\n<strong>7.<\/strong> Dekaton <span style=\"font-style: italic;\">(03:40)<\/span><br \/>\r\n<strong>8.<\/strong> Cabelwaag <span style=\"font-style: italic;\">(05:33)<\/span><br \/>\r\n<br \/>\r\n<strong>Total length:<\/strong> 41:49<br \/>\r\n<br \/>\r\nA typical tune by the Norwegian guitar-based trio The Leif will always have a distinct melody. It can be a stretched-out composition or a simple theme. So also, on Then There, the band&#39;s third release.<br \/>\r\n<br \/>\r\nOne of the tunes on this new album (Den som gikk for fort) was originally simply a row of chords that Tellef occasionally used to play on his trusted old Martin D28 acoustic guitar. When he eventually suggested it as a tune at a rehearsal, Magnus quickly added a melody with his Fender Jazzmaster, then Tellef added a groovy side theme, and a new song was born. Another tune (Fall Sideways) has a falling phrase as its melody, repeated in various tone heights. Other times the musicians&#39; jamming on a tune can evolve into bullying with both the composition and the band&#39;s musical starting points (Laugarbakki). Otherwise, The Leif has always maintained a quiet side on the border of the meditative, a side that is represented, also on this third release (&Oslash;.TO and Cabelwaag).<br \/>\r\n<br \/>\r\nWhen Tenge and &Oslash;grim started the band as a duo in 2015, they did not define a musical program, style, or aim. All the two neighbors wanted was to check out some ideas for possible guitar tunes.<br \/>\r\n<br \/>\r\nThey quickly realized, despite their different musical backgrounds, that they had stumbled upon a chemistry that tempted them to continue meeting in Tellef&#39;s basement. (On the band name they chose: Magnus&#39;s middle name is Leif, and the old rural name Tellef was originally a contraction of the two names Thor and Leif. By the way, Leif is an Old Norse name, meaning &quot;heir&quot; or &quot;descendant &quot;).<br \/>\r\n<br \/>\r\nBy the band&#39;s second release, the EP But you move on (2020), drummer Per Oddvar Johansen had joined the band, making the duo a trio, and adding new rhythmic progress to their music, as well as Johansen&#39;s well-known deep musical imagination . (A side-fact: Tenge and Johansen played together in bands during high school many years ago, making their cooperation in The Leif somewhat of a revival project for the two friends.)<br \/>\r\n<br \/>\r\nTogether, the three musicians have approximately 132 years of experience in genres as rock, pop, older and newer jazz, free impro music and other fun things.<br \/>\r\n<br \/>\r\nA musical detective, or historian, would therefore probably be able to find sounds in the band&#39;s music that point some years back into the last century, also on this third release. If you listened carefully maybe you could hear some glimpses of the 1970-ties, for example.<br \/>\r\n<br \/>\r\nFor the musicians themselves, however, the most important thing is to find versions of their original compositions that work and feel good here and now.<br \/>\r\n<br \/>\r\nThe band&#39;s aim is to present their new tunes, but not least to relay an impression of how the music is filtered through just these three musicians as they play together here and now, and to give their highly personal impression of contemporary live-sounding music, that hopefully will tickle your imagination and make your mind wander.<br \/>\r\n<br \/>\r\nThe recording of this new album was done as early as March 2021. Then the hard disc with all the recordings died. After several attempts, most of the recorded material was retrieved, many months later.<br \/>\r\n<br \/>\r\nFive of the tunes were mixed by Audun Borrmann. The mastering was done by Sverre Erik Henriksen.<br \/>\r\n<br \/>\r\nMagnus Tenge: guitar<br \/>\r\nPer Oddvar Johansen: trommer<br \/>\r\nTellef &Oslash;grim: guitar<br \/>\r\n<br \/>\r\nMore information: <a rel=\"noreferrer\" target=\"_blank\" href=\"https:\/\/www.deezer.com\/us\/album\/413883487\">https:\/\/www.deezer.com\/us\/album\/413883487<\/a>",
"wikiBBcode": "[size=4][b]Tracklist[\/b][\/size]\r\n[b]1.[\/b] Den som gikk for fort [i](04:32)[\/i]\r\n[b]2.[\/b] Then There [i](04:53)[\/i]\r\n[b]3.[\/b] Fall Sideways [i](06:40)[\/i]\r\n[b]4.[\/b] Laugarbakki [i](05:06)[\/i]\r\n[b]5.[\/b] Seventyfour [i](06:25)[\/i]\r\n[b]6.[\/b] &Oslash;.T.O. [i](05:00)[\/i]\r\n[b]7.[\/b] Dekaton [i](03:40)[\/i]\r\n[b]8.[\/b] Cabelwaag [i](05:33)[\/i]\r\n\r\n[b]Total length:[\/b] 41:49\r\n\r\nA typical tune by the Norwegian guitar-based trio The Leif will always have a distinct melody. It can be a stretched-out composition or a simple theme. So also, on Then There, the band&#39;s third release.\r\n\r\nOne of the tunes on this new album (Den som gikk for fort) was originally simply a row of chords that Tellef occasionally used to play on his trusted old Martin D28 acoustic guitar. When he eventually suggested it as a tune at a rehearsal, Magnus quickly added a melody with his Fender Jazzmaster, then Tellef added a groovy side theme, and a new song was born. Another tune (Fall Sideways) has a falling phrase as its melody, repeated in various tone heights. Other times the musicians&#39; jamming on a tune can evolve into bullying with both the composition and the band&#39;s musical starting points (Laugarbakki). Otherwise, The Leif has always maintained a quiet side on the border of the meditative, a side that is represented, also on this third release (&Oslash;.TO and Cabelwaag).\r\n\r\nWhen Tenge and &Oslash;grim started the band as a duo in 2015, they did not define a musical program, style, or aim. All the two neighbors wanted was to check out some ideas for possible guitar tunes.\r\n\r\nThey quickly realized, despite their different musical backgrounds, that they had stumbled upon a chemistry that tempted them to continue meeting in Tellef&#39;s basement. (On the band name they chose: Magnus&#39;s middle name is Leif, and the old rural name Tellef was originally a contraction of the two names Thor and Leif. By the way, Leif is an Old Norse name, meaning &quot;heir&quot; or &quot;descendant &quot;).\r\n\r\nBy the band&#39;s second release, the EP But you move on (2020), drummer Per Oddvar Johansen had joined the band, making the duo a trio, and adding new rhythmic progress to their music, as well as Johansen&#39;s well-known deep musical imagination . (A side-fact: Tenge and Johansen played together in bands during high school many years ago, making their cooperation in The Leif somewhat of a revival project for the two friends.)\r\n\r\nTogether, the three musicians have approximately 132 years of experience in genres as rock, pop, older and newer jazz, free impro music and other fun things.\r\n\r\nA musical detective, or historian, would therefore probably be able to find sounds in the band&#39;s music that point some years back into the last century, also on this third release. If you listened carefully maybe you could hear some glimpses of the 1970-ties, for example.\r\n\r\nFor the musicians themselves, however, the most important thing is to find versions of their original compositions that work and feel good here and now.\r\n\r\nThe band&#39;s aim is to present their new tunes, but not least to relay an impression of how the music is filtered through just these three musicians as they play together here and now, and to give their highly personal impression of contemporary live-sounding music, that hopefully will tickle your imagination and make your mind wander.\r\n\r\nThe recording of this new album was done as early as March 2021. Then the hard disc with all the recordings died. After several attempts, most of the recorded material was retrieved, many months later.\r\n\r\nFive of the tunes were mixed by Audun Borrmann. The mastering was done by Sverre Erik Henriksen.\r\n\r\nMagnus Tenge: guitar\r\nPer Oddvar Johansen: trommer\r\nTellef &Oslash;grim: guitar\r\n\r\nMore information: [url]https:\/\/www.deezer.com\/us\/album\/413883487[\/url]",
"wikiImage": "https:\/\/ptpimg.me\/bs5j24.jpg",
"id": 1009968,
"name": "Then There",
"year": 2023,
"recordLabel": "",
"catalogueNumber": "",
"releaseType": 1,
"releaseTypeName": "Album",
"categoryId": 1,
"categoryName": "Music",
"time": "2023-04-26 16:12:08",
"vanityHouse": false,
"isBookmarked": false,
"tags": [
"jazz",
"jazz.rock"
],
"musicInfo": {
"artists": [
{
"id": 895381,
"name": "The Leif"
}
],
"with": [],
"remixedBy": [],
"composers": [],
"conductor": [],
"dj": [],
"producer": [],
"arranger": []
}
},
"torrent": {
"id": 2156788,
"media": "WEB",
"format": "FLAC",
"encoding": "Lossless",
"remastered": true,
"remasterYear": 2023,
"remasterTitle": "",
"remasterRecordLabel": "FAJo Music",
"remasterCatalogueNumber": "",
"scene": false,
"hasLog": false,
"hasCue": false,
"logScore": 0,
"logChecksum": false,
"logCount": 0,
"ripLogIds": [],
"fileCount": 9,
"size": 255299244,
"seeders": 0,
"leechers": 0,
"snatched": 0,
"freeTorrent": "0",
"reported": false,
"time": "2023-04-26 16:12:08",
"description": "Converted with SoX",
"fileList": "01 - Den som gikk for fort.flac{{{33226159}}}|||02 - Then There.flac{{{34072761}}}|||03 - Fall Sideways.flac{{{39623945}}}|||04 - Laugarbakki.flac{{{30333579}}}|||05 - Seventyfour.flac{{{37082402}}}|||06 - o.T.O..flac{{{16351581}}}|||07 - Dekaton.flac{{{26135063}}}|||08 - Cabelwaag.flac{{{37468849}}}|||folder.jpg{{{1004905}}}",
"filePath": "The Leif - Then There (2023)",
"userId": 1010,
"username": "uploader"
}
},
"info": {
"source": "Orpheus",
"version": 5
}
}

View file

@ -0,0 +1 @@
{"status":"failure","error":"bad id parameter"}