mirror of
https://github.com/idanoo/autobrr
synced 2025-07-22 16:29:12 +00:00
feat(rss): implement relative URL support (#484)
* feat(rss): implement relative URL support * bp had some words * feat(gh): uplift golang to 1.19.1 * feat(docker): uplift golang to 1.19 * feat(docker): uplift to go 1.19 * cleanup * chore: update go version in go.mod * feat: refactor and add test feed process item * unescape query params after join * break out logic into separate method * add tests
This commit is contained in:
Kyle Sanderson
GitHub
parent
5f85a9a990
commit
47eaeaa635
6 changed files with 177 additions and 47 deletions
4
.github/workflows/release.yml
vendored
4
.github/workflows/release.yml
vendored
|
|
@ -61,7 +61,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.18.5'
|
||||
go-version: '1.19.1'
|
||||
cache: true
|
||||
|
||||
- name: Run GoReleaser build
|
||||
|
|
@ -99,7 +99,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.18.5'
|
||||
go-version: '1.19.1'
|
||||
cache: true
|
||||
|
||||
- name: Run GoReleaser build and publish tags
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ COPY web .
|
|||
RUN yarn build
|
||||
|
||||
# build app
|
||||
FROM golang:1.18-alpine3.16 AS app-builder
|
||||
FROM golang:1.19-alpine3.16 AS app-builder
|
||||
|
||||
ARG VERSION=dev
|
||||
ARG REVISION=dev
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# build app
|
||||
FROM golang:1.18-alpine3.16 AS app-builder
|
||||
FROM golang:1.19-alpine3.16 AS app-builder
|
||||
|
||||
ARG VERSION=dev
|
||||
ARG REVISION=dev
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -1,6 +1,6 @@
|
|||
module github.com/autobrr/autobrr
|
||||
|
||||
go 1.18
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/Masterminds/sprig/v3 v3.2.2
|
||||
|
|
@ -35,6 +35,7 @@ require (
|
|||
github.com/stretchr/testify v1.8.0
|
||||
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
|
|
@ -84,7 +85,6 @@ require (
|
|||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.1 // indirect
|
||||
golang.org/x/mod v0.4.1 // indirect
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package feed
|
|||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
|
|
@ -68,47 +69,7 @@ func (j *RSSJob) process() error {
|
|||
releases := make([]*domain.Release, 0)
|
||||
|
||||
for _, item := range items {
|
||||
rls := domain.NewRelease(j.IndexerIdentifier)
|
||||
rls.Implementation = domain.ReleaseImplementationRSS
|
||||
|
||||
rls.ParseString(item.Title)
|
||||
|
||||
if len(item.Enclosures) > 0 {
|
||||
e := item.Enclosures[0]
|
||||
if e.Type == "application/x-bittorrent" && e.URL != "" {
|
||||
rls.TorrentURL = e.URL
|
||||
}
|
||||
if e.Length != "" {
|
||||
rls.ParseSizeBytesString(e.Length)
|
||||
}
|
||||
}
|
||||
|
||||
if rls.TorrentURL == "" && item.Link != "" {
|
||||
rls.TorrentURL = item.Link
|
||||
}
|
||||
|
||||
for _, v := range item.Categories {
|
||||
if len(rls.Category) != 0 {
|
||||
rls.Category += ", "
|
||||
}
|
||||
|
||||
rls.Category += v
|
||||
}
|
||||
|
||||
for _, v := range item.Authors {
|
||||
if len(rls.Uploader) != 0 {
|
||||
rls.Uploader += ", "
|
||||
}
|
||||
|
||||
rls.Uploader += v.Name
|
||||
}
|
||||
|
||||
if rls.Size == 0 {
|
||||
// parse size bytes string
|
||||
if sz, ok := item.Custom["size"]; ok {
|
||||
rls.ParseSizeBytesString(sz)
|
||||
}
|
||||
}
|
||||
rls := j.processItem(item)
|
||||
|
||||
releases = append(releases, rls)
|
||||
}
|
||||
|
|
@ -119,6 +80,65 @@ func (j *RSSJob) process() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (j *RSSJob) processItem(item *gofeed.Item) *domain.Release {
|
||||
rls := domain.NewRelease(j.IndexerIdentifier)
|
||||
rls.Implementation = domain.ReleaseImplementationRSS
|
||||
|
||||
rls.ParseString(item.Title)
|
||||
|
||||
if len(item.Enclosures) > 0 {
|
||||
e := item.Enclosures[0]
|
||||
if e.Type == "application/x-bittorrent" && e.URL != "" {
|
||||
rls.TorrentURL = e.URL
|
||||
}
|
||||
if e.Length != "" {
|
||||
rls.ParseSizeBytesString(e.Length)
|
||||
}
|
||||
}
|
||||
|
||||
if rls.TorrentURL == "" && item.Link != "" {
|
||||
rls.TorrentURL = item.Link
|
||||
}
|
||||
|
||||
if rls.TorrentURL != "" {
|
||||
// handle no baseurl with only relative url
|
||||
// grab url from feed url and create full url
|
||||
if parsedURL, _ := url.Parse(rls.TorrentURL); parsedURL != nil && len(parsedURL.Hostname()) == 0 {
|
||||
if parentURL, _ := url.Parse(j.URL); parentURL != nil {
|
||||
parentURL.Path, parentURL.RawPath = "", ""
|
||||
|
||||
// unescape the query params for max compatibility
|
||||
escapedUrl, _ := url.QueryUnescape(parentURL.JoinPath(rls.TorrentURL).String())
|
||||
rls.TorrentURL = escapedUrl
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range item.Categories {
|
||||
if len(rls.Category) != 0 {
|
||||
rls.Category += ", "
|
||||
}
|
||||
|
||||
rls.Category += v
|
||||
}
|
||||
|
||||
for _, v := range item.Authors {
|
||||
if len(rls.Uploader) != 0 {
|
||||
rls.Uploader += ", "
|
||||
}
|
||||
|
||||
rls.Uploader += v.Name
|
||||
}
|
||||
|
||||
if rls.Size == 0 {
|
||||
// parse size bytes string
|
||||
if sz, ok := item.Custom["size"]; ok {
|
||||
rls.ParseSizeBytesString(sz)
|
||||
}
|
||||
}
|
||||
return rls
|
||||
}
|
||||
|
||||
func (j *RSSJob) getFeed() (items []*gofeed.Item, err error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
|
|
|||
110
internal/feed/rss_test.go
Normal file
110
internal/feed/rss_test.go
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
package feed
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/autobrr/autobrr/internal/domain"
|
||||
"github.com/autobrr/autobrr/internal/release"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRSSJob_processItem(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
type fields struct {
|
||||
Name string
|
||||
IndexerIdentifier string
|
||||
Log zerolog.Logger
|
||||
URL string
|
||||
Repo domain.FeedCacheRepo
|
||||
ReleaseSvc release.Service
|
||||
attempts int
|
||||
errors []error
|
||||
JobID int
|
||||
}
|
||||
type args struct {
|
||||
item *gofeed.Item
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
fields fields
|
||||
args args
|
||||
want *domain.Release
|
||||
}{
|
||||
{
|
||||
name: "no_baseurl",
|
||||
fields: fields{
|
||||
Name: "test feed",
|
||||
IndexerIdentifier: "mock-feed",
|
||||
Log: zerolog.Logger{},
|
||||
URL: "https://fake-feed.com/rss",
|
||||
Repo: nil,
|
||||
ReleaseSvc: nil,
|
||||
attempts: 0,
|
||||
errors: nil,
|
||||
JobID: 0,
|
||||
},
|
||||
args: args{item: &gofeed.Item{
|
||||
Title: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP",
|
||||
Description: `Category: Example
|
||||
Size: 1.49 GB
|
||||
Status: 27 seeders and 1 leechers
|
||||
Speed: 772.16 kB/s
|
||||
Added: 2022-09-29 16:06:08
|
||||
`,
|
||||
Link: "/details.php?id=00000&hit=1",
|
||||
GUID: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP",
|
||||
}},
|
||||
want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", TorrentURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 0x0, Title: "Some Release Title", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: "", Proper: false, Repack: false, Website: "", Artists: "", Type: "", LogScore: 0, IsScene: false, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)},
|
||||
},
|
||||
{
|
||||
name: "with_baseurl",
|
||||
fields: fields{
|
||||
Name: "test feed",
|
||||
IndexerIdentifier: "mock-feed",
|
||||
Log: zerolog.Logger{},
|
||||
URL: "https://fake-feed.com/rss",
|
||||
Repo: nil,
|
||||
ReleaseSvc: nil,
|
||||
attempts: 0,
|
||||
errors: nil,
|
||||
JobID: 0,
|
||||
},
|
||||
args: args{item: &gofeed.Item{
|
||||
Title: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP",
|
||||
Description: `Category: Example
|
||||
Size: 1.49 GB
|
||||
Status: 27 seeders and 1 leechers
|
||||
Speed: 772.16 kB/s
|
||||
Added: 2022-09-29 16:06:08
|
||||
`,
|
||||
Link: "https://fake-feed.com/details.php?id=00000&hit=1",
|
||||
GUID: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP",
|
||||
}},
|
||||
want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", TorrentURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 0x0, Title: "Some Release Title", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: "", Proper: false, Repack: false, Website: "", Artists: "", Type: "", LogScore: 0, IsScene: false, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
j := &RSSJob{
|
||||
Name: tt.fields.Name,
|
||||
IndexerIdentifier: tt.fields.IndexerIdentifier,
|
||||
Log: tt.fields.Log,
|
||||
URL: tt.fields.URL,
|
||||
Repo: tt.fields.Repo,
|
||||
ReleaseSvc: tt.fields.ReleaseSvc,
|
||||
attempts: tt.fields.attempts,
|
||||
errors: tt.fields.errors,
|
||||
JobID: tt.fields.JobID,
|
||||
}
|
||||
got := j.processItem(tt.args.item)
|
||||
got.Timestamp = now // override to match
|
||||
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue