From 47eaeaa6350a9f1fc1bf4d1df35b7353bfc5cee4 Mon Sep 17 00:00:00 2001 From: Kyle Sanderson Date: Tue, 4 Oct 2022 03:59:29 -0700 Subject: [PATCH] feat(rss): implement relative URL support (#484) * feat(rss): implement relative URL support * bp had some words * feat(gh): uplift golang to 1.19.1 * feat(docker): uplift golang to 1.19 * feat(docker): uplift to go 1.19 * cleanup * chore: update go version in go.mod * feat: refactor and add test feed process item * unescape query params after join * break out logic into separate method * add tests --- .github/workflows/release.yml | 4 +- Dockerfile | 2 +- Dockerfile.ci | 2 +- go.mod | 4 +- internal/feed/rss.go | 102 ++++++++++++++++++------------- internal/feed/rss_test.go | 110 ++++++++++++++++++++++++++++++++++ 6 files changed, 177 insertions(+), 47 deletions(-) create mode 100644 internal/feed/rss_test.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec1c37b..7fe06db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,7 +61,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.18.5' + go-version: '1.19.1' cache: true - name: Run GoReleaser build @@ -99,7 +99,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.18.5' + go-version: '1.19.1' cache: true - name: Run GoReleaser build and publish tags diff --git a/Dockerfile b/Dockerfile index 82d4a77..3b7d0d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ COPY web . RUN yarn build # build app -FROM golang:1.18-alpine3.16 AS app-builder +FROM golang:1.19-alpine3.16 AS app-builder ARG VERSION=dev ARG REVISION=dev diff --git a/Dockerfile.ci b/Dockerfile.ci index 54cf335..594a99e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,5 +1,5 @@ # build app -FROM golang:1.18-alpine3.16 AS app-builder +FROM golang:1.19-alpine3.16 AS app-builder ARG VERSION=dev ARG REVISION=dev diff --git a/go.mod b/go.mod index e583e8c..697729f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/autobrr/autobrr -go 1.18 +go 1.19 require ( github.com/Masterminds/sprig/v3 v3.2.2 @@ -35,6 +35,7 @@ require ( github.com/stretchr/testify v1.8.0 golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 golang.org/x/net v0.0.0-20220909164309-bea034e7d591 + golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 @@ -84,7 +85,6 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect golang.org/x/mod v0.4.1 // indirect - golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect diff --git a/internal/feed/rss.go b/internal/feed/rss.go index 5c17430..fb02dab 100644 --- a/internal/feed/rss.go +++ b/internal/feed/rss.go @@ -2,6 +2,7 @@ package feed import ( "context" + "net/url" "sort" "time" @@ -68,47 +69,7 @@ func (j *RSSJob) process() error { releases := make([]*domain.Release, 0) for _, item := range items { - rls := domain.NewRelease(j.IndexerIdentifier) - rls.Implementation = domain.ReleaseImplementationRSS - - rls.ParseString(item.Title) - - if len(item.Enclosures) > 0 { - e := item.Enclosures[0] - if e.Type == "application/x-bittorrent" && e.URL != "" { - rls.TorrentURL = e.URL - } - if e.Length != "" { - rls.ParseSizeBytesString(e.Length) - } - } - - if rls.TorrentURL == "" && item.Link != "" { - rls.TorrentURL = item.Link - } - - for _, v := range item.Categories { - if len(rls.Category) != 0 { - rls.Category += ", " - } - - rls.Category += v - } - - for _, v := range item.Authors { - if len(rls.Uploader) != 0 { - rls.Uploader += ", " - } - - rls.Uploader += v.Name - } - - if rls.Size == 0 { - // parse size bytes string - if sz, ok := item.Custom["size"]; ok { - rls.ParseSizeBytesString(sz) - } - } + rls := j.processItem(item) releases = append(releases, rls) } @@ -119,6 +80,65 @@ func (j *RSSJob) process() error { return nil } +func (j *RSSJob) processItem(item *gofeed.Item) *domain.Release { + rls := domain.NewRelease(j.IndexerIdentifier) + rls.Implementation = domain.ReleaseImplementationRSS + + rls.ParseString(item.Title) + + if len(item.Enclosures) > 0 { + e := item.Enclosures[0] + if e.Type == "application/x-bittorrent" && e.URL != "" { + rls.TorrentURL = e.URL + } + if e.Length != "" { + rls.ParseSizeBytesString(e.Length) + } + } + + if rls.TorrentURL == "" && item.Link != "" { + rls.TorrentURL = item.Link + } + + if rls.TorrentURL != "" { + // handle no baseurl with only relative url + // grab url from feed url and create full url + if parsedURL, _ := url.Parse(rls.TorrentURL); parsedURL != nil && len(parsedURL.Hostname()) == 0 { + if parentURL, _ := url.Parse(j.URL); parentURL != nil { + parentURL.Path, parentURL.RawPath = "", "" + + // unescape the query params for max compatibility + escapedUrl, _ := url.QueryUnescape(parentURL.JoinPath(rls.TorrentURL).String()) + rls.TorrentURL = escapedUrl + } + } + } + + for _, v := range item.Categories { + if len(rls.Category) != 0 { + rls.Category += ", " + } + + rls.Category += v + } + + for _, v := range item.Authors { + if len(rls.Uploader) != 0 { + rls.Uploader += ", " + } + + rls.Uploader += v.Name + } + + if rls.Size == 0 { + // parse size bytes string + if sz, ok := item.Custom["size"]; ok { + rls.ParseSizeBytesString(sz) + } + } + return rls +} + func (j *RSSJob) getFeed() (items []*gofeed.Item, err error) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() diff --git a/internal/feed/rss_test.go b/internal/feed/rss_test.go new file mode 100644 index 0000000..2739c85 --- /dev/null +++ b/internal/feed/rss_test.go @@ -0,0 +1,110 @@ +package feed + +import ( + "testing" + "time" + + "github.com/autobrr/autobrr/internal/domain" + "github.com/autobrr/autobrr/internal/release" + + "github.com/mmcdole/gofeed" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" +) + +func TestRSSJob_processItem(t *testing.T) { + now := time.Now() + + type fields struct { + Name string + IndexerIdentifier string + Log zerolog.Logger + URL string + Repo domain.FeedCacheRepo + ReleaseSvc release.Service + attempts int + errors []error + JobID int + } + type args struct { + item *gofeed.Item + } + tests := []struct { + name string + fields fields + args args + want *domain.Release + }{ + { + name: "no_baseurl", + fields: fields{ + Name: "test feed", + IndexerIdentifier: "mock-feed", + Log: zerolog.Logger{}, + URL: "https://fake-feed.com/rss", + Repo: nil, + ReleaseSvc: nil, + attempts: 0, + errors: nil, + JobID: 0, + }, + args: args{item: &gofeed.Item{ + Title: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", + Description: `Category: Example + Size: 1.49 GB + Status: 27 seeders and 1 leechers + Speed: 772.16 kB/s + Added: 2022-09-29 16:06:08 +`, + Link: "/details.php?id=00000&hit=1", + GUID: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", + }}, + want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", TorrentURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 0x0, Title: "Some Release Title", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: "", Proper: false, Repack: false, Website: "", Artists: "", Type: "", LogScore: 0, IsScene: false, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, + }, + { + name: "with_baseurl", + fields: fields{ + Name: "test feed", + IndexerIdentifier: "mock-feed", + Log: zerolog.Logger{}, + URL: "https://fake-feed.com/rss", + Repo: nil, + ReleaseSvc: nil, + attempts: 0, + errors: nil, + JobID: 0, + }, + args: args{item: &gofeed.Item{ + Title: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", + Description: `Category: Example + Size: 1.49 GB + Status: 27 seeders and 1 leechers + Speed: 772.16 kB/s + Added: 2022-09-29 16:06:08 +`, + Link: "https://fake-feed.com/details.php?id=00000&hit=1", + GUID: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", + }}, + want: &domain.Release{ID: 0, FilterStatus: "PENDING", Rejections: []string{}, Indexer: "mock-feed", FilterName: "", Protocol: "torrent", Implementation: "RSS", Timestamp: now, GroupID: "", TorrentID: "", TorrentURL: "https://fake-feed.com/details.php?id=00000&hit=1", TorrentTmpFile: "", TorrentDataRawBytes: []uint8(nil), TorrentHash: "", TorrentName: "Some.Release.Title.2022.09.22.720p.WEB.h264-GROUP", Size: 0x0, Title: "Some Release Title", Category: "", Season: 0, Episode: 0, Year: 2022, Resolution: "720p", Source: "WEB", Codec: []string{"H.264"}, Container: "", HDR: []string(nil), Audio: []string(nil), AudioChannels: "", Group: "GROUP", Region: "", Language: "", Proper: false, Repack: false, Website: "", Artists: "", Type: "", LogScore: 0, IsScene: false, Origin: "", Tags: []string{}, ReleaseTags: "", Freeleech: false, FreeleechPercent: 0, Bonus: []string(nil), Uploader: "", PreTime: "", Other: []string(nil), RawCookie: "", AdditionalSizeCheckRequired: false, FilterID: 0, Filter: (*domain.Filter)(nil), ActionStatus: []domain.ReleaseActionStatus(nil)}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + j := &RSSJob{ + Name: tt.fields.Name, + IndexerIdentifier: tt.fields.IndexerIdentifier, + Log: tt.fields.Log, + URL: tt.fields.URL, + Repo: tt.fields.Repo, + ReleaseSvc: tt.fields.ReleaseSvc, + attempts: tt.fields.attempts, + errors: tt.fields.errors, + JobID: tt.fields.JobID, + } + got := j.processItem(tt.args.item) + got.Timestamp = now // override to match + + assert.Equal(t, tt.want, got) + }) + } +}