From b50688159ea89d1baa90331ff293a0e0eb541f2c Mon Sep 17 00:00:00 2001 From: ze0s <43699394+zze0s@users.noreply.github.com> Date: Sat, 20 Aug 2022 00:34:46 +0200 Subject: [PATCH] feat(feeds): add generic RSS support (#410) * feat(feeds): add generic rss support * feat(feeds/web): add generic rss support * implement rss downloading * gosum + mod * re-add size from Custom field. * implement uploader + category * sync * remove double assignment (+torznab) * didn't save the rss file >.> * cleanup * fixfeeds): create rss indexer * fix(feeds): stop feed * feat(feeds): support nexusphp rss enclosure link * feat(feeds): check size for custom size * fix(feeds): race condition and only stop enabled feeds * fix(feeds): unify indexer implementation badge Co-authored-by: Kyle Sanderson --- go.mod | 7 + go.sum | 19 ++ internal/domain/feed.go | 1 + internal/domain/indexer.go | 6 + internal/domain/release.go | 1 + internal/feed/rss.go | 173 ++++++++++++++++++ internal/feed/service.go | 116 +++++++++--- internal/feed/torznab.go | 1 - internal/indexer/definitions/rss_generic.yaml | 22 +++ internal/indexer/service.go | 43 ++++- internal/scheduler/service.go | 22 +-- web/src/forms/settings/FeedForms.tsx | 23 ++- web/src/forms/settings/IndexerForms.tsx | 68 ++++++- web/src/screens/settings/Feed.tsx | 38 ++-- web/src/screens/settings/Indexer.tsx | 33 ++-- web/src/types/Feed.d.ts | 8 +- web/src/types/Indexer.d.ts | 6 + 17 files changed, 498 insertions(+), 89 deletions(-) create mode 100644 internal/feed/rss.go create mode 100644 internal/indexer/definitions/rss_generic.yaml diff --git a/go.mod b/go.mod index 63a6383..5b89ca5 100644 --- a/go.mod +++ b/go.mod @@ -39,10 +39,12 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.5.1 // indirect github.com/ajg/form v1.5.1 // indirect github.com/anacrolix/dht/v2 v2.18.0 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/v2 v2.7.0 // indirect + github.com/andybalholm/cascadia v1.1.0 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gdm85/go-rencode v0.1.8 // indirect @@ -53,6 +55,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -60,6 +63,10 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcdole/gofeed v1.1.3 // indirect + github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect diff --git a/go.sum b/go.sum index b4ff6d8..26c6e42 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvdHpkZc= github.com/Masterminds/squirrel v1.5.3/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= @@ -77,6 +79,8 @@ github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pm github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= github.com/anacrolix/torrent v1.46.0 h1:m5SGlW4p0dJkqrIh4bCqzcKbKFFmfJorf9LSpSM5dEc= github.com/anacrolix/torrent v1.46.0/go.mod h1:3DE+VA4AgyfKDPjZcIo70D3VFZRo3bfdEBn70CGjca4= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= @@ -101,6 +105,7 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cytec/releaseparser v0.0.0-20200706155913-2341b265c370 h1:g9q5BGfDdhcXn4EmVZD8UydPXrvhSvgz3FRBn7zAJNs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -249,6 +254,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -284,10 +292,17 @@ github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= +github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moistari/rls v0.2.6 h1:0PfId/3t2hC+uSKVvtvlH7UO6k4YLu8/tHR10dgcplQ= github.com/moistari/rls v0.2.6/go.mod h1:2oVpWLhkuUzu2xqRINGnvvlcAGizZGMfMv8UYnntUCg= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -345,9 +360,11 @@ github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -379,6 +396,7 @@ github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -438,6 +456,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/internal/domain/feed.go b/internal/domain/feed.go index e255a26..97dff14 100644 --- a/internal/domain/feed.go +++ b/internal/domain/feed.go @@ -50,4 +50,5 @@ type FeedType string const ( FeedTypeTorznab FeedType = "TORZNAB" + FeedTypeRSS FeedType = "RSS" ) diff --git a/internal/domain/indexer.go b/internal/domain/indexer.go index 1659f04..9c48485 100644 --- a/internal/domain/indexer.go +++ b/internal/domain/indexer.go @@ -44,6 +44,7 @@ type IndexerDefinition struct { SettingsMap map[string]string `json:"-"` IRC *IndexerIRC `json:"irc,omitempty"` Torznab *Torznab `json:"torznab,omitempty"` + RSS *FeedSettings `json:"rss,omitempty"` Parse *IndexerParse `json:"parse,omitempty"` } @@ -73,6 +74,11 @@ type Torznab struct { Settings []IndexerSetting `json:"settings"` } +type FeedSettings struct { + MinInterval int `json:"minInterval"` + Settings []IndexerSetting `json:"settings"` +} + type IndexerIRC struct { Network string `json:"network"` Server string `json:"server"` diff --git a/internal/domain/release.go b/internal/domain/release.go index a3775cd..caf3c46 100644 --- a/internal/domain/release.go +++ b/internal/domain/release.go @@ -155,6 +155,7 @@ type ReleaseImplementation string const ( ReleaseImplementationIRC ReleaseImplementation = "IRC" ReleaseImplementationTorznab ReleaseImplementation = "TORZNAB" + ReleaseImplementationRSS ReleaseImplementation = "RSS" ) type ReleaseQueryParams struct { diff --git a/internal/feed/rss.go b/internal/feed/rss.go new file mode 100644 index 0000000..5c17430 --- /dev/null +++ b/internal/feed/rss.go @@ -0,0 +1,173 @@ +package feed + +import ( + "context" + "sort" + "time" + + "github.com/autobrr/autobrr/internal/domain" + "github.com/autobrr/autobrr/internal/release" + "github.com/autobrr/autobrr/pkg/errors" + + "github.com/mmcdole/gofeed" + "github.com/rs/zerolog" +) + +type RSSJob struct { + Name string + IndexerIdentifier string + Log zerolog.Logger + URL string + Repo domain.FeedCacheRepo + ReleaseSvc release.Service + + attempts int + errors []error + + JobID int +} + +func NewRSSJob(name string, indexerIdentifier string, log zerolog.Logger, url string, repo domain.FeedCacheRepo, releaseSvc release.Service) *RSSJob { + return &RSSJob{ + Name: name, + IndexerIdentifier: indexerIdentifier, + Log: log, + URL: url, + Repo: repo, + ReleaseSvc: releaseSvc, + } +} + +func (j *RSSJob) Run() { + if err := j.process(); err != nil { + j.Log.Err(err).Int("attempts", j.attempts).Msg("rss feed process error") + + j.errors = append(j.errors, err) + return + } + + j.attempts = 0 + j.errors = []error{} + + return +} + +func (j *RSSJob) process() error { + items, err := j.getFeed() + if err != nil { + j.Log.Error().Err(err).Msgf("error fetching rss feed items") + return errors.Wrap(err, "error getting rss feed items") + } + + j.Log.Debug().Msgf("found (%d) new items to process", len(items)) + + if len(items) == 0 { + return nil + } + + releases := make([]*domain.Release, 0) + + for _, item := range items { + rls := domain.NewRelease(j.IndexerIdentifier) + rls.Implementation = domain.ReleaseImplementationRSS + + rls.ParseString(item.Title) + + if len(item.Enclosures) > 0 { + e := item.Enclosures[0] + if e.Type == "application/x-bittorrent" && e.URL != "" { + rls.TorrentURL = e.URL + } + if e.Length != "" { + rls.ParseSizeBytesString(e.Length) + } + } + + if rls.TorrentURL == "" && item.Link != "" { + rls.TorrentURL = item.Link + } + + for _, v := range item.Categories { + if len(rls.Category) != 0 { + rls.Category += ", " + } + + rls.Category += v + } + + for _, v := range item.Authors { + if len(rls.Uploader) != 0 { + rls.Uploader += ", " + } + + rls.Uploader += v.Name + } + + if rls.Size == 0 { + // parse size bytes string + if sz, ok := item.Custom["size"]; ok { + rls.ParseSizeBytesString(sz) + } + } + + releases = append(releases, rls) + } + + // process all new releases + go j.ReleaseSvc.ProcessMultiple(releases) + + return nil +} + +func (j *RSSJob) getFeed() (items []*gofeed.Item, err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + feed, err := gofeed.NewParser().ParseURLWithContext(j.URL, ctx) // there's an RSS specific parser as well. + if err != nil { + j.Log.Error().Err(err).Msgf("error fetching rss feed items") + return nil, errors.Wrap(err, "error fetching rss feed items") + } + + j.Log.Debug().Msgf("refreshing rss feed: %v, found (%d) items", j.Name, len(feed.Items)) + + if len(feed.Items) == 0 { + return + } + + sort.Sort(feed) + + for _, i := range feed.Items { + s := i.GUID + if len(s) == 0 { + s = i.Title + if len(s) == 0 { + continue + } + } + + exists, err := j.Repo.Exists(j.Name, s) + if err != nil { + j.Log.Error().Err(err).Msg("could not check if item exists") + continue + } + if exists { + j.Log.Trace().Msgf("cache item exists, skipping release: %v", i.Title) + continue + } + + // set ttl to 1 month + ttl := time.Now().AddDate(0, 1, 0) + + if err := j.Repo.Put(j.Name, s, []byte(i.Title), ttl); err != nil { + j.Log.Error().Stack().Err(err).Str("entry", s).Msg("cache.Put: error storing item in cache") + continue + } + + // only append if we successfully added to cache + items = append(items, i) + } + + // send to filters + return +} diff --git a/internal/feed/service.go b/internal/feed/service.go index dfa39a9..5d83236 100644 --- a/internal/feed/service.go +++ b/internal/feed/service.go @@ -2,6 +2,7 @@ package feed import ( "context" + "time" "github.com/autobrr/autobrr/internal/domain" "github.com/autobrr/autobrr/internal/logger" @@ -12,7 +13,6 @@ import ( "github.com/dcarbone/zadapters/zstdlog" "github.com/rs/zerolog" - "time" ) type Service interface { @@ -148,9 +148,17 @@ func (s *service) delete(ctx context.Context, id int) error { return err } - if err := s.stopTorznabJob(f.Indexer); err != nil { - s.log.Error().Err(err).Msg("error stopping torznab job") - return err + switch f.Type { + case string(domain.FeedTypeTorznab): + if err := s.stopTorznabJob(f.Indexer); err != nil { + s.log.Error().Err(err).Msg("error stopping torznab job") + return err + } + case string(domain.FeedTypeRSS): + if err := s.stopRSSJob(f.Indexer); err != nil { + s.log.Error().Err(err).Msg("error stopping rss job") + return err + } } if err := s.repo.Delete(ctx, id); err != nil { @@ -169,21 +177,29 @@ func (s *service) delete(ctx context.Context, id int) error { } func (s *service) toggleEnabled(ctx context.Context, id int, enabled bool) error { - if err := s.repo.ToggleEnabled(ctx, id, enabled); err != nil { - s.log.Error().Err(err).Msg("feed.ToggleEnabled: error toggle enabled") - return err - } - f, err := s.repo.FindByID(ctx, id) if err != nil { s.log.Error().Err(err).Msg("feed.ToggleEnabled: error finding feed") return err } - if !enabled { - if err := s.stopTorznabJob(f.Indexer); err != nil { - s.log.Error().Err(err).Msg("feed.ToggleEnabled: error stopping torznab job") - return err + if err := s.repo.ToggleEnabled(ctx, id, enabled); err != nil { + s.log.Error().Err(err).Msg("feed.ToggleEnabled: error toggle enabled") + return err + } + + if f.Enabled && !enabled { + switch f.Type { + case string(domain.FeedTypeTorznab): + if err := s.stopTorznabJob(f.Indexer); err != nil { + s.log.Error().Err(err).Msg("feed.ToggleEnabled: error stopping torznab job") + return err + } + case string(domain.FeedTypeRSS): + if err := s.stopRSSJob(f.Indexer); err != nil { + s.log.Error().Err(err).Msg("feed.ToggleEnabled: error stopping rss job") + return err + } } s.log.Debug().Msgf("feed.ToggleEnabled: stopping feed: %v", f.Name) @@ -205,17 +221,20 @@ func (s *service) Test(ctx context.Context, feed *domain.Feed) error { subLogger := zstdlog.NewStdLoggerWithLevel(s.log.With().Logger(), zerolog.DebugLevel) - // setup torznab Client - c := torznab.NewClient(torznab.Config{Host: feed.URL, ApiKey: feed.ApiKey, Log: subLogger}) - caps, err := c.GetCaps() - if err != nil { - s.log.Error().Err(err).Msg("error testing feed") - return err - } + // implementation == TORZNAB + if feed.Type == string(domain.FeedTypeTorznab) { + // setup torznab Client + c := torznab.NewClient(torznab.Config{Host: feed.URL, ApiKey: feed.ApiKey, Log: subLogger}) + caps, err := c.GetCaps() + if err != nil { + s.log.Error().Err(err).Msg("error testing feed") + return err + } - if caps == nil { - s.log.Error().Msg("could not test feed and get caps") - return errors.New("could not test feed and get caps") + if caps == nil { + s.log.Error().Msg("could not test feed and get caps") + return errors.New("could not test feed and get caps") + } } s.log.Debug().Msgf("test successful - connected to feed: %+v", feed.URL) @@ -286,11 +305,14 @@ func (s *service) startJob(f domain.Feed) error { switch fi.Implementation { case string(domain.FeedTypeTorznab): if err := s.addTorznabJob(fi); err != nil { - s.log.Error().Err(err).Msg("feed.startJob: failed to initialize feed") + s.log.Error().Err(err).Msg("feed.startJob: failed to initialize torznab feed") + return err + } + case string(domain.FeedTypeRSS): + if err := s.addRSSJob(fi); err != nil { + s.log.Error().Err(err).Msg("feed.startJob: failed to initialize rss feed") return err } - //case "rss": - } return nil @@ -300,7 +322,7 @@ func (s *service) addTorznabJob(f feedInstance) error { if f.URL == "" { return errors.New("torznab feed requires URL") } - if f.CronSchedule < time.Duration(5 * time.Minute) { + if f.CronSchedule < time.Duration(5*time.Minute) { f.CronSchedule = time.Duration(15 * time.Minute) } @@ -338,3 +360,43 @@ func (s *service) stopTorznabJob(indexer string) error { return nil } + +func (s *service) addRSSJob(f feedInstance) error { + if f.URL == "" { + return errors.New("rss feed requires URL") + } + if f.CronSchedule < time.Duration(5*time.Minute) { + f.CronSchedule = time.Duration(15 * time.Minute) + } + + // setup logger + l := s.log.With().Str("feed", f.Name).Logger() + + // create job + job := NewRSSJob(f.Name, f.IndexerIdentifier, l, f.URL, s.cacheRepo, s.releaseSvc) + + // schedule job + id, err := s.scheduler.AddJob(job, f.CronSchedule, f.IndexerIdentifier) + if err != nil { + return errors.Wrap(err, "feed.AddRSSJob: add job failed") + } + job.JobID = id + + // add to job map + s.jobs[f.IndexerIdentifier] = id + + s.log.Debug().Msgf("feed.AddRSSJob: %v", f.Name) + + return nil +} + +func (s *service) stopRSSJob(indexer string) error { + // remove job from scheduler + if err := s.scheduler.RemoveJobByIdentifier(indexer); err != nil { + return errors.Wrap(err, "feed.stopRSSJob: stop job failed") + } + + s.log.Debug().Msgf("feed.stopRSSJob: %v", indexer) + + return nil +} diff --git a/internal/feed/torznab.go b/internal/feed/torznab.go index 1d50a60..2e59e28 100644 --- a/internal/feed/torznab.go +++ b/internal/feed/torznab.go @@ -73,7 +73,6 @@ func (j *TorznabJob) process() error { rls.TorrentName = item.Title rls.TorrentURL = item.Link rls.Implementation = domain.ReleaseImplementationTorznab - rls.Indexer = j.IndexerIdentifier // parse size bytes string rls.ParseSizeBytesString(item.Size) diff --git a/internal/indexer/definitions/rss_generic.yaml b/internal/indexer/definitions/rss_generic.yaml new file mode 100644 index 0000000..eee6c96 --- /dev/null +++ b/internal/indexer/definitions/rss_generic.yaml @@ -0,0 +1,22 @@ +--- +#id: rss +name: Generic RSS +identifier: rss +description: Generic RSS +language: en-us +urls: + - https://domain.com +privacy: private +protocol: torrent +implementation: rss +supports: + - rss +source: rss + +rss: + minInterval: 15 + settings: + - name: url + type: text + required: true + label: RSS URL \ No newline at end of file diff --git a/internal/indexer/service.go b/internal/indexer/service.go index 5756700..7d9e2a9 100644 --- a/internal/indexer/service.go +++ b/internal/indexer/service.go @@ -49,6 +49,8 @@ type service struct { lookupIRCServerDefinition map[string]map[string]*domain.IndexerDefinition // torznab indexers torznabIndexers map[string]*domain.IndexerDefinition + // rss indexers + rssIndexers map[string]*domain.IndexerDefinition } func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRepo, apiService APIService, scheduler scheduler.Service) Service { @@ -60,6 +62,7 @@ func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRep scheduler: scheduler, lookupIRCServerDefinition: make(map[string]map[string]*domain.IndexerDefinition), torznabIndexers: make(map[string]*domain.IndexerDefinition), + rssIndexers: make(map[string]*domain.IndexerDefinition), definitions: make(map[string]domain.IndexerDefinition), mappedDefinitions: make(map[string]*domain.IndexerDefinition), } @@ -67,11 +70,17 @@ func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRep func (s *service) Store(ctx context.Context, indexer domain.Indexer) (*domain.Indexer, error) { identifier := indexer.Identifier - //if indexer.Identifier == "torznab" { - if indexer.Implementation == "torznab" { + + switch indexer.Implementation { + case "torznab": // if the name already contains torznab remove it cleanName := strings.ReplaceAll(strings.ToLower(indexer.Name), "torznab", "") identifier = slug.Make(fmt.Sprintf("%v-%v", indexer.Implementation, cleanName)) // torznab-name + + case "rss": + // if the name already contains rss remove it + cleanName := strings.ReplaceAll(strings.ToLower(indexer.Name), "rss", "") + identifier = slug.Make(fmt.Sprintf("%v-%v", indexer.Implementation, cleanName)) // rss-name } indexer.Identifier = identifier @@ -106,7 +115,7 @@ func (s *service) Update(ctx context.Context, indexer domain.Indexer) (*domain.I return nil, err } - if indexer.Implementation == "torznab" { + if indexer.Implementation == "torznab" || indexer.Implementation == "rss" { if !indexer.Enabled { s.stopFeed(indexer.Identifier) } @@ -210,6 +219,8 @@ func (s *service) mapIndexer(indexer domain.Indexer) (*domain.IndexerDefinition, definitionName := indexer.Identifier if indexer.Implementation == "torznab" { definitionName = "torznab" + } else if indexer.Implementation == "rss" { + definitionName = "rss" } d := s.getDefinitionByName(definitionName) @@ -331,6 +342,8 @@ func (s *service) Start() error { // handle Torznab if indexer.Implementation == "torznab" { s.torznabIndexers[indexer.Identifier] = indexer + } else if indexer.Implementation == "rss" { + s.rssIndexers[indexer.Identifier] = indexer } } @@ -343,6 +356,8 @@ func (s *service) removeIndexer(indexer domain.Indexer) { // remove Torznab if indexer.Implementation == "torznab" { delete(s.torznabIndexers, indexer.Identifier) + } else if indexer.Implementation == "rss" { + delete(s.rssIndexers, indexer.Identifier) } // remove mapped definition @@ -373,9 +388,11 @@ func (s *service) addIndexer(indexer domain.Indexer) error { } } - // handle Torznab + // handle Torznab and RSS if indexerDefinition.Implementation == "torznab" { s.torznabIndexers[indexer.Identifier] = indexerDefinition + } else if indexerDefinition.Implementation == "rss" { + s.rssIndexers[indexer.Identifier] = indexerDefinition } s.mappedDefinitions[indexer.Identifier] = indexerDefinition @@ -408,6 +425,8 @@ func (s *service) updateIndexer(indexer domain.Indexer) error { // handle Torznab if indexerDefinition.Implementation == "torznab" { s.torznabIndexers[indexer.Identifier] = indexerDefinition + } else if indexerDefinition.Implementation == "rss" { + s.rssIndexers[indexer.Identifier] = indexerDefinition } s.mappedDefinitions[indexer.Identifier] = indexerDefinition @@ -570,6 +589,18 @@ func (s *service) GetTorznabIndexers() []domain.IndexerDefinition { return indexerDefinitions } +func (s *service) GetRSSIndexers() []domain.IndexerDefinition { + indexerDefinitions := make([]domain.IndexerDefinition, 0) + + for _, definition := range s.rssIndexers { + if definition != nil { + indexerDefinitions = append(indexerDefinitions, *definition) + } + } + + return indexerDefinitions +} + func (s *service) getDefinitionByName(name string) *domain.IndexerDefinition { if v, ok := s.definitions[name]; ok { return &v @@ -582,6 +613,10 @@ func (s *service) stopFeed(indexer string) { // verify indexer is torznab indexer _, ok := s.torznabIndexers[indexer] if !ok { + _, rssOK := s.rssIndexers[indexer] + if !rssOK { + return + } return } diff --git a/internal/scheduler/service.go b/internal/scheduler/service.go index 9df58a1..61841f4 100644 --- a/internal/scheduler/service.go +++ b/internal/scheduler/service.go @@ -1,6 +1,7 @@ package scheduler import ( + "sync" "time" "github.com/autobrr/autobrr/internal/logger" @@ -14,7 +15,6 @@ type Service interface { Start() Stop() AddJob(job cron.Job, interval time.Duration, identifier string) (int, error) - RemoveJobByID(id cron.EntryID) error RemoveJobByIdentifier(id string) error } @@ -25,6 +25,7 @@ type service struct { cron *cron.Cron jobs map[string]cron.EntryID + m sync.RWMutex } func NewService(log logger.Logger, version string, notificationSvc notification.Service) Service { @@ -62,7 +63,9 @@ func (s *service) addAppJobs() { lastCheckVersion: "", } - s.AddJob(checkUpdates, time.Duration(36 * time.Hour), "app-check-updates") + if id, err := s.AddJob(checkUpdates, time.Duration(36*time.Hour), "app-check-updates"); err != nil { + s.log.Error().Err(err).Msgf("scheduler.addAppJobs: error adding job: %v", id) + } } func (s *service) Stop() { @@ -79,23 +82,18 @@ func (s *service) AddJob(job cron.Job, interval time.Duration, identifier string s.log.Debug().Msgf("scheduler.AddJob: job successfully added: %v", id) + s.m.Lock() // add to job map s.jobs[identifier] = id + s.m.Unlock() return int(id), nil } -func (s *service) RemoveJobByID(id cron.EntryID) error { - v, ok := s.jobs[""] - if !ok { - return nil - } - - s.cron.Remove(v) - return nil -} - func (s *service) RemoveJobByIdentifier(id string) error { + s.m.Lock() + defer s.m.Unlock() + v, ok := s.jobs[id] if !ok { return nil diff --git a/web/src/forms/settings/FeedForms.tsx b/web/src/forms/settings/FeedForms.tsx index 6cf8f1e..2a3cd20 100644 --- a/web/src/forms/settings/FeedForms.tsx +++ b/web/src/forms/settings/FeedForms.tsx @@ -5,10 +5,10 @@ import { toast } from "react-hot-toast"; import Toast from "../../components/notifications/Toast"; import { SlideOver } from "../../components/panels"; import { NumberFieldWide, PasswordFieldWide, SwitchGroupWide, TextFieldWide } from "../../components/inputs"; -import { ImplementationMap } from "../../screens/settings/Feed"; import { componentMapType } from "./DownloadClientForms"; import { sleep } from "../../utils"; import { useState } from "react"; +import { ImplementationBadges } from "../../screens/settings/Indexer"; interface UpdateProps { isOpen: boolean; @@ -126,11 +126,11 @@ export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) {
- {ImplementationMap[feed.type]} + {ImplementationBadges[feed.type.toLowerCase()]}
-
+
@@ -158,6 +158,21 @@ function FormFieldsTorznab() { ); } +function FormFieldsRSS() { + return ( +
+ + + +
+ ); +} + const componentMap: componentMapType = { - TORZNAB: + TORZNAB: , + RSS: }; \ No newline at end of file diff --git a/web/src/forms/settings/IndexerForms.tsx b/web/src/forms/settings/IndexerForms.tsx index 76c4bc6..3caad2d 100644 --- a/web/src/forms/settings/IndexerForms.tsx +++ b/web/src/forms/settings/IndexerForms.tsx @@ -119,6 +119,37 @@ const FeedSettingFields = (ind: IndexerDefinition, indexer: string) => { } }; +const RSSFeedSettingFields = (ind: IndexerDefinition, indexer: string) => { + if (indexer !== "") { + return ( + + {ind && ind.rss && ind.rss.settings && ( +
+
+ RSS +

+ RSS feed +

+
+ + + + {ind.rss.settings.map((f: IndexerSetting, idx: number) => { + switch (f.type) { + case "text": + return ; + case "secret": + return ; + } + return null; + })} +
+ )} + + ); + } +}; + const SettingFields = (ind: IndexerDefinition, indexer: string) => { if (indexer !== "") { return ( @@ -144,10 +175,13 @@ const SettingFields = (ind: IndexerDefinition, indexer: string) => { } }; -function slugIdentifier(name: string) { +function slugIdentifier(name: string, prefix?: string) { const l = name.toLowerCase(); - const r = l.replaceAll("torznab", ""); - return slugify(`torznab-${r}`); + if (prefix && prefix !== "") { + const r = l.replaceAll(prefix, ""); + return slugify(`${prefix}-${r}`); + } + return slugify(l); } // interface initialValues { @@ -210,7 +244,7 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) { if (formData.implementation === "torznab") { // create slug for indexer identifier as "torznab-indexer_name" - const name = slugIdentifier(formData.name); + const name = slugIdentifier(formData.name, "torznab"); const createFeed: FeedCreate = { name: formData.name, @@ -234,6 +268,31 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) { return; } + if (formData.implementation === "rss") { + // create slug for indexer identifier as "torznab-indexer_name" + const name = slugIdentifier(formData.name, "rss"); + + const createFeed: FeedCreate = { + name: formData.name, + enabled: false, + type: "RSS", + url: formData.feed.url, + interval: 30, + indexer: name, + indexer_id: 0 + }; + + mutation.mutate(formData as Indexer, { + onSuccess: (indexer) => { + // @eslint-ignore + createFeed.indexer_id = indexer.id; + + feedMutation.mutate(createFeed); + } + }); + return; + } + if (formData.implementation === "irc") { const channels: IrcChannel[] = []; @@ -399,6 +458,7 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) { {IrcSettingFields(indexer, values.identifier)} {FeedSettingFields(indexer, values.identifier)} + {RSSFeedSettingFields(indexer, values.identifier)}
Enabled
Name + className="col-span-4 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Name
Type + className="col-span-3 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Indexer +
+
Type
{/*
Events
*/} @@ -65,18 +68,6 @@ function FeedSettings() { ); } -const ImplementationTorznab = () => ( - - Torznab - -); - -export const ImplementationMap: componentMapType = { - "TORZNAB": -}; - interface ListItemProps { feed: Feed; } @@ -129,13 +120,16 @@ function ListItem({ feed }: ListItemProps) { /> -
+
{feed.name}
-
- {ImplementationMap[feed.type]} +
+ {feed.indexer}
-
+
+ {ImplementationBadges[feed.type.toLowerCase()]} +
+
@@ -117,19 +126,19 @@ function IndexerSettings() { scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider" > - Enabled + Enabled - Name + Name - Implementation + Implementation Edit diff --git a/web/src/types/Feed.d.ts b/web/src/types/Feed.d.ts index 436898f..bda7158 100644 --- a/web/src/types/Feed.d.ts +++ b/web/src/types/Feed.d.ts @@ -2,7 +2,7 @@ interface Feed { id: number; indexer: string; name: string; - type: string; + type: FeedType; enabled: boolean; url: string; interval: number; @@ -11,13 +11,15 @@ interface Feed { updated_at: Date; } +type FeedType = "TORZNAB" | "RSS"; + interface FeedCreate { indexer: string; name: string; - type: string; + type: FeedType; enabled: boolean; url: string; interval: number; - api_key: string; + api_key?: string; indexer_id: number; } diff --git a/web/src/types/Indexer.d.ts b/web/src/types/Indexer.d.ts index f367216..b529f94 100644 --- a/web/src/types/Indexer.d.ts +++ b/web/src/types/Indexer.d.ts @@ -22,6 +22,7 @@ interface IndexerDefinition { settings: IndexerSetting[]; irc: IndexerIRC; torznab: IndexerTorznab; + rss: IndexerFeed; parse: IndexerParse; } @@ -53,6 +54,11 @@ interface IndexerTorznab { settings: IndexerSetting[]; } +interface IndexerFeed { + minInterval: number; + settings: IndexerSetting[]; +} + interface IndexerParse { type: string; lines: IndexerParseLines[];