feat(feeds): add generic RSS support (#410)

* feat(feeds): add generic rss support

* feat(feeds/web): add generic rss support

* implement rss downloading

* gosum + mod

* re-add size from Custom field.

* implement uploader + category

* sync

* remove double assignment (+torznab)

* didn't save the rss file >.>

* cleanup

* fixfeeds): create rss indexer

* fix(feeds): stop feed

* feat(feeds): support nexusphp rss enclosure link

* feat(feeds): check size for custom size

* fix(feeds): race condition and only stop enabled feeds

* fix(feeds): unify indexer implementation badge

Co-authored-by: Kyle Sanderson <kyle.leet@gmail.com>

internal/domain/feed.go

@@ -50,4 +50,5 @@ type FeedType string
 
 const (
 	FeedTypeTorznab FeedType = "TORZNAB"
+	FeedTypeRSS     FeedType = "RSS"
 )

internal/domain/indexer.go

@@ -44,6 +44,7 @@ type IndexerDefinition struct {
 	SettingsMap    map[string]string `json:"-"`
 	IRC            *IndexerIRC       `json:"irc,omitempty"`
 	Torznab        *Torznab          `json:"torznab,omitempty"`
+	RSS            *FeedSettings     `json:"rss,omitempty"`
 	Parse          *IndexerParse     `json:"parse,omitempty"`
 }
 
@@ -73,6 +74,11 @@ type Torznab struct {
 	Settings    []IndexerSetting `json:"settings"`
 }
 
+type FeedSettings struct {
+	MinInterval int              `json:"minInterval"`
+	Settings    []IndexerSetting `json:"settings"`
+}
+
 type IndexerIRC struct {
 	Network     string            `json:"network"`
 	Server      string            `json:"server"`

internal/domain/release.go

@@ -155,6 +155,7 @@ type ReleaseImplementation string
 const (
 	ReleaseImplementationIRC     ReleaseImplementation = "IRC"
 	ReleaseImplementationTorznab ReleaseImplementation = "TORZNAB"
+	ReleaseImplementationRSS     ReleaseImplementation = "RSS"
 )
 
 type ReleaseQueryParams struct {

internal/feed/rss.go

@@ -0,0 +1,173 @@
+package feed
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	"github.com/autobrr/autobrr/internal/domain"
+	"github.com/autobrr/autobrr/internal/release"
+	"github.com/autobrr/autobrr/pkg/errors"
+
+	"github.com/mmcdole/gofeed"
+	"github.com/rs/zerolog"
+)
+
+type RSSJob struct {
+	Name              string
+	IndexerIdentifier string
+	Log               zerolog.Logger
+	URL               string
+	Repo              domain.FeedCacheRepo
+	ReleaseSvc        release.Service
+
+	attempts int
+	errors   []error
+
+	JobID int
+}
+
+func NewRSSJob(name string, indexerIdentifier string, log zerolog.Logger, url string, repo domain.FeedCacheRepo, releaseSvc release.Service) *RSSJob {
+	return &RSSJob{
+		Name:              name,
+		IndexerIdentifier: indexerIdentifier,
+		Log:               log,
+		URL:               url,
+		Repo:              repo,
+		ReleaseSvc:        releaseSvc,
+	}
+}
+
+func (j *RSSJob) Run() {
+	if err := j.process(); err != nil {
+		j.Log.Err(err).Int("attempts", j.attempts).Msg("rss feed process error")
+
+		j.errors = append(j.errors, err)
+		return
+	}
+
+	j.attempts = 0
+	j.errors = []error{}
+
+	return
+}
+
+func (j *RSSJob) process() error {
+	items, err := j.getFeed()
+	if err != nil {
+		j.Log.Error().Err(err).Msgf("error fetching rss feed items")
+		return errors.Wrap(err, "error getting rss feed items")
+	}
+
+	j.Log.Debug().Msgf("found (%d) new items to process", len(items))
+
+	if len(items) == 0 {
+		return nil
+	}
+
+	releases := make([]*domain.Release, 0)
+
+	for _, item := range items {
+		rls := domain.NewRelease(j.IndexerIdentifier)
+		rls.Implementation = domain.ReleaseImplementationRSS
+
+		rls.ParseString(item.Title)
+
+		if len(item.Enclosures) > 0 {
+			e := item.Enclosures[0]
+			if e.Type == "application/x-bittorrent" && e.URL != "" {
+				rls.TorrentURL = e.URL
+			}
+			if e.Length != "" {
+				rls.ParseSizeBytesString(e.Length)
+			}
+		}
+
+		if rls.TorrentURL == "" && item.Link != "" {
+			rls.TorrentURL = item.Link
+		}
+
+		for _, v := range item.Categories {
+			if len(rls.Category) != 0 {
+				rls.Category += ", "
+			}
+
+			rls.Category += v
+		}
+
+		for _, v := range item.Authors {
+			if len(rls.Uploader) != 0 {
+				rls.Uploader += ", "
+			}
+
+			rls.Uploader += v.Name
+		}
+
+		if rls.Size == 0 {
+			// parse size bytes string
+			if sz, ok := item.Custom["size"]; ok {
+				rls.ParseSizeBytesString(sz)
+			}
+		}
+
+		releases = append(releases, rls)
+	}
+
+	// process all new releases
+	go j.ReleaseSvc.ProcessMultiple(releases)
+
+	return nil
+}
+
+func (j *RSSJob) getFeed() (items []*gofeed.Item, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	feed, err := gofeed.NewParser().ParseURLWithContext(j.URL, ctx) // there's an RSS specific parser as well.
+	if err != nil {
+		j.Log.Error().Err(err).Msgf("error fetching rss feed items")
+		return nil, errors.Wrap(err, "error fetching rss feed items")
+	}
+
+	j.Log.Debug().Msgf("refreshing rss feed: %v, found (%d) items", j.Name, len(feed.Items))
+
+	if len(feed.Items) == 0 {
+		return
+	}
+
+	sort.Sort(feed)
+
+	for _, i := range feed.Items {
+		s := i.GUID
+		if len(s) == 0 {
+			s = i.Title
+			if len(s) == 0 {
+				continue
+			}
+		}
+
+		exists, err := j.Repo.Exists(j.Name, s)
+		if err != nil {
+			j.Log.Error().Err(err).Msg("could not check if item exists")
+			continue
+		}
+		if exists {
+			j.Log.Trace().Msgf("cache item exists, skipping release: %v", i.Title)
+			continue
+		}
+
+		// set ttl to 1 month
+		ttl := time.Now().AddDate(0, 1, 0)
+
+		if err := j.Repo.Put(j.Name, s, []byte(i.Title), ttl); err != nil {
+			j.Log.Error().Stack().Err(err).Str("entry", s).Msg("cache.Put: error storing item in cache")
+			continue
+		}
+
+		// only append if we successfully added to cache
+		items = append(items, i)
+	}
+
+	// send to filters
+	return
+}

internal/feed/service.go

@@ -2,6 +2,7 @@ package feed
 
 import (
 	"context"
+	"time"
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/internal/logger"
@@ -12,7 +13,6 @@ import (
 
 	"github.com/dcarbone/zadapters/zstdlog"
 	"github.com/rs/zerolog"
-	"time"
 )
 
 type Service interface {
@@ -148,9 +148,17 @@ func (s *service) delete(ctx context.Context, id int) error {
 		return err
 	}
 
-	if err := s.stopTorznabJob(f.Indexer); err != nil {
-		s.log.Error().Err(err).Msg("error stopping torznab job")
-		return err
+	switch f.Type {
+	case string(domain.FeedTypeTorznab):
+		if err := s.stopTorznabJob(f.Indexer); err != nil {
+			s.log.Error().Err(err).Msg("error stopping torznab job")
+			return err
+		}
+	case string(domain.FeedTypeRSS):
+		if err := s.stopRSSJob(f.Indexer); err != nil {
+			s.log.Error().Err(err).Msg("error stopping rss job")
+			return err
+		}
 	}
 
 	if err := s.repo.Delete(ctx, id); err != nil {
@@ -169,21 +177,29 @@ func (s *service) delete(ctx context.Context, id int) error {
 }
 
 func (s *service) toggleEnabled(ctx context.Context, id int, enabled bool) error {
-	if err := s.repo.ToggleEnabled(ctx, id, enabled); err != nil {
-		s.log.Error().Err(err).Msg("feed.ToggleEnabled: error toggle enabled")
-		return err
-	}
-
 	f, err := s.repo.FindByID(ctx, id)
 	if err != nil {
 		s.log.Error().Err(err).Msg("feed.ToggleEnabled: error finding feed")
 		return err
 	}
 
-	if !enabled {
-		if err := s.stopTorznabJob(f.Indexer); err != nil {
-			s.log.Error().Err(err).Msg("feed.ToggleEnabled: error stopping torznab job")
-			return err
+	if err := s.repo.ToggleEnabled(ctx, id, enabled); err != nil {
+		s.log.Error().Err(err).Msg("feed.ToggleEnabled: error toggle enabled")
+		return err
+	}
+
+	if f.Enabled && !enabled {
+		switch f.Type {
+		case string(domain.FeedTypeTorznab):
+			if err := s.stopTorznabJob(f.Indexer); err != nil {
+				s.log.Error().Err(err).Msg("feed.ToggleEnabled: error stopping torznab job")
+				return err
+			}
+		case string(domain.FeedTypeRSS):
+			if err := s.stopRSSJob(f.Indexer); err != nil {
+				s.log.Error().Err(err).Msg("feed.ToggleEnabled: error stopping rss job")
+				return err
+			}
 		}
 
 		s.log.Debug().Msgf("feed.ToggleEnabled: stopping feed: %v", f.Name)
@@ -205,17 +221,20 @@ func (s *service) Test(ctx context.Context, feed *domain.Feed) error {
 
 	subLogger := zstdlog.NewStdLoggerWithLevel(s.log.With().Logger(), zerolog.DebugLevel)
 
-	// setup torznab Client
-	c := torznab.NewClient(torznab.Config{Host: feed.URL, ApiKey: feed.ApiKey, Log: subLogger})
-	caps, err := c.GetCaps()
-	if err != nil {
-		s.log.Error().Err(err).Msg("error testing feed")
-		return err
-	}
+	// implementation == TORZNAB
+	if feed.Type == string(domain.FeedTypeTorznab) {
+		// setup torznab Client
+		c := torznab.NewClient(torznab.Config{Host: feed.URL, ApiKey: feed.ApiKey, Log: subLogger})
+		caps, err := c.GetCaps()
+		if err != nil {
+			s.log.Error().Err(err).Msg("error testing feed")
+			return err
+		}
 
-	if caps == nil {
-		s.log.Error().Msg("could not test feed and get caps")
-		return errors.New("could not test feed and get caps")
+		if caps == nil {
+			s.log.Error().Msg("could not test feed and get caps")
+			return errors.New("could not test feed and get caps")
+		}
 	}
 
 	s.log.Debug().Msgf("test successful - connected to feed: %+v", feed.URL)
@@ -286,11 +305,14 @@ func (s *service) startJob(f domain.Feed) error {
 	switch fi.Implementation {
 	case string(domain.FeedTypeTorznab):
 		if err := s.addTorznabJob(fi); err != nil {
-			s.log.Error().Err(err).Msg("feed.startJob: failed to initialize feed")
+			s.log.Error().Err(err).Msg("feed.startJob: failed to initialize torznab feed")
+			return err
+		}
+	case string(domain.FeedTypeRSS):
+		if err := s.addRSSJob(fi); err != nil {
+			s.log.Error().Err(err).Msg("feed.startJob: failed to initialize rss feed")
 			return err
 		}
-		//case "rss":
-
 	}
 
 	return nil
@@ -300,7 +322,7 @@ func (s *service) addTorznabJob(f feedInstance) error {
 	if f.URL == "" {
 		return errors.New("torznab feed requires URL")
 	}
-	if f.CronSchedule < time.Duration(5 * time.Minute) {
+	if f.CronSchedule < time.Duration(5*time.Minute) {
 		f.CronSchedule = time.Duration(15 * time.Minute)
 	}
 
@@ -338,3 +360,43 @@ func (s *service) stopTorznabJob(indexer string) error {
 
 	return nil
 }
+
+func (s *service) addRSSJob(f feedInstance) error {
+	if f.URL == "" {
+		return errors.New("rss feed requires URL")
+	}
+	if f.CronSchedule < time.Duration(5*time.Minute) {
+		f.CronSchedule = time.Duration(15 * time.Minute)
+	}
+
+	// setup logger
+	l := s.log.With().Str("feed", f.Name).Logger()
+
+	// create job
+	job := NewRSSJob(f.Name, f.IndexerIdentifier, l, f.URL, s.cacheRepo, s.releaseSvc)
+
+	// schedule job
+	id, err := s.scheduler.AddJob(job, f.CronSchedule, f.IndexerIdentifier)
+	if err != nil {
+		return errors.Wrap(err, "feed.AddRSSJob: add job failed")
+	}
+	job.JobID = id
+
+	// add to job map
+	s.jobs[f.IndexerIdentifier] = id
+
+	s.log.Debug().Msgf("feed.AddRSSJob: %v", f.Name)
+
+	return nil
+}
+
+func (s *service) stopRSSJob(indexer string) error {
+	// remove job from scheduler
+	if err := s.scheduler.RemoveJobByIdentifier(indexer); err != nil {
+		return errors.Wrap(err, "feed.stopRSSJob: stop job failed")
+	}
+
+	s.log.Debug().Msgf("feed.stopRSSJob: %v", indexer)
+
+	return nil
+}

internal/feed/torznab.go

@@ -73,7 +73,6 @@ func (j *TorznabJob) process() error {
 		rls.TorrentName = item.Title
 		rls.TorrentURL = item.Link
 		rls.Implementation = domain.ReleaseImplementationTorznab
-		rls.Indexer = j.IndexerIdentifier
 
 		// parse size bytes string
 		rls.ParseSizeBytesString(item.Size)

internal/indexer/definitions/rss_generic.yaml

@@ -0,0 +1,22 @@
+---
+#id: rss
+name: Generic RSS
+identifier: rss
+description: Generic RSS
+language: en-us
+urls:
+  - https://domain.com
+privacy: private
+protocol: torrent
+implementation: rss
+supports:
+  - rss
+source: rss
+
+rss:
+  minInterval: 15
+  settings:
+    - name: url
+      type: text
+      required: true
+      label: RSS URL

internal/indexer/service.go

@@ -49,6 +49,8 @@ type service struct {
 	lookupIRCServerDefinition map[string]map[string]*domain.IndexerDefinition
 	// torznab indexers
 	torznabIndexers map[string]*domain.IndexerDefinition
+	// rss indexers
+	rssIndexers map[string]*domain.IndexerDefinition
 }
 
 func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRepo, apiService APIService, scheduler scheduler.Service) Service {
@@ -60,6 +62,7 @@ func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRep
 		scheduler:                 scheduler,
 		lookupIRCServerDefinition: make(map[string]map[string]*domain.IndexerDefinition),
 		torznabIndexers:           make(map[string]*domain.IndexerDefinition),
+		rssIndexers:               make(map[string]*domain.IndexerDefinition),
 		definitions:               make(map[string]domain.IndexerDefinition),
 		mappedDefinitions:         make(map[string]*domain.IndexerDefinition),
 	}
@@ -67,11 +70,17 @@ func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRep
 
 func (s *service) Store(ctx context.Context, indexer domain.Indexer) (*domain.Indexer, error) {
 	identifier := indexer.Identifier
-	//if indexer.Identifier == "torznab" {
-	if indexer.Implementation == "torznab" {
+
+	switch indexer.Implementation {
+	case "torznab":
 		// if the name already contains torznab remove it
 		cleanName := strings.ReplaceAll(strings.ToLower(indexer.Name), "torznab", "")
 		identifier = slug.Make(fmt.Sprintf("%v-%v", indexer.Implementation, cleanName)) // torznab-name
+
+	case "rss":
+		// if the name already contains rss remove it
+		cleanName := strings.ReplaceAll(strings.ToLower(indexer.Name), "rss", "")
+		identifier = slug.Make(fmt.Sprintf("%v-%v", indexer.Implementation, cleanName)) // rss-name
 	}
 
 	indexer.Identifier = identifier
@@ -106,7 +115,7 @@ func (s *service) Update(ctx context.Context, indexer domain.Indexer) (*domain.I
 		return nil, err
 	}
 
-	if indexer.Implementation == "torznab" {
+	if indexer.Implementation == "torznab" || indexer.Implementation == "rss" {
 		if !indexer.Enabled {
 			s.stopFeed(indexer.Identifier)
 		}
@@ -210,6 +219,8 @@ func (s *service) mapIndexer(indexer domain.Indexer) (*domain.IndexerDefinition,
 	definitionName := indexer.Identifier
 	if indexer.Implementation == "torznab" {
 		definitionName = "torznab"
+	} else if indexer.Implementation == "rss" {
+		definitionName = "rss"
 	}
 
 	d := s.getDefinitionByName(definitionName)
@@ -331,6 +342,8 @@ func (s *service) Start() error {
 		// handle Torznab
 		if indexer.Implementation == "torznab" {
 			s.torznabIndexers[indexer.Identifier] = indexer
+		} else if indexer.Implementation == "rss" {
+			s.rssIndexers[indexer.Identifier] = indexer
 		}
 	}
 
@@ -343,6 +356,8 @@ func (s *service) removeIndexer(indexer domain.Indexer) {
 	// remove Torznab
 	if indexer.Implementation == "torznab" {
 		delete(s.torznabIndexers, indexer.Identifier)
+	} else if indexer.Implementation == "rss" {
+		delete(s.rssIndexers, indexer.Identifier)
 	}
 
 	// remove mapped definition
@@ -373,9 +388,11 @@ func (s *service) addIndexer(indexer domain.Indexer) error {
 		}
 	}
 
-	// handle Torznab
+	// handle Torznab and RSS
 	if indexerDefinition.Implementation == "torznab" {
 		s.torznabIndexers[indexer.Identifier] = indexerDefinition
+	} else if indexerDefinition.Implementation == "rss" {
+		s.rssIndexers[indexer.Identifier] = indexerDefinition
 	}
 
 	s.mappedDefinitions[indexer.Identifier] = indexerDefinition
@@ -408,6 +425,8 @@ func (s *service) updateIndexer(indexer domain.Indexer) error {
 	// handle Torznab
 	if indexerDefinition.Implementation == "torznab" {
 		s.torznabIndexers[indexer.Identifier] = indexerDefinition
+	} else if indexerDefinition.Implementation == "rss" {
+		s.rssIndexers[indexer.Identifier] = indexerDefinition
 	}
 
 	s.mappedDefinitions[indexer.Identifier] = indexerDefinition
@@ -570,6 +589,18 @@ func (s *service) GetTorznabIndexers() []domain.IndexerDefinition {
 	return indexerDefinitions
 }
 
+func (s *service) GetRSSIndexers() []domain.IndexerDefinition {
+	indexerDefinitions := make([]domain.IndexerDefinition, 0)
+
+	for _, definition := range s.rssIndexers {
+		if definition != nil {
+			indexerDefinitions = append(indexerDefinitions, *definition)
+		}
+	}
+
+	return indexerDefinitions
+}
+
 func (s *service) getDefinitionByName(name string) *domain.IndexerDefinition {
 	if v, ok := s.definitions[name]; ok {
 		return &v
@@ -582,6 +613,10 @@ func (s *service) stopFeed(indexer string) {
 	// verify indexer is torznab indexer
 	_, ok := s.torznabIndexers[indexer]
 	if !ok {
+		_, rssOK := s.rssIndexers[indexer]
+		if !rssOK {
+			return
+		}
 		return
 	}
 

internal/scheduler/service.go

@@ -1,6 +1,7 @@
 package scheduler
 
 import (
+	"sync"
 	"time"
 
 	"github.com/autobrr/autobrr/internal/logger"
@@ -14,7 +15,6 @@ type Service interface {
 	Start()
 	Stop()
 	AddJob(job cron.Job, interval time.Duration, identifier string) (int, error)
-	RemoveJobByID(id cron.EntryID) error
 	RemoveJobByIdentifier(id string) error
 }
 
@@ -25,6 +25,7 @@ type service struct {
 
 	cron *cron.Cron
 	jobs map[string]cron.EntryID
+	m    sync.RWMutex
 }
 
 func NewService(log logger.Logger, version string, notificationSvc notification.Service) Service {
@@ -62,7 +63,9 @@ func (s *service) addAppJobs() {
 		lastCheckVersion: "",
 	}
 
-	s.AddJob(checkUpdates, time.Duration(36 * time.Hour), "app-check-updates")
+	if id, err := s.AddJob(checkUpdates, time.Duration(36*time.Hour), "app-check-updates"); err != nil {
+		s.log.Error().Err(err).Msgf("scheduler.addAppJobs: error adding job: %v", id)
+	}
 }
 
 func (s *service) Stop() {
@@ -79,23 +82,18 @@ func (s *service) AddJob(job cron.Job, interval time.Duration, identifier string
 
 	s.log.Debug().Msgf("scheduler.AddJob: job successfully added: %v", id)
 
+	s.m.Lock()
 	// add to job map
 	s.jobs[identifier] = id
+	s.m.Unlock()
 
 	return int(id), nil
 }
 
-func (s *service) RemoveJobByID(id cron.EntryID) error {
-	v, ok := s.jobs[""]
-	if !ok {
-		return nil
-	}
-
-	s.cron.Remove(v)
-	return nil
-}
-
 func (s *service) RemoveJobByIdentifier(id string) error {
+	s.m.Lock()
+	defer s.m.Unlock()
+
 	v, ok := s.jobs[id]
 	if !ok {
 		return nil