feat: add usenet support (#543)

* feat(autobrr): implement usenet support

* feat(sonarr): implement usenet support

* feat(radarr): implement usenet support

* feat(announce): implement usenet support

* announce: cast a line

* feat(release): prevent unknown protocol transfer

* release: lines for days.

* feat: add newznab and sabnzbd support

* feat: add category to sabnzbd

* feat(newznab): map categories

* feat(newznab): map categories

---------

Co-authored-by: ze0s <43699394+zze0s@users.noreply.github.com>
Co-authored-by: ze0s <ze0s@riseup.net>
This commit is contained in:
Kyle Sanderson 2023-03-04 11:27:18 -08:00 committed by GitHub
parent b2d93d50c5
commit 13a74f7cc8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 1588 additions and 37 deletions

View file

@ -47,8 +47,8 @@ func (s *service) radarr(ctx context.Context, action *domain.Action, release dom
MagnetUrl: release.MagnetURI,
Size: int64(release.Size),
Indexer: release.Indexer,
DownloadProtocol: "torrent",
Protocol: "torrent",
DownloadProtocol: string(release.Protocol),
Protocol: string(release.Protocol),
PublishDate: time.Now().Format(time.RFC3339),
}

View file

@ -85,6 +85,9 @@ func (s *service) RunAction(ctx context.Context, action *domain.Action, release
case domain.ActionTypeReadarr:
rejections, err = s.readarr(ctx, action, release)
case domain.ActionTypeSabnzbd:
rejections, err = s.sabnzbd(ctx, action, release)
default:
s.log.Warn().Msgf("unsupported action type: %v", action.Type)
return rejections, err

View file

@ -0,0 +1,52 @@
package action
import (
"context"
"github.com/autobrr/autobrr/internal/domain"
"github.com/autobrr/autobrr/pkg/errors"
"github.com/autobrr/autobrr/pkg/sabnzbd"
)
func (s *service) sabnzbd(ctx context.Context, action *domain.Action, release domain.Release) ([]string, error) {
s.log.Trace().Msg("action Sabnzbd")
if release.Protocol != domain.ReleaseProtocolNzb {
return nil, errors.New("action type: %s invalid protocol: %s", action.Type, release.Protocol)
}
// get client for action
client, err := s.clientSvc.FindByID(ctx, action.ClientID)
if err != nil {
return nil, errors.Wrap(err, "sonarr could not find client: %d", action.ClientID)
}
// return early if no client found
if client == nil {
return nil, errors.New("no sabnzbd client found by id: %d", action.ClientID)
}
opts := sabnzbd.Options{
Addr: client.Host,
ApiKey: client.Settings.APIKey,
Log: nil,
}
if client.Settings.Basic.Auth {
opts.BasicUser = client.Settings.Basic.Username
opts.BasicPass = client.Settings.Basic.Password
}
sab := sabnzbd.New(opts)
ids, err := sab.AddFromUrl(ctx, sabnzbd.AddNzbRequest{Url: release.TorrentURL, Category: action.Category})
if err != nil {
return nil, errors.Wrap(err, "could not add nzb to sabnzbd")
}
s.log.Trace().Msgf("nzb successfully added to client: '%+v'", ids)
s.log.Info().Msgf("nzb successfully added to client: '%s'", client.Name)
return nil, nil
}

View file

@ -47,8 +47,8 @@ func (s *service) sonarr(ctx context.Context, action *domain.Action, release dom
MagnetUrl: release.MagnetURI,
Size: int64(release.Size),
Indexer: release.Indexer,
DownloadProtocol: "torrent",
Protocol: "torrent",
DownloadProtocol: string(release.Protocol),
Protocol: string(release.Protocol),
PublishDate: time.Now().Format(time.RFC3339),
}

View file

@ -99,6 +99,7 @@ func (a *announceProcessor) processQueue(queue chan string) {
}
rls := domain.NewRelease(a.indexer.Identifier)
rls.Protocol = domain.ReleaseProtocol(a.indexer.Protocol)
// on lines matched
if err := a.onLinesMatched(a.indexer, tmpVars, rls); err != nil {

View file

@ -89,6 +89,7 @@ const (
ActionTypeLidarr ActionType = "LIDARR"
ActionTypeWhisparr ActionType = "WHISPARR"
ActionTypeReadarr ActionType = "READARR"
ActionTypeSabnzbd ActionType = "SABNZBD"
)
type ActionContentLayout string

View file

@ -79,6 +79,7 @@ const (
DownloadClientTypeLidarr DownloadClientType = "LIDARR"
DownloadClientTypeWhisparr DownloadClientType = "WHISPARR"
DownloadClientTypeReadarr DownloadClientType = "READARR"
DownloadClientTypeSabnzbd DownloadClientType = "SABNZBD"
)
// Validate basic validation of client

View file

@ -64,6 +64,7 @@ type FeedType string
const (
FeedTypeTorznab FeedType = "TORZNAB"
FeedTypeNewznab FeedType = "NEWZNAB"
FeedTypeRSS FeedType = "RSS"
)

View file

@ -48,9 +48,37 @@ type IndexerDefinition struct {
SettingsMap map[string]string `json:"-"`
IRC *IndexerIRC `json:"irc,omitempty"`
Torznab *Torznab `json:"torznab,omitempty"`
Newznab *Newznab `json:"newznab,omitempty"`
RSS *FeedSettings `json:"rss,omitempty"`
}
type IndexerImplementation string
const (
IndexerImplementationIRC IndexerImplementation = "irc"
IndexerImplementationTorznab IndexerImplementation = "torznab"
IndexerImplementationNewznab IndexerImplementation = "newznab"
IndexerImplementationRSS IndexerImplementation = "rss"
IndexerImplementationLegacy IndexerImplementation = ""
)
func (i IndexerImplementation) String() string {
switch i {
case IndexerImplementationIRC:
return "irc"
case IndexerImplementationTorznab:
return "torznab"
case IndexerImplementationNewznab:
return "newznab"
case IndexerImplementationRSS:
return "rss"
case IndexerImplementationLegacy:
return ""
}
return ""
}
func (i IndexerDefinition) HasApi() bool {
for _, a := range i.Supports {
if a == "api" {
@ -77,6 +105,7 @@ type IndexerDefinitionCustom struct {
SettingsMap map[string]string `json:"-"`
IRC *IndexerIRC `json:"irc,omitempty"`
Torznab *Torznab `json:"torznab,omitempty"`
Newznab *Newznab `json:"newznab,omitempty"`
RSS *FeedSettings `json:"rss,omitempty"`
Parse *IndexerIRCParse `json:"parse,omitempty"`
}
@ -99,6 +128,7 @@ func (i *IndexerDefinitionCustom) ToIndexerDefinition() *IndexerDefinition {
SettingsMap: i.SettingsMap,
IRC: i.IRC,
Torznab: i.Torznab,
Newznab: i.Newznab,
RSS: i.RSS,
}
@ -126,6 +156,11 @@ type Torznab struct {
Settings []IndexerSetting `json:"settings"`
}
type Newznab struct {
MinInterval int `json:"minInterval"`
Settings []IndexerSetting `json:"settings"`
}
type FeedSettings struct {
MinInterval int `json:"minInterval"`
Settings []IndexerSetting `json:"settings"`

View file

@ -154,16 +154,44 @@ type ReleaseProtocol string
const (
ReleaseProtocolTorrent ReleaseProtocol = "torrent"
ReleaseProtocolNzb ReleaseProtocol = "nzb"
)
func (r ReleaseProtocol) String() string {
switch r {
case ReleaseProtocolTorrent:
return "torrent"
case ReleaseProtocolNzb:
return "nzb"
default:
return "torrent"
}
}
type ReleaseImplementation string
const (
ReleaseImplementationIRC ReleaseImplementation = "IRC"
ReleaseImplementationTorznab ReleaseImplementation = "TORZNAB"
ReleaseImplementationNewznab ReleaseImplementation = "NEWZNAB"
ReleaseImplementationRSS ReleaseImplementation = "RSS"
)
func (r ReleaseImplementation) String() string {
switch r {
case ReleaseImplementationIRC:
return "IRC"
case ReleaseImplementationTorznab:
return "TORZNAB"
case ReleaseImplementationNewznab:
return "NEWZNAB"
case ReleaseImplementationRSS:
return "RSS"
default:
return "IRC"
}
}
type ReleaseQueryParams struct {
Limit uint64
Offset uint64
@ -291,7 +319,9 @@ func (r *Release) DownloadTorrentFile() error {
}
func (r *Release) downloadTorrentFile(ctx context.Context) error {
if r.HasMagnetUri() {
if r.Protocol != ReleaseProtocolTorrent {
return errors.New("download_file: protocol is not %s: %s", ReleaseProtocolTorrent, r.Protocol)
} else if r.HasMagnetUri() {
return fmt.Errorf("error trying to download magnet link: %s", r.MagnetURI)
}

View file

@ -10,6 +10,7 @@ import (
"github.com/autobrr/autobrr/pkg/porla"
"github.com/autobrr/autobrr/pkg/radarr"
"github.com/autobrr/autobrr/pkg/readarr"
"github.com/autobrr/autobrr/pkg/sabnzbd"
"github.com/autobrr/autobrr/pkg/sonarr"
"github.com/autobrr/autobrr/pkg/whisparr"
"github.com/autobrr/go-qbittorrent"
@ -51,6 +52,9 @@ func (s *service) testConnection(ctx context.Context, client domain.DownloadClie
case domain.DownloadClientTypeReadarr:
return s.testReadarrConnection(ctx, client)
case domain.DownloadClientTypeSabnzbd:
return s.testSabnzbdConnection(ctx, client)
default:
return errors.New("unsupported client")
}
@ -285,3 +289,23 @@ func (s *service) testPorlaConnection(client domain.DownloadClient) error {
return nil
}
func (s *service) testSabnzbdConnection(ctx context.Context, client domain.DownloadClient) error {
opts := sabnzbd.Options{
Addr: client.Host,
ApiKey: client.Settings.APIKey,
BasicUser: client.Settings.Basic.Username,
BasicPass: client.Settings.Basic.Password,
Log: nil,
}
sab := sabnzbd.New(opts)
version, err := sab.Version(ctx)
if err != nil {
return errors.Wrap(err, "error getting version from sabnzbd")
}
s.log.Debug().Msgf("test client connection for sabnzbd: success got version: %s", version.Version)
return nil
}

168
internal/feed/newznab.go Normal file
View file

@ -0,0 +1,168 @@
package feed
import (
"context"
"sort"
"strconv"
"time"
"github.com/autobrr/autobrr/internal/domain"
"github.com/autobrr/autobrr/internal/release"
"github.com/autobrr/autobrr/internal/scheduler"
"github.com/autobrr/autobrr/pkg/errors"
"github.com/autobrr/autobrr/pkg/newznab"
"github.com/rs/zerolog"
)
type NewznabJob struct {
Feed *domain.Feed
Name string
IndexerIdentifier string
Log zerolog.Logger
URL string
Client newznab.Client
Repo domain.FeedRepo
CacheRepo domain.FeedCacheRepo
ReleaseSvc release.Service
SchedulerSvc scheduler.Service
attempts int
errors []error
JobID int
}
func NewNewznabJob(feed *domain.Feed, name string, indexerIdentifier string, log zerolog.Logger, url string, client newznab.Client, repo domain.FeedRepo, cacheRepo domain.FeedCacheRepo, releaseSvc release.Service) *NewznabJob {
return &NewznabJob{
Feed: feed,
Name: name,
IndexerIdentifier: indexerIdentifier,
Log: log,
URL: url,
Client: client,
Repo: repo,
CacheRepo: cacheRepo,
ReleaseSvc: releaseSvc,
}
}
func (j *NewznabJob) Run() {
ctx := context.Background()
if err := j.process(ctx); err != nil {
j.Log.Err(err).Int("attempts", j.attempts).Msg("newznab process error")
j.errors = append(j.errors, err)
}
j.attempts = 0
j.errors = j.errors[:0]
}
func (j *NewznabJob) process(ctx context.Context) error {
// get feed
items, err := j.getFeed(ctx)
if err != nil {
j.Log.Error().Err(err).Msgf("error fetching feed items")
return errors.Wrap(err, "error getting feed items")
}
j.Log.Debug().Msgf("found (%d) new items to process", len(items))
if len(items) == 0 {
return nil
}
releases := make([]*domain.Release, 0)
for _, item := range items {
rls := domain.NewRelease(j.IndexerIdentifier)
rls.TorrentName = item.Title
rls.InfoURL = item.GUID
rls.Implementation = domain.ReleaseImplementationNewznab
rls.Protocol = domain.ReleaseProtocolNzb
// parse size bytes string
rls.ParseSizeBytesString(item.Size)
rls.ParseString(item.Title)
if item.Enclosure != nil {
if item.Enclosure.Type == "application/x-nzb" {
rls.TorrentURL = item.Enclosure.Url
}
}
// map newznab categories ID and Name into rls.Categories
// so we can filter on both ID and Name
for _, category := range item.Categories {
rls.Categories = append(rls.Categories, []string{category.Name, strconv.Itoa(category.ID)}...)
}
releases = append(releases, rls)
}
// process all new releases
go j.ReleaseSvc.ProcessMultiple(releases)
return nil
}
func (j *NewznabJob) getFeed(ctx context.Context) ([]newznab.FeedItem, error) {
// get feed
feed, err := j.Client.GetFeed(ctx)
if err != nil {
j.Log.Error().Err(err).Msgf("error fetching feed items")
return nil, errors.Wrap(err, "error fetching feed items")
}
if err := j.Repo.UpdateLastRunWithData(ctx, j.Feed.ID, feed.Raw); err != nil {
j.Log.Error().Err(err).Msgf("error updating last run for feed id: %v", j.Feed.ID)
}
j.Log.Debug().Msgf("refreshing feed: %v, found (%d) items", j.Name, len(feed.Channel.Items))
items := make([]newznab.FeedItem, 0)
if len(feed.Channel.Items) == 0 {
return items, nil
}
sort.SliceStable(feed.Channel.Items, func(i, j int) bool {
return feed.Channel.Items[i].PubDate.After(feed.Channel.Items[j].PubDate.Time)
})
for _, i := range feed.Channel.Items {
if i.GUID == "" {
j.Log.Error().Err(err).Msgf("missing GUID from feed: %s", j.Feed.Name)
continue
}
exists, err := j.CacheRepo.Exists(j.Name, i.GUID)
if err != nil {
j.Log.Error().Err(err).Msg("could not check if item exists")
continue
}
if exists {
j.Log.Trace().Msgf("cache item exists, skipping release: %s", i.Title)
continue
}
j.Log.Debug().Msgf("found new release: %s", i.Title)
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
if err := j.CacheRepo.Put(j.Name, i.GUID, []byte(i.Title), ttl); err != nil {
j.Log.Error().Stack().Err(err).Str("guid", i.GUID).Msg("cache.Put: error storing item in cache")
continue
}
// only append if we successfully added to cache
items = append(items, i)
}
// send to filters
return items, nil
}

View file

@ -12,6 +12,7 @@ import (
"github.com/autobrr/autobrr/internal/release"
"github.com/autobrr/autobrr/internal/scheduler"
"github.com/autobrr/autobrr/pkg/errors"
"github.com/autobrr/autobrr/pkg/newznab"
"github.com/autobrr/autobrr/pkg/torznab"
"github.com/dcarbone/zadapters/zstdlog"
@ -222,17 +223,27 @@ func (s *service) test(ctx context.Context, feed *domain.Feed) error {
subLogger := zstdlog.NewStdLoggerWithLevel(s.log.With().Logger(), zerolog.DebugLevel)
// test feeds
if feed.Type == string(domain.FeedTypeTorznab) {
switch feed.Type {
case string(domain.FeedTypeTorznab):
if err := s.testTorznab(ctx, feed, subLogger); err != nil {
return err
}
} else if feed.Type == string(domain.FeedTypeRSS) {
case string(domain.FeedTypeNewznab):
if err := s.testNewznab(ctx, feed, subLogger); err != nil {
return err
}
case string(domain.FeedTypeRSS):
if err := s.testRSS(ctx, feed); err != nil {
return err
}
default:
return errors.New("unsupported feed type: %s", feed.Type)
}
s.log.Info().Msgf("feed test successful - connected to feed: %v", feed.URL)
s.log.Info().Msgf("feed test successful - connected to feed: %s", feed.URL)
return nil
}
@ -264,6 +275,21 @@ func (s *service) testTorznab(ctx context.Context, feed *domain.Feed, subLogger
return nil
}
func (s *service) testNewznab(ctx context.Context, feed *domain.Feed, subLogger *log.Logger) error {
// setup newznab Client
c := newznab.NewClient(newznab.Config{Host: feed.URL, ApiKey: feed.ApiKey, Log: subLogger})
items, err := c.GetFeed(ctx)
if err != nil {
s.log.Error().Err(err).Msg("error getting newznab feed")
return err
}
s.log.Info().Msgf("refreshing newznab feed: %v, found (%d) items", feed.Name, len(items.Channel.Items))
return nil
}
func (s *service) start() error {
// get all torznab indexer definitions
feeds, err := s.repo.Find(context.TODO())
@ -335,6 +361,13 @@ func (s *service) startJob(f *domain.Feed) error {
s.log.Error().Err(err).Msg("failed to initialize torznab feed")
return err
}
case string(domain.FeedTypeNewznab):
if err := s.addNewznabJob(fi); err != nil {
s.log.Error().Err(err).Msg("failed to initialize newznab feed")
return err
}
case string(domain.FeedTypeRSS):
if err := s.addRSSJob(fi); err != nil {
s.log.Error().Err(err).Msg("failed to initialize rss feed")
@ -380,6 +413,37 @@ func (s *service) addTorznabJob(f feedInstance) error {
return nil
}
func (s *service) addNewznabJob(f feedInstance) error {
if f.URL == "" {
return errors.New("newznab feed requires URL")
}
// setup logger
l := s.log.With().Str("feed", f.Name).Logger()
// setup newznab Client
c := newznab.NewClient(newznab.Config{Host: f.URL, ApiKey: f.ApiKey, Timeout: f.Timeout})
// create job
job := NewNewznabJob(f.Feed, f.Name, f.IndexerIdentifier, l, f.URL, c, s.repo, s.cacheRepo, s.releaseSvc)
identifierKey := feedKey{f.Feed.ID, f.Feed.Indexer, f.Feed.Name}.ToString()
// schedule job
id, err := s.scheduler.AddJob(job, f.CronSchedule, identifierKey)
if err != nil {
return errors.Wrap(err, "feed.AddNewznabJob: add job failed")
}
job.JobID = id
// add to job map
s.jobs[identifierKey] = id
s.log.Debug().Msgf("add newznab job: %v", f.Name)
return nil
}
func (s *service) addRSSJob(f feedInstance) error {
if f.URL == "" {
return errors.New("rss feed requires URL")

View file

@ -49,6 +49,8 @@ type service struct {
lookupIRCServerDefinition map[string]map[string]*domain.IndexerDefinition
// torznab indexers
torznabIndexers map[string]*domain.IndexerDefinition
// newznab indexers
newznabIndexers map[string]*domain.IndexerDefinition
// rss indexers
rssIndexers map[string]*domain.IndexerDefinition
}
@ -62,6 +64,7 @@ func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRep
scheduler: scheduler,
lookupIRCServerDefinition: make(map[string]map[string]*domain.IndexerDefinition),
torznabIndexers: make(map[string]*domain.IndexerDefinition),
newznabIndexers: make(map[string]*domain.IndexerDefinition),
rssIndexers: make(map[string]*domain.IndexerDefinition),
definitions: make(map[string]domain.IndexerDefinition),
mappedDefinitions: make(map[string]*domain.IndexerDefinition),
@ -72,7 +75,7 @@ func (s *service) Store(ctx context.Context, indexer domain.Indexer) (*domain.In
// if indexer is rss or torznab do additional cleanup for identifier
switch indexer.Implementation {
case "torznab", "rss":
case "torznab", "newznab", "rss":
// make lowercase
cleanName := strings.ToLower(indexer.Name)
@ -213,6 +216,8 @@ func (s *service) mapIndexer(indexer domain.Indexer) (*domain.IndexerDefinition,
definitionName := indexer.Identifier
if indexer.Implementation == "torznab" {
definitionName = "torznab"
} else if indexer.Implementation == "newznab" {
definitionName = "newznab"
} else if indexer.Implementation == "rss" {
definitionName = "rss"
}
@ -336,6 +341,8 @@ func (s *service) Start() error {
// handle Torznab
if indexer.Implementation == "torznab" {
s.torznabIndexers[indexer.Identifier] = indexer
} else if indexer.Implementation == "newznab" {
s.newznabIndexers[indexer.Identifier] = indexer
} else if indexer.Implementation == "rss" {
s.rssIndexers[indexer.Identifier] = indexer
}
@ -350,6 +357,8 @@ func (s *service) removeIndexer(indexer domain.Indexer) {
// remove Torznab
if indexer.Implementation == "torznab" {
delete(s.torznabIndexers, indexer.Identifier)
} else if indexer.Implementation == "newznab" {
delete(s.newznabIndexers, indexer.Identifier)
} else if indexer.Implementation == "rss" {
delete(s.rssIndexers, indexer.Identifier)
}
@ -383,6 +392,8 @@ func (s *service) addIndexer(indexer domain.Indexer) error {
// handle Torznab and RSS
if indexerDefinition.Implementation == "torznab" {
s.torznabIndexers[indexer.Identifier] = indexerDefinition
} else if indexer.Implementation == "newznab" {
s.newznabIndexers[indexer.Identifier] = indexerDefinition
} else if indexerDefinition.Implementation == "rss" {
s.rssIndexers[indexer.Identifier] = indexerDefinition
}
@ -417,6 +428,8 @@ func (s *service) updateIndexer(indexer domain.Indexer) error {
// handle Torznab
if indexerDefinition.Implementation == "torznab" {
s.torznabIndexers[indexer.Identifier] = indexerDefinition
} else if indexer.Implementation == "newznab" {
s.newznabIndexers[indexer.Identifier] = indexerDefinition
} else if indexerDefinition.Implementation == "rss" {
s.rssIndexers[indexer.Identifier] = indexerDefinition
}