From 72be86a34f2a79d11b8ee1ed56dcfc65ea301e43 Mon Sep 17 00:00:00 2001 From: paperclip-go-brr <112905434+paperclip-go-brr@users.noreply.github.com> Date: Tue, 4 Oct 2022 17:33:35 +0200 Subject: [PATCH] feat(feed): Configurable request timeout (#456) * feat(feed): Add field for setting request timeout * fix: missing type in interface * feat: add postgres migration and column to base schema --- internal/database/feed.go | 12 +++++++++--- internal/database/postgres_migrate.go | 4 ++++ internal/database/sqlite_migrate.go | 4 ++++ internal/domain/feed.go | 1 + internal/feed/rss.go | 6 ++++-- internal/feed/service.go | 6 ++++-- pkg/torznab/torznab.go | 7 ++++--- web/src/forms/settings/FeedForms.tsx | 12 ++++++++---- web/src/forms/settings/IndexerForms.tsx | 2 ++ web/src/types/Feed.d.ts | 2 ++ 10 files changed, 42 insertions(+), 14 deletions(-) diff --git a/internal/database/feed.go b/internal/database/feed.go index 49fe0b7..e40596c 100644 --- a/internal/database/feed.go +++ b/internal/database/feed.go @@ -34,6 +34,7 @@ func (r *FeedRepo) FindByID(ctx context.Context, id int) (*domain.Feed, error) { "enabled", "url", "interval", + "timeout", "api_key", "created_at", "updated_at", @@ -55,7 +56,7 @@ func (r *FeedRepo) FindByID(ctx context.Context, id int) (*domain.Feed, error) { var apiKey sql.NullString - if err := row.Scan(&f.ID, &f.Indexer, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &apiKey, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := row.Scan(&f.ID, &f.Indexer, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &apiKey, &f.CreatedAt, &f.UpdatedAt); err != nil { return nil, errors.Wrap(err, "error scanning row") } @@ -75,6 +76,7 @@ func (r *FeedRepo) FindByIndexerIdentifier(ctx context.Context, indexer string) "enabled", "url", "interval", + "timeout", "api_key", "created_at", "updated_at", @@ -96,7 +98,7 @@ func (r *FeedRepo) FindByIndexerIdentifier(ctx context.Context, indexer string) var apiKey sql.NullString - if err := row.Scan(&f.ID, &f.Indexer, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &apiKey, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := row.Scan(&f.ID, &f.Indexer, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &apiKey, &f.CreatedAt, &f.UpdatedAt); err != nil { return nil, errors.Wrap(err, "error scanning row") } @@ -116,6 +118,7 @@ func (r *FeedRepo) Find(ctx context.Context) ([]domain.Feed, error) { "enabled", "url", "interval", + "timeout", "api_key", "created_at", "updated_at", @@ -141,7 +144,7 @@ func (r *FeedRepo) Find(ctx context.Context) ([]domain.Feed, error) { var apiKey sql.NullString - if err := rows.Scan(&f.ID, &f.Indexer, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &apiKey, &f.CreatedAt, &f.UpdatedAt); err != nil { + if err := rows.Scan(&f.ID, &f.Indexer, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &apiKey, &f.CreatedAt, &f.UpdatedAt); err != nil { return nil, errors.Wrap(err, "error scanning row") } @@ -164,6 +167,7 @@ func (r *FeedRepo) Store(ctx context.Context, feed *domain.Feed) error { "enabled", "url", "interval", + "timeout", "api_key", "indexer_id", ). @@ -174,6 +178,7 @@ func (r *FeedRepo) Store(ctx context.Context, feed *domain.Feed) error { feed.Enabled, feed.URL, feed.Interval, + feed.Timeout, feed.ApiKey, feed.IndexerID, ). @@ -199,6 +204,7 @@ func (r *FeedRepo) Update(ctx context.Context, feed *domain.Feed) error { Set("enabled", feed.Enabled). Set("url", feed.URL). Set("interval", feed.Interval). + Set("timeout", feed.Timeout). Set("api_key", feed.ApiKey). Where("id = ?", feed.ID) diff --git a/internal/database/postgres_migrate.go b/internal/database/postgres_migrate.go index 9e6a6e5..eb26419 100644 --- a/internal/database/postgres_migrate.go +++ b/internal/database/postgres_migrate.go @@ -298,6 +298,7 @@ CREATE TABLE feed enabled BOOLEAN, url TEXT, interval INTEGER, + timeout INTEGER DEFAULT 60, categories TEXT [] DEFAULT '{}' NOT NULL, capabilities TEXT [] DEFAULT '{}' NOT NULL, api_key TEXT, @@ -557,4 +558,7 @@ CREATE INDEX indexer_identifier_index created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); `, + `ALTER TABLE feed + ADD COLUMN timeout INTEGER DEFAULT 60; + `, } diff --git a/internal/database/sqlite_migrate.go b/internal/database/sqlite_migrate.go index 1c17194..3d6d5bf 100644 --- a/internal/database/sqlite_migrate.go +++ b/internal/database/sqlite_migrate.go @@ -281,6 +281,7 @@ CREATE TABLE feed enabled BOOLEAN, url TEXT, interval INTEGER, + timeout INTEGER DEFAULT 60, categories TEXT [] DEFAULT '{}' NOT NULL, capabilities TEXT [] DEFAULT '{}' NOT NULL, api_key TEXT, @@ -877,4 +878,7 @@ CREATE INDEX indexer_identifier_index created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); `, + `ALTER TABLE feed + ADD COLUMN timeout INTEGER DEFAULT 60; + `, } diff --git a/internal/domain/feed.go b/internal/domain/feed.go index 97dff14..fd1040b 100644 --- a/internal/domain/feed.go +++ b/internal/domain/feed.go @@ -31,6 +31,7 @@ type Feed struct { Enabled bool `json:"enabled"` URL string `json:"url"` Interval int `json:"interval"` + Timeout int `json:"timeout"` Capabilities []string `json:"capabilities"` ApiKey string `json:"api_key"` Settings map[string]string `json:"settings"` diff --git a/internal/feed/rss.go b/internal/feed/rss.go index fb02dab..6fcd2db 100644 --- a/internal/feed/rss.go +++ b/internal/feed/rss.go @@ -21,6 +21,7 @@ type RSSJob struct { URL string Repo domain.FeedCacheRepo ReleaseSvc release.Service + Timeout time.Duration attempts int errors []error @@ -28,7 +29,7 @@ type RSSJob struct { JobID int } -func NewRSSJob(name string, indexerIdentifier string, log zerolog.Logger, url string, repo domain.FeedCacheRepo, releaseSvc release.Service) *RSSJob { +func NewRSSJob(name string, indexerIdentifier string, log zerolog.Logger, url string, repo domain.FeedCacheRepo, releaseSvc release.Service, timeout time.Duration) *RSSJob { return &RSSJob{ Name: name, IndexerIdentifier: indexerIdentifier, @@ -36,6 +37,7 @@ func NewRSSJob(name string, indexerIdentifier string, log zerolog.Logger, url st URL: url, Repo: repo, ReleaseSvc: releaseSvc, + Timeout: timeout, } } @@ -140,7 +142,7 @@ func (j *RSSJob) processItem(item *gofeed.Item) *domain.Release { } func (j *RSSJob) getFeed() (items []*gofeed.Item, err error) { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), j.Timeout) defer cancel() feed, err := gofeed.NewParser().ParseURLWithContext(j.URL, ctx) // there's an RSS specific parser as well. diff --git a/internal/feed/service.go b/internal/feed/service.go index 5d83236..b3b2cce 100644 --- a/internal/feed/service.go +++ b/internal/feed/service.go @@ -35,6 +35,7 @@ type feedInstance struct { ApiKey string Implementation string CronSchedule time.Duration + Timeout time.Duration } type service struct { @@ -300,6 +301,7 @@ func (s *service) startJob(f domain.Feed) error { URL: f.URL, ApiKey: f.ApiKey, CronSchedule: time.Duration(f.Interval) * time.Minute, + Timeout: time.Duration(f.Timeout) * time.Second, } switch fi.Implementation { @@ -330,7 +332,7 @@ func (s *service) addTorznabJob(f feedInstance) error { l := s.log.With().Str("feed", f.Name).Logger() // setup torznab Client - c := torznab.NewClient(torznab.Config{Host: f.URL, ApiKey: f.ApiKey}) + c := torznab.NewClient(torznab.Config{Host: f.URL, ApiKey: f.ApiKey, Timeout: f.Timeout}) // create job job := NewTorznabJob(f.Name, f.IndexerIdentifier, l, f.URL, c, s.cacheRepo, s.releaseSvc) @@ -373,7 +375,7 @@ func (s *service) addRSSJob(f feedInstance) error { l := s.log.With().Str("feed", f.Name).Logger() // create job - job := NewRSSJob(f.Name, f.IndexerIdentifier, l, f.URL, s.cacheRepo, s.releaseSvc) + job := NewRSSJob(f.Name, f.IndexerIdentifier, l, f.URL, s.cacheRepo, s.releaseSvc, f.Timeout) // schedule job id, err := s.scheduler.AddJob(job, f.CronSchedule, f.IndexerIdentifier) diff --git a/pkg/torznab/torznab.go b/pkg/torznab/torznab.go index 1e448ce..d82e97a 100644 --- a/pkg/torznab/torznab.go +++ b/pkg/torznab/torznab.go @@ -37,8 +37,9 @@ type BasicAuth struct { } type Config struct { - Host string - ApiKey string + Host string + ApiKey string + Timeout time.Duration UseBasicAuth bool BasicAuth BasicAuth @@ -48,7 +49,7 @@ type Config struct { func NewClient(config Config) Client { httpClient := &http.Client{ - Timeout: time.Second * 20, + Timeout: config.Timeout, } c := &client{ diff --git a/web/src/forms/settings/FeedForms.tsx b/web/src/forms/settings/FeedForms.tsx index 4698d8d..5a9f593 100644 --- a/web/src/forms/settings/FeedForms.tsx +++ b/web/src/forms/settings/FeedForms.tsx @@ -25,6 +25,7 @@ interface InitialValues { url: string; api_key: string; interval: number; + timeout: number; } export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) { @@ -103,7 +104,8 @@ export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) { name: feed.name, url: feed.url, api_key: feed.api_key, - interval: feed.interval + interval: feed.interval, + timeout: feed.timeout }; return ( @@ -162,8 +164,9 @@ function FormFieldsTorznab() { - + + + ); } @@ -177,7 +180,8 @@ function FormFieldsRSS() { help="RSS url" /> - + + ); } diff --git a/web/src/forms/settings/IndexerForms.tsx b/web/src/forms/settings/IndexerForms.tsx index 4a21ab3..ce7fc3b 100644 --- a/web/src/forms/settings/IndexerForms.tsx +++ b/web/src/forms/settings/IndexerForms.tsx @@ -253,6 +253,7 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) { url: formData.feed.url, api_key: formData.feed.api_key, interval: 30, + timeout: 60, indexer: name, indexer_id: 0 }; @@ -278,6 +279,7 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) { type: "RSS", url: formData.feed.url, interval: 30, + timeout: 60, indexer: name, indexer_id: 0 }; diff --git a/web/src/types/Feed.d.ts b/web/src/types/Feed.d.ts index bda7158..dec8d6c 100644 --- a/web/src/types/Feed.d.ts +++ b/web/src/types/Feed.d.ts @@ -6,6 +6,7 @@ interface Feed { enabled: boolean; url: string; interval: number; + timeout: number; api_key: string; created_at: Date; updated_at: Date; @@ -20,6 +21,7 @@ interface FeedCreate { enabled: boolean; url: string; interval: number; + timeout: number; api_key?: string; indexer_id: number; }