From 20801366695f329f4e3ff7a396a600cc70e0a067 Mon Sep 17 00:00:00 2001 From: Steven Kreitzer Date: Fri, 27 Oct 2023 10:37:57 -0500 Subject: [PATCH] feat(filters): external webhook retry on status codes (#1206) * feat: external filter retry status codes * chore: go mod tidy * fix(database): migrations --------- Co-authored-by: ze0s --- go.mod | 1 + go.sum | 2 + internal/database/filter.go | 122 ++++++++++++----- internal/database/postgres_migrate.go | 46 ++++--- internal/database/sqlite_migrate.go | 46 ++++--- internal/domain/filter.go | 182 ++++++++++++++------------ internal/filter/service.go | 57 ++++++-- web/src/screens/filters/Details.tsx | 4 + web/src/screens/filters/External.tsx | 23 +++- web/src/screens/filters/List.tsx | 8 ++ web/src/types/Filter.d.ts | 4 + 11 files changed, 330 insertions(+), 165 deletions(-) diff --git a/go.mod b/go.mod index b29ba45..8d374b5 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/autobrr/go-qbittorrent v1.6.0 github.com/autobrr/go-rtorrent v1.10.0 github.com/avast/retry-go v3.0.0+incompatible + github.com/avast/retry-go/v4 v4.5.0 github.com/dcarbone/zadapters/zstdlog v1.0.0 github.com/dustin/go-humanize v1.0.1 github.com/ergochat/irc-go v0.4.0 diff --git a/go.sum b/go.sum index c7c0764..cd100d5 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,8 @@ github.com/autobrr/sse/v2 v2.0.0-20230520125637-530e06346d7d h1:9EGCYgeugAVWLBAt github.com/autobrr/sse/v2 v2.0.0-20230520125637-530e06346d7d/go.mod h1:zCozZ9lp4DE340T2+wfMPL/eoQwLVIGDOCKCDEFwTQU= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/avast/retry-go/v4 v4.5.0 h1:QoRAZZ90cj5oni2Lsgl2GW8mNTnUCnmpx/iKpwVisHg= +github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5Aq1fboC3+I= github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= diff --git a/internal/database/filter.go b/internal/database/filter.go index cfcb9f7..fe77e2b 100644 --- a/internal/database/filter.go +++ b/internal/database/filter.go @@ -247,6 +247,10 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter "fe.webhook_data", "fe.webhook_headers", "fe.webhook_expect_status", + "fe.webhook_retry_status", + "fe.webhook_retry_attempts", + "fe.webhook_retry_delay_seconds", + "fe.webhook_retry_max_jitter_seconds", ). From("filter f"). LeftJoin("filter_external fe ON f.id = fe.filter_id"). @@ -276,8 +280,8 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter var delay, maxDownloads, logScore sql.NullInt32 // filter external - var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData sql.NullString - var extId, extIndex, extWebhookStatus, extExecStatus sql.NullInt32 + var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData, extWebhookRetryStatus sql.NullString + var extId, extIndex, extWebhookStatus, extWebhookRetryAttempts, extWebhookDelaySeconds, extWebhookRetryJitterSeconds, extExecStatus sql.NullInt32 var extEnabled sql.NullBool if err := rows.Scan( @@ -354,6 +358,10 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter &extWebhookData, &extWebhookHeaders, &extWebhookStatus, + &extWebhookRetryStatus, + &extWebhookRetryAttempts, + &extWebhookDelaySeconds, + &extWebhookRetryJitterSeconds, ); err != nil { return nil, errors.Wrap(err, "error scanning row") } @@ -396,19 +404,23 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter if extId.Valid { external := domain.FilterExternal{ - ID: int(extId.Int32), - Name: extName.String, - Index: int(extIndex.Int32), - Type: domain.FilterExternalType(extType.String), - Enabled: extEnabled.Bool, - ExecCmd: extExecCmd.String, - ExecArgs: extExecArgs.String, - ExecExpectStatus: int(extExecStatus.Int32), - WebhookHost: extWebhookHost.String, - WebhookMethod: extWebhookMethod.String, - WebhookData: extWebhookData.String, - WebhookHeaders: extWebhookHeaders.String, - WebhookExpectStatus: int(extWebhookStatus.Int32), + ID: int(extId.Int32), + Name: extName.String, + Index: int(extIndex.Int32), + Type: domain.FilterExternalType(extType.String), + Enabled: extEnabled.Bool, + ExecCmd: extExecCmd.String, + ExecArgs: extExecArgs.String, + ExecExpectStatus: int(extExecStatus.Int32), + WebhookHost: extWebhookHost.String, + WebhookMethod: extWebhookMethod.String, + WebhookData: extWebhookData.String, + WebhookHeaders: extWebhookHeaders.String, + WebhookExpectStatus: int(extWebhookStatus.Int32), + WebhookRetryStatus: extWebhookRetryStatus.String, + WebhookRetryAttempts: int(extWebhookRetryAttempts.Int32), + WebhookRetryDelaySeconds: int(extWebhookDelaySeconds.Int32), + WebhookRetryMaxJitterSeconds: int(extWebhookRetryJitterSeconds.Int32), } externalMap[external.ID] = external } @@ -502,6 +514,10 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string "fe.webhook_data", "fe.webhook_headers", "fe.webhook_expect_status", + "fe.webhook_retry_status", + "fe.webhook_retry_attempts", + "fe.webhook_retry_delay_seconds", + "fe.webhook_retry_max_jitter_seconds", "fe.filter_id", ). From("filter f"). @@ -537,8 +553,8 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string var delay, maxDownloads, logScore sql.NullInt32 // filter external - var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData sql.NullString - var extId, extIndex, extWebhookStatus, extExecStatus, extFilterId sql.NullInt32 + var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData, extWebhookRetryStatus sql.NullString + var extId, extIndex, extWebhookStatus, extWebhookRetryAttempts, extWebhookDelaySeconds, extWebhookRetryJitterSeconds, extExecStatus, extFilterId sql.NullInt32 var extEnabled sql.NullBool if err := rows.Scan( @@ -615,6 +631,10 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string &extWebhookData, &extWebhookHeaders, &extWebhookStatus, + &extWebhookRetryStatus, + &extWebhookRetryAttempts, + &extWebhookDelaySeconds, + &extWebhookRetryJitterSeconds, &extFilterId, ); err != nil { return nil, errors.Wrap(err, "error scanning row") @@ -658,20 +678,24 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string if extId.Valid { external := domain.FilterExternal{ - ID: int(extId.Int32), - Name: extName.String, - Index: int(extIndex.Int32), - Type: domain.FilterExternalType(extType.String), - Enabled: extEnabled.Bool, - ExecCmd: extExecCmd.String, - ExecArgs: extExecArgs.String, - ExecExpectStatus: int(extExecStatus.Int32), - WebhookHost: extWebhookHost.String, - WebhookMethod: extWebhookMethod.String, - WebhookData: extWebhookData.String, - WebhookHeaders: extWebhookHeaders.String, - WebhookExpectStatus: int(extWebhookStatus.Int32), - FilterId: int(extFilterId.Int32), + ID: int(extId.Int32), + Name: extName.String, + Index: int(extIndex.Int32), + Type: domain.FilterExternalType(extType.String), + Enabled: extEnabled.Bool, + ExecCmd: extExecCmd.String, + ExecArgs: extExecArgs.String, + ExecExpectStatus: int(extExecStatus.Int32), + WebhookHost: extWebhookHost.String, + WebhookMethod: extWebhookMethod.String, + WebhookData: extWebhookData.String, + WebhookHeaders: extWebhookHeaders.String, + WebhookExpectStatus: int(extWebhookStatus.Int32), + WebhookRetryStatus: extWebhookRetryStatus.String, + WebhookRetryAttempts: int(extWebhookRetryAttempts.Int32), + WebhookRetryDelaySeconds: int(extWebhookDelaySeconds.Int32), + WebhookRetryMaxJitterSeconds: int(extWebhookRetryJitterSeconds.Int32), + FilterId: int(extFilterId.Int32), } externalMap[external.FilterId] = append(externalMap[external.FilterId], external) } @@ -709,6 +733,10 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int) "fe.webhook_data", "fe.webhook_headers", "fe.webhook_expect_status", + "fe.webhook_retry_status", + "fe.webhook_retry_attempts", + "fe.webhook_retry_delay_seconds", + "fe.webhook_retry_max_jitter_seconds", ). From("filter_external fe"). Where(sq.Eq{"fe.filter_id": filterId}) @@ -732,8 +760,8 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int) var external domain.FilterExternal // filter external - var extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData sql.NullString - var extWebhookStatus, extExecStatus sql.NullInt32 + var extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData, extWebhookRetryStatus sql.NullString + var extWebhookStatus, extWebhookRetryAttempts, extWebhookDelaySeconds, extWebhookRetryJitterSeconds, extExecStatus sql.NullInt32 if err := rows.Scan( &external.ID, @@ -749,6 +777,10 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int) &extWebhookData, &extWebhookHeaders, &extWebhookStatus, + &extWebhookRetryStatus, + &extWebhookRetryAttempts, + &extWebhookDelaySeconds, + &extWebhookRetryJitterSeconds, ); err != nil { return nil, errors.Wrap(err, "error scanning row") } @@ -762,6 +794,10 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int) external.WebhookData = extWebhookData.String external.WebhookHeaders = extWebhookHeaders.String external.WebhookExpectStatus = int(extWebhookStatus.Int32) + external.WebhookRetryStatus = extWebhookRetryStatus.String + external.WebhookRetryAttempts = int(extWebhookRetryAttempts.Int32) + external.WebhookRetryDelaySeconds = int(extWebhookDelaySeconds.Int32) + external.WebhookRetryMaxJitterSeconds = int(extWebhookRetryJitterSeconds.Int32) externalFilters = append(externalFilters, external) } @@ -1182,6 +1218,18 @@ func (r *FilterRepo) UpdatePartial(ctx context.Context, filter domain.FilterUpda if filter.ExternalWebhookExpectStatus != nil { q = q.Set("external_webhook_expect_status", filter.ExternalWebhookExpectStatus) } + if filter.ExternalWebhookRetryStatus != nil { + q = q.Set("external_webhook_retry_status", filter.ExternalWebhookRetryStatus) + } + if filter.ExternalWebhookRetryAttempts != nil { + q = q.Set("external_webhook_retry_attempts", filter.ExternalWebhookRetryAttempts) + } + if filter.ExternalWebhookRetryDelaySeconds != nil { + q = q.Set("external_webhook_retry_delay_seconds", filter.ExternalWebhookRetryDelaySeconds) + } + if filter.ExternalWebhookRetryMaxJitterSeconds != nil { + q = q.Set("external_webhook_retry_max_jitter_seconds", filter.ExternalWebhookRetryMaxJitterSeconds) + } q = q.Where(sq.Eq{"id": filter.ID}) @@ -1462,6 +1510,10 @@ func (r *FilterRepo) StoreFilterExternal(ctx context.Context, filterID int, exte "webhook_data", "webhook_headers", "webhook_expect_status", + "webhook_retry_status", + "webhook_retry_attempts", + "webhook_retry_delay_seconds", + "webhook_retry_max_jitter_seconds", "filter_id", ) @@ -1479,6 +1531,10 @@ func (r *FilterRepo) StoreFilterExternal(ctx context.Context, filterID int, exte toNullString(external.WebhookData), toNullString(external.WebhookHeaders), toNullInt32(int32(external.WebhookExpectStatus)), + toNullString(external.WebhookRetryStatus), + toNullInt32(int32(external.WebhookRetryAttempts)), + toNullInt32(int32(external.WebhookRetryDelaySeconds)), + toNullInt32(int32(external.WebhookRetryMaxJitterSeconds)), filterID, ) } diff --git a/internal/database/postgres_migrate.go b/internal/database/postgres_migrate.go index a6ce260..3a69e4f 100644 --- a/internal/database/postgres_migrate.go +++ b/internal/database/postgres_migrate.go @@ -133,21 +133,25 @@ CREATE TABLE filter CREATE TABLE filter_external ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - idx INTEGER, - type TEXT, - enabled BOOLEAN, - exec_cmd TEXT, - exec_args TEXT, - exec_expect_status INTEGER, - webhook_host TEXT, - webhook_method TEXT, - webhook_data TEXT, - webhook_headers TEXT, - webhook_expect_status INTEGER, - filter_id INTEGER NOT NULL, - FOREIGN KEY (filter_id) REFERENCES filter(id) ON DELETE CASCADE + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + idx INTEGER, + type TEXT, + enabled BOOLEAN, + exec_cmd TEXT, + exec_args TEXT, + exec_expect_status INTEGER, + webhook_host TEXT, + webhook_method TEXT, + webhook_data TEXT, + webhook_headers TEXT, + webhook_expect_status INTEGER, + webhook_retry_status TEXT, + webhook_retry_attempts INTEGER, + webhook_retry_delay_seconds INTEGER, + webhook_retry_max_jitter_seconds INTEGER, + filter_id INTEGER NOT NULL, + FOREIGN KEY (filter_id) REFERENCES filter(id) ON DELETE CASCADE ); CREATE TABLE filter_indexer @@ -797,5 +801,17 @@ CREATE INDEX feed_cache_feed_id_key_index `, `ALTER TABLE action ADD COLUMN external_client_id INTEGER; +`, + `ALTER TABLE filter_external +ADD COLUMN external_webhook_retry_status TEXT; + +ALTER TABLE filter_external + ADD COLUMN external_webhook_retry_attempts INTEGER; + +ALTER TABLE filter_external + ADD COLUMN external_webhook_retry_delay_seconds INTEGER; + +ALTER TABLE filter_external + ADD COLUMN external_webhook_retry_max_jitter_seconds INTEGER; `, } diff --git a/internal/database/sqlite_migrate.go b/internal/database/sqlite_migrate.go index fdde6f7..9d73428 100644 --- a/internal/database/sqlite_migrate.go +++ b/internal/database/sqlite_migrate.go @@ -133,21 +133,25 @@ CREATE TABLE filter CREATE TABLE filter_external ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - idx INTEGER, - type TEXT, - enabled BOOLEAN, - exec_cmd TEXT, - exec_args TEXT, - exec_expect_status INTEGER, - webhook_host TEXT, - webhook_method TEXT, - webhook_data TEXT, - webhook_headers TEXT, - webhook_expect_status INTEGER, - filter_id INTEGER NOT NULL, - FOREIGN KEY (filter_id) REFERENCES filter(id) ON DELETE CASCADE + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + idx INTEGER, + type TEXT, + enabled BOOLEAN, + exec_cmd TEXT, + exec_args TEXT, + exec_expect_status INTEGER, + webhook_host TEXT, + webhook_method TEXT, + webhook_data TEXT, + webhook_headers TEXT, + webhook_expect_status INTEGER, + webhook_retry_status TEXT, + webhook_retry_attempts INTEGER, + webhook_retry_delay_seconds INTEGER, + webhook_retry_max_jitter_seconds INTEGER, + filter_id INTEGER NOT NULL, + FOREIGN KEY (filter_id) REFERENCES filter(id) ON DELETE CASCADE ); CREATE TABLE filter_indexer @@ -1348,5 +1352,17 @@ CREATE INDEX feed_cache_feed_id_key_index `, `ALTER TABLE action ADD COLUMN external_client_id INTEGER; +`, + `ALTER TABLE filter_external +ADD COLUMN external_webhook_retry_status TEXT; + +ALTER TABLE filter_external + ADD COLUMN external_webhook_retry_attempts INTEGER; + +ALTER TABLE filter_external + ADD COLUMN external_webhook_retry_delay_seconds INTEGER; + +ALTER TABLE filter_external + ADD COLUMN external_webhook_retry_max_jitter_seconds INTEGER; `, } diff --git a/internal/domain/filter.go b/internal/domain/filter.go index 9718bd7..2451610 100644 --- a/internal/domain/filter.go +++ b/internal/domain/filter.go @@ -138,20 +138,24 @@ type Filter struct { } type FilterExternal struct { - ID int `json:"id"` - Name string `json:"name"` - Index int `json:"index"` - Type FilterExternalType `json:"type"` - Enabled bool `json:"enabled"` - ExecCmd string `json:"exec_cmd,omitempty"` - ExecArgs string `json:"exec_args,omitempty"` - ExecExpectStatus int `json:"exec_expect_status,omitempty"` - WebhookHost string `json:"webhook_host,omitempty"` - WebhookMethod string `json:"webhook_method,omitempty"` - WebhookData string `json:"webhook_data,omitempty"` - WebhookHeaders string `json:"webhook_headers,omitempty"` - WebhookExpectStatus int `json:"webhook_expect_status,omitempty"` - FilterId int `json:"-"` + ID int `json:"id"` + Name string `json:"name"` + Index int `json:"index"` + Type FilterExternalType `json:"type"` + Enabled bool `json:"enabled"` + ExecCmd string `json:"exec_cmd,omitempty"` + ExecArgs string `json:"exec_args,omitempty"` + ExecExpectStatus int `json:"exec_expect_status,omitempty"` + WebhookHost string `json:"webhook_host,omitempty"` + WebhookMethod string `json:"webhook_method,omitempty"` + WebhookData string `json:"webhook_data,omitempty"` + WebhookHeaders string `json:"webhook_headers,omitempty"` + WebhookExpectStatus int `json:"webhook_expect_status,omitempty"` + WebhookRetryStatus string `json:"webhook_retry_status,omitempty"` + WebhookRetryAttempts int `json:"webhook_retry_attempts,omitempty"` + WebhookRetryDelaySeconds int `json:"webhook_retry_delay_seconds,omitempty"` + WebhookRetryMaxJitterSeconds int `json:"webhook_retry_max_jitter_seconds,omitempty"` + FilterId int `json:"-"` } type FilterExternalType string @@ -162,79 +166,83 @@ const ( ) type FilterUpdate struct { - ID int `json:"id"` - Name *string `json:"name,omitempty"` - Enabled *bool `json:"enabled,omitempty"` - MinSize *string `json:"min_size,omitempty"` - MaxSize *string `json:"max_size,omitempty"` - Delay *int `json:"delay,omitempty"` - Priority *int32 `json:"priority,omitempty"` - MaxDownloads *int `json:"max_downloads,omitempty"` - MaxDownloadsUnit *FilterMaxDownloadsUnit `json:"max_downloads_unit,omitempty"` - MatchReleases *string `json:"match_releases,omitempty"` - ExceptReleases *string `json:"except_releases,omitempty"` - UseRegex *bool `json:"use_regex,omitempty"` - MatchReleaseGroups *string `json:"match_release_groups,omitempty"` - ExceptReleaseGroups *string `json:"except_release_groups,omitempty"` - MatchReleaseTags *string `json:"match_release_tags,omitempty"` - ExceptReleaseTags *string `json:"except_release_tags,omitempty"` - UseRegexReleaseTags *bool `json:"use_regex_release_tags,omitempty"` - MatchDescription *string `json:"match_description,omitempty"` - ExceptDescription *string `json:"except_description,omitempty"` - UseRegexDescription *bool `json:"use_regex_description,omitempty"` - Scene *bool `json:"scene,omitempty"` - Origins *[]string `json:"origins,omitempty"` - ExceptOrigins *[]string `json:"except_origins,omitempty"` - Bonus *[]string `json:"bonus,omitempty"` - Freeleech *bool `json:"freeleech,omitempty"` - FreeleechPercent *string `json:"freeleech_percent,omitempty"` - SmartEpisode *bool `json:"smart_episode,omitempty"` - Shows *string `json:"shows,omitempty"` - Seasons *string `json:"seasons,omitempty"` - Episodes *string `json:"episodes,omitempty"` - Resolutions *[]string `json:"resolutions,omitempty"` // SD, 480i, 480p, 576p, 720p, 810p, 1080i, 1080p. - Codecs *[]string `json:"codecs,omitempty"` // XviD, DivX, x264, h.264 (or h264), mpeg2 (or mpeg-2), VC-1 (or VC1), WMV, Remux, h.264 Remux (or h264 Remux), VC-1 Remux (or VC1 Remux). - Sources *[]string `json:"sources,omitempty"` // DSR, PDTV, HDTV, HR.PDTV, HR.HDTV, DVDRip, DVDScr, BDr, BD5, BD9, BDRip, BRRip, DVDR, MDVDR, HDDVD, HDDVDRip, BluRay, WEB-DL, TVRip, CAM, R5, TELESYNC, TS, TELECINE, TC. TELESYNC and TS are synonyms (you don't need both). Same for TELECINE and TC - Containers *[]string `json:"containers,omitempty"` - MatchHDR *[]string `json:"match_hdr,omitempty"` - ExceptHDR *[]string `json:"except_hdr,omitempty"` - MatchOther *[]string `json:"match_other,omitempty"` - ExceptOther *[]string `json:"except_other,omitempty"` - Years *string `json:"years,omitempty"` - Artists *string `json:"artists,omitempty"` - Albums *string `json:"albums,omitempty"` - MatchReleaseTypes *[]string `json:"match_release_types,omitempty"` // Album,Single,EP - ExceptReleaseTypes *string `json:"except_release_types,omitempty"` - Formats *[]string `json:"formats,omitempty"` // MP3, FLAC, Ogg, AAC, AC3, DTS - Quality *[]string `json:"quality,omitempty"` // 192, 320, APS (VBR), V2 (VBR), V1 (VBR), APX (VBR), V0 (VBR), q8.x (VBR), Lossless, 24bit Lossless, Other - Media *[]string `json:"media,omitempty"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other - PerfectFlac *bool `json:"perfect_flac,omitempty"` - Cue *bool `json:"cue,omitempty"` - Log *bool `json:"log,omitempty"` - LogScore *int `json:"log_score,omitempty"` - MatchCategories *string `json:"match_categories,omitempty"` - ExceptCategories *string `json:"except_categories,omitempty"` - MatchUploaders *string `json:"match_uploaders,omitempty"` - ExceptUploaders *string `json:"except_uploaders,omitempty"` - MatchLanguage *[]string `json:"match_language,omitempty"` - ExceptLanguage *[]string `json:"except_language,omitempty"` - Tags *string `json:"tags,omitempty"` - ExceptTags *string `json:"except_tags,omitempty"` - TagsAny *string `json:"tags_any,omitempty"` - ExceptTagsAny *string `json:"except_tags_any,omitempty"` - TagsMatchLogic *string `json:"tags_match_logic,omitempty"` - ExceptTagsMatchLogic *string `json:"except_tags_match_logic,omitempty"` - ExternalScriptEnabled *bool `json:"external_script_enabled,omitempty"` - ExternalScriptCmd *string `json:"external_script_cmd,omitempty"` - ExternalScriptArgs *string `json:"external_script_args,omitempty"` - ExternalScriptExpectStatus *int `json:"external_script_expect_status,omitempty"` - ExternalWebhookEnabled *bool `json:"external_webhook_enabled,omitempty"` - ExternalWebhookHost *string `json:"external_webhook_host,omitempty"` - ExternalWebhookData *string `json:"external_webhook_data,omitempty"` - ExternalWebhookExpectStatus *int `json:"external_webhook_expect_status,omitempty"` - Actions []*Action `json:"actions,omitempty"` - External []FilterExternal `json:"external,omitempty"` - Indexers []Indexer `json:"indexers,omitempty"` + ID int `json:"id"` + Name *string `json:"name,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + MinSize *string `json:"min_size,omitempty"` + MaxSize *string `json:"max_size,omitempty"` + Delay *int `json:"delay,omitempty"` + Priority *int32 `json:"priority,omitempty"` + MaxDownloads *int `json:"max_downloads,omitempty"` + MaxDownloadsUnit *FilterMaxDownloadsUnit `json:"max_downloads_unit,omitempty"` + MatchReleases *string `json:"match_releases,omitempty"` + ExceptReleases *string `json:"except_releases,omitempty"` + UseRegex *bool `json:"use_regex,omitempty"` + MatchReleaseGroups *string `json:"match_release_groups,omitempty"` + ExceptReleaseGroups *string `json:"except_release_groups,omitempty"` + MatchReleaseTags *string `json:"match_release_tags,omitempty"` + ExceptReleaseTags *string `json:"except_release_tags,omitempty"` + UseRegexReleaseTags *bool `json:"use_regex_release_tags,omitempty"` + MatchDescription *string `json:"match_description,omitempty"` + ExceptDescription *string `json:"except_description,omitempty"` + UseRegexDescription *bool `json:"use_regex_description,omitempty"` + Scene *bool `json:"scene,omitempty"` + Origins *[]string `json:"origins,omitempty"` + ExceptOrigins *[]string `json:"except_origins,omitempty"` + Bonus *[]string `json:"bonus,omitempty"` + Freeleech *bool `json:"freeleech,omitempty"` + FreeleechPercent *string `json:"freeleech_percent,omitempty"` + SmartEpisode *bool `json:"smart_episode,omitempty"` + Shows *string `json:"shows,omitempty"` + Seasons *string `json:"seasons,omitempty"` + Episodes *string `json:"episodes,omitempty"` + Resolutions *[]string `json:"resolutions,omitempty"` // SD, 480i, 480p, 576p, 720p, 810p, 1080i, 1080p. + Codecs *[]string `json:"codecs,omitempty"` // XviD, DivX, x264, h.264 (or h264), mpeg2 (or mpeg-2), VC-1 (or VC1), WMV, Remux, h.264 Remux (or h264 Remux), VC-1 Remux (or VC1 Remux). + Sources *[]string `json:"sources,omitempty"` // DSR, PDTV, HDTV, HR.PDTV, HR.HDTV, DVDRip, DVDScr, BDr, BD5, BD9, BDRip, BRRip, DVDR, MDVDR, HDDVD, HDDVDRip, BluRay, WEB-DL, TVRip, CAM, R5, TELESYNC, TS, TELECINE, TC. TELESYNC and TS are synonyms (you don't need both). Same for TELECINE and TC + Containers *[]string `json:"containers,omitempty"` + MatchHDR *[]string `json:"match_hdr,omitempty"` + ExceptHDR *[]string `json:"except_hdr,omitempty"` + MatchOther *[]string `json:"match_other,omitempty"` + ExceptOther *[]string `json:"except_other,omitempty"` + Years *string `json:"years,omitempty"` + Artists *string `json:"artists,omitempty"` + Albums *string `json:"albums,omitempty"` + MatchReleaseTypes *[]string `json:"match_release_types,omitempty"` // Album,Single,EP + ExceptReleaseTypes *string `json:"except_release_types,omitempty"` + Formats *[]string `json:"formats,omitempty"` // MP3, FLAC, Ogg, AAC, AC3, DTS + Quality *[]string `json:"quality,omitempty"` // 192, 320, APS (VBR), V2 (VBR), V1 (VBR), APX (VBR), V0 (VBR), q8.x (VBR), Lossless, 24bit Lossless, Other + Media *[]string `json:"media,omitempty"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other + PerfectFlac *bool `json:"perfect_flac,omitempty"` + Cue *bool `json:"cue,omitempty"` + Log *bool `json:"log,omitempty"` + LogScore *int `json:"log_score,omitempty"` + MatchCategories *string `json:"match_categories,omitempty"` + ExceptCategories *string `json:"except_categories,omitempty"` + MatchUploaders *string `json:"match_uploaders,omitempty"` + ExceptUploaders *string `json:"except_uploaders,omitempty"` + MatchLanguage *[]string `json:"match_language,omitempty"` + ExceptLanguage *[]string `json:"except_language,omitempty"` + Tags *string `json:"tags,omitempty"` + ExceptTags *string `json:"except_tags,omitempty"` + TagsAny *string `json:"tags_any,omitempty"` + ExceptTagsAny *string `json:"except_tags_any,omitempty"` + TagsMatchLogic *string `json:"tags_match_logic,omitempty"` + ExceptTagsMatchLogic *string `json:"except_tags_match_logic,omitempty"` + ExternalScriptEnabled *bool `json:"external_script_enabled,omitempty"` + ExternalScriptCmd *string `json:"external_script_cmd,omitempty"` + ExternalScriptArgs *string `json:"external_script_args,omitempty"` + ExternalScriptExpectStatus *int `json:"external_script_expect_status,omitempty"` + ExternalWebhookEnabled *bool `json:"external_webhook_enabled,omitempty"` + ExternalWebhookHost *string `json:"external_webhook_host,omitempty"` + ExternalWebhookData *string `json:"external_webhook_data,omitempty"` + ExternalWebhookExpectStatus *int `json:"external_webhook_expect_status,omitempty"` + ExternalWebhookRetryStatus *string `json:"external_webhook_retry_status,omitempty"` + ExternalWebhookRetryAttempts *int `json:"external_webhook_retry_attempts,omitempty"` + ExternalWebhookRetryDelaySeconds *int `json:"external_webhook_retry_delay_seconds,omitempty"` + ExternalWebhookRetryMaxJitterSeconds *int `json:"external_webhook_retry_max_jitter_seconds,omitempty"` + Actions []*Action `json:"actions,omitempty"` + External []FilterExternal `json:"external,omitempty"` + Indexers []Indexer `json:"indexers,omitempty"` } func (f Filter) CheckFilter(r *Release) ([]string, bool) { diff --git a/internal/filter/service.go b/internal/filter/service.go index 69e6e32..c7e50d9 100644 --- a/internal/filter/service.go +++ b/internal/filter/service.go @@ -13,14 +13,17 @@ import ( "os" "os/exec" "sort" + "strconv" "strings" "time" "github.com/autobrr/autobrr/internal/domain" "github.com/autobrr/autobrr/internal/indexer" "github.com/autobrr/autobrr/internal/logger" + "github.com/autobrr/autobrr/internal/utils" "github.com/autobrr/autobrr/pkg/errors" + "github.com/avast/retry-go/v4" "github.com/dustin/go-humanize" "github.com/mattn/go-shellwords" "github.com/rs/zerolog" @@ -730,25 +733,53 @@ func (s *service) webhook(ctx context.Context, external domain.FilterExternal, r } } + var opts []retry.Option + + if external.WebhookRetryAttempts > 0 { + option := retry.Attempts(uint(external.WebhookRetryAttempts)) + opts = append(opts, option) + } + if external.WebhookRetryDelaySeconds > 0 { + option := retry.Delay(time.Duration(external.WebhookRetryDelaySeconds) * time.Second) + opts = append(opts, option) + } + if external.WebhookRetryMaxJitterSeconds > 0 { + option := retry.MaxJitter(time.Duration(external.WebhookRetryMaxJitterSeconds) * time.Second) + opts = append(opts, option) + } + start := time.Now() - res, err := client.Do(req) - if err != nil { - return 0, errors.Wrap(err, "could not make request for webhook") - } + statusCode, err := retry.DoWithData( + func() (int, error) { + res, err := client.Do(req) + if err != nil { + return 0, errors.Wrap(err, "could not make request for webhook") + } - defer res.Body.Close() + defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - return 0, errors.Wrap(err, "could not read request body") - } + body, err := io.ReadAll(res.Body) + if err != nil { + return 0, errors.Wrap(err, "could not read request body") + } - if len(body) > 0 { - s.log.Debug().Msgf("filter external webhook response status: %d body: %s", res.StatusCode, body) - } + if len(body) > 0 { + s.log.Debug().Msgf("filter external webhook response status: %d body: %s", res.StatusCode, body) + } + + if external.WebhookRetryStatus != "" { + retryStatusCodes := strings.Split(strings.ReplaceAll(external.WebhookRetryStatus, " ", ""), ",") + if utils.StrSliceContains(retryStatusCodes, strconv.Itoa(res.StatusCode)) { + return 0, errors.New("retrying webhook request, got status code: %d", res.StatusCode) + } + } + + return res.StatusCode, nil + }, + opts...) s.log.Debug().Msgf("successfully ran external webhook filter to: (%s) payload: (%s) finished in %s", external.WebhookHost, dataArgs, time.Since(start)) - return res.StatusCode, nil + return statusCode, err } diff --git a/web/src/screens/filters/Details.tsx b/web/src/screens/filters/Details.tsx index 5c1c2c5..e314b06 100644 --- a/web/src/screens/filters/Details.tsx +++ b/web/src/screens/filters/Details.tsx @@ -218,6 +218,10 @@ const externalFilterSchema = z.object({ webhook_method: z.string().optional(), webhook_data: z.string().optional(), webhook_expect_status: z.number().optional(), + webhook_retry_status: z.string().optional(), + webhook_retry_attempts: z.number().optional(), + webhook_retry_delay_seconds: z.number().optional(), + webhook_retry_max_jitter_seconds: z.number().optional(), }); const indexerSchema = z.object({ diff --git a/web/src/screens/filters/External.tsx b/web/src/screens/filters/External.tsx index 7198cd7..5a9f9bb 100644 --- a/web/src/screens/filters/External.tsx +++ b/web/src/screens/filters/External.tsx @@ -312,12 +312,31 @@ const TypeForm = ({ external, idx }: TypeFormProps) => { rows={5} placeholder={"Request data: { \"key\": \"value\" }"} /> - + + + + ); diff --git a/web/src/screens/filters/List.tsx b/web/src/screens/filters/List.tsx index 83d559c..dc0bbd7 100644 --- a/web/src/screens/filters/List.tsx +++ b/web/src/screens/filters/List.tsx @@ -292,6 +292,10 @@ const FilterItemDropdown = ({ filter, onToggle }: FilterItemDropdownProps) => { external_webhook_host: any; external_webhook_data: any; external_webhook_expect_status: any; + external_webhook_retry_status: any; + external_webhook_retry_attempts: any; + external_webhook_retry_delay_seconds: any; + external_webhook_retry_max_jitter_seconds: any; }; const completeFilter = await APIClient.filters.getByID(filter.id) as Partial; @@ -313,6 +317,10 @@ const FilterItemDropdown = ({ filter, onToggle }: FilterItemDropdownProps) => { delete completeFilter.external_webhook_host; delete completeFilter.external_webhook_data; delete completeFilter.external_webhook_expect_status; + delete completeFilter.external_webhook_retry_status; + delete completeFilter.external_webhook_retry_attempts; + delete completeFilter.external_webhook_retry_delay_seconds; + delete completeFilter.external_webhook_retry_max_jitter_seconds; // Remove properties with default values from the exported filter to minimize the size of the JSON string ["enabled", "priority", "smart_episode", "resolutions", "sources", "codecs", "containers", "tags_match_logic", "except_tags_match_logic"].forEach((key) => { diff --git a/web/src/types/Filter.d.ts b/web/src/types/Filter.d.ts index d7cbba7..55b4a72 100644 --- a/web/src/types/Filter.d.ts +++ b/web/src/types/Filter.d.ts @@ -130,5 +130,9 @@ interface ExternalFilter { webhook_data?: string, webhook_headers?: string; webhook_expect_status?: number; + webhook_retry_status?: string, + webhook_retry_attempts?: number; + webhook_retry_delay_seconds?: number; + webhook_retry_max_jitter_seconds?: number; filter_id?: number; }