idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 00:39:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(filters): external webhook retry on status codes (#1206)

Browse source 
* feat: external filter retry status codes

* chore: go mod tidy

* fix(database): migrations

---------

Co-authored-by: ze0s <ze0s@riseup.net>
This commit is contained in:
Steven Kreitzer 2023-10-27 10:37:57 -05:00 • committed by GitHub
parent 40a1a4c014
commit 2080136669
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 330 additions and 165 deletions
Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -13,6 +13,7 @@ require (
github.com/autobrr/go-qbittorrent v1.6.0
github.com/autobrr/go-rtorrent v1.10.0
github.com/avast/retry-go v3.0.0+incompatible
github.com/avast/retry-go/v4 v4.5.0
github.com/dcarbone/zadapters/zstdlog v1.0.0
github.com/dustin/go-humanize v1.0.1
github.com/ergochat/irc-go v0.4.0

2
go.sum
View file

@ -102,6 +102,8 @@ github.com/autobrr/sse/v2 v2.0.0-20230520125637-530e06346d7d h1:9EGCYgeugAVWLBAt
github.com/autobrr/sse/v2 v2.0.0-20230520125637-530e06346d7d/go.mod h1:zCozZ9lp4DE340T2+wfMPL/eoQwLVIGDOCKCDEFwTQU=
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/avast/retry-go/v4 v4.5.0 h1:QoRAZZ90cj5oni2Lsgl2GW8mNTnUCnmpx/iKpwVisHg=
github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5Aq1fboC3+I=
github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=

68
internal/database/filter.go
View file

@ -247,6 +247,10 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
"fe.webhook_data",
"fe.webhook_headers",
"fe.webhook_expect_status",
"fe.webhook_retry_status",
"fe.webhook_retry_attempts",
"fe.webhook_retry_delay_seconds",
"fe.webhook_retry_max_jitter_seconds",
).
From("filter f").
LeftJoin("filter_external fe ON f.id = fe.filter_id").
@ -276,8 +280,8 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
var delay, maxDownloads, logScore sql.NullInt32
// filter external
var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData sql.NullString
var extId, extIndex, extWebhookStatus, extExecStatus sql.NullInt32
var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData, extWebhookRetryStatus sql.NullString
var extId, extIndex, extWebhookStatus, extWebhookRetryAttempts, extWebhookDelaySeconds, extWebhookRetryJitterSeconds, extExecStatus sql.NullInt32
var extEnabled sql.NullBool
if err := rows.Scan(
@ -354,6 +358,10 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
&extWebhookData,
&extWebhookHeaders,
&extWebhookStatus,
&extWebhookRetryStatus,
&extWebhookRetryAttempts,
&extWebhookDelaySeconds,
&extWebhookRetryJitterSeconds,
); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}
@ -409,6 +417,10 @@ func (r *FilterRepo) FindByID(ctx context.Context, filterID int) (*domain.Filter
WebhookData: extWebhookData.String,
WebhookHeaders: extWebhookHeaders.String,
WebhookExpectStatus: int(extWebhookStatus.Int32),
WebhookRetryStatus: extWebhookRetryStatus.String,
WebhookRetryAttempts: int(extWebhookRetryAttempts.Int32),
WebhookRetryDelaySeconds: int(extWebhookDelaySeconds.Int32),
WebhookRetryMaxJitterSeconds: int(extWebhookRetryJitterSeconds.Int32),
}
externalMap[external.ID] = external
}
@ -502,6 +514,10 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
"fe.webhook_data",
"fe.webhook_headers",
"fe.webhook_expect_status",
"fe.webhook_retry_status",
"fe.webhook_retry_attempts",
"fe.webhook_retry_delay_seconds",
"fe.webhook_retry_max_jitter_seconds",
"fe.filter_id",
).
From("filter f").
@ -537,8 +553,8 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
var delay, maxDownloads, logScore sql.NullInt32
// filter external
var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData sql.NullString
var extId, extIndex, extWebhookStatus, extExecStatus, extFilterId sql.NullInt32
var extName, extType, extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData, extWebhookRetryStatus sql.NullString
var extId, extIndex, extWebhookStatus, extWebhookRetryAttempts, extWebhookDelaySeconds, extWebhookRetryJitterSeconds, extExecStatus, extFilterId sql.NullInt32
var extEnabled sql.NullBool
if err := rows.Scan(
@ -615,6 +631,10 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
&extWebhookData,
&extWebhookHeaders,
&extWebhookStatus,
&extWebhookRetryStatus,
&extWebhookRetryAttempts,
&extWebhookDelaySeconds,
&extWebhookRetryJitterSeconds,
&extFilterId,
); err != nil {
return nil, errors.Wrap(err, "error scanning row")
@ -671,6 +691,10 @@ func (r *FilterRepo) findByIndexerIdentifier(ctx context.Context, indexer string
WebhookData: extWebhookData.String,
WebhookHeaders: extWebhookHeaders.String,
WebhookExpectStatus: int(extWebhookStatus.Int32),
WebhookRetryStatus: extWebhookRetryStatus.String,
WebhookRetryAttempts: int(extWebhookRetryAttempts.Int32),
WebhookRetryDelaySeconds: int(extWebhookDelaySeconds.Int32),
WebhookRetryMaxJitterSeconds: int(extWebhookRetryJitterSeconds.Int32),
FilterId: int(extFilterId.Int32),
}
externalMap[external.FilterId] = append(externalMap[external.FilterId], external)
@ -709,6 +733,10 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int)
"fe.webhook_data",
"fe.webhook_headers",
"fe.webhook_expect_status",
"fe.webhook_retry_status",
"fe.webhook_retry_attempts",
"fe.webhook_retry_delay_seconds",
"fe.webhook_retry_max_jitter_seconds",
).
From("filter_external fe").
Where(sq.Eq{"fe.filter_id": filterId})
@ -732,8 +760,8 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int)
var external domain.FilterExternal
// filter external
var extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData sql.NullString
var extWebhookStatus, extExecStatus sql.NullInt32
var extExecCmd, extExecArgs, extWebhookHost, extWebhookMethod, extWebhookHeaders, extWebhookData, extWebhookRetryStatus sql.NullString
var extWebhookStatus, extWebhookRetryAttempts, extWebhookDelaySeconds, extWebhookRetryJitterSeconds, extExecStatus sql.NullInt32
if err := rows.Scan(
&external.ID,
@ -749,6 +777,10 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int)
&extWebhookData,
&extWebhookHeaders,
&extWebhookStatus,
&extWebhookRetryStatus,
&extWebhookRetryAttempts,
&extWebhookDelaySeconds,
&extWebhookRetryJitterSeconds,
); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}
@ -762,6 +794,10 @@ func (r *FilterRepo) FindExternalFiltersByID(ctx context.Context, filterId int)
external.WebhookData = extWebhookData.String
external.WebhookHeaders = extWebhookHeaders.String
external.WebhookExpectStatus = int(extWebhookStatus.Int32)
external.WebhookRetryStatus = extWebhookRetryStatus.String
external.WebhookRetryAttempts = int(extWebhookRetryAttempts.Int32)
external.WebhookRetryDelaySeconds = int(extWebhookDelaySeconds.Int32)
external.WebhookRetryMaxJitterSeconds = int(extWebhookRetryJitterSeconds.Int32)
externalFilters = append(externalFilters, external)
}
@ -1182,6 +1218,18 @@ func (r *FilterRepo) UpdatePartial(ctx context.Context, filter domain.FilterUpda
if filter.ExternalWebhookExpectStatus != nil {
q = q.Set("external_webhook_expect_status", filter.ExternalWebhookExpectStatus)
}
if filter.ExternalWebhookRetryStatus != nil {
q = q.Set("external_webhook_retry_status", filter.ExternalWebhookRetryStatus)
}
if filter.ExternalWebhookRetryAttempts != nil {
q = q.Set("external_webhook_retry_attempts", filter.ExternalWebhookRetryAttempts)
}
if filter.ExternalWebhookRetryDelaySeconds != nil {
q = q.Set("external_webhook_retry_delay_seconds", filter.ExternalWebhookRetryDelaySeconds)
}
if filter.ExternalWebhookRetryMaxJitterSeconds != nil {
q = q.Set("external_webhook_retry_max_jitter_seconds", filter.ExternalWebhookRetryMaxJitterSeconds)
}
q = q.Where(sq.Eq{"id": filter.ID})
@ -1462,6 +1510,10 @@ func (r *FilterRepo) StoreFilterExternal(ctx context.Context, filterID int, exte
"webhook_data",
"webhook_headers",
"webhook_expect_status",
"webhook_retry_status",
"webhook_retry_attempts",
"webhook_retry_delay_seconds",
"webhook_retry_max_jitter_seconds",
"filter_id",
)
@ -1479,6 +1531,10 @@ func (r *FilterRepo) StoreFilterExternal(ctx context.Context, filterID int, exte
toNullString(external.WebhookData),
toNullString(external.WebhookHeaders),
toNullInt32(int32(external.WebhookExpectStatus)),
toNullString(external.WebhookRetryStatus),
toNullInt32(int32(external.WebhookRetryAttempts)),
toNullInt32(int32(external.WebhookRetryDelaySeconds)),
toNullInt32(int32(external.WebhookRetryMaxJitterSeconds)),
filterID,
)
}

16
internal/database/postgres_migrate.go
View file

@ -146,6 +146,10 @@ CREATE TABLE filter_external
webhook_data TEXT,
webhook_headers TEXT,
webhook_expect_status INTEGER,
webhook_retry_status TEXT,
webhook_retry_attempts INTEGER,
webhook_retry_delay_seconds INTEGER,
webhook_retry_max_jitter_seconds INTEGER,
filter_id INTEGER NOT NULL,
FOREIGN KEY (filter_id) REFERENCES filter(id) ON DELETE CASCADE
);
@ -797,5 +801,17 @@ CREATE INDEX feed_cache_feed_id_key_index
`,
`ALTER TABLE action
ADD COLUMN external_client_id INTEGER;
`,
`ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_status TEXT;
ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_attempts INTEGER;
ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_delay_seconds INTEGER;
ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_max_jitter_seconds INTEGER;
`,
}

16
internal/database/sqlite_migrate.go
View file

@ -146,6 +146,10 @@ CREATE TABLE filter_external
webhook_data TEXT,
webhook_headers TEXT,
webhook_expect_status INTEGER,
webhook_retry_status TEXT,
webhook_retry_attempts INTEGER,
webhook_retry_delay_seconds INTEGER,
webhook_retry_max_jitter_seconds INTEGER,
filter_id INTEGER NOT NULL,
FOREIGN KEY (filter_id) REFERENCES filter(id) ON DELETE CASCADE
);
@ -1348,5 +1352,17 @@ CREATE INDEX feed_cache_feed_id_key_index
`,
`ALTER TABLE action
ADD COLUMN external_client_id INTEGER;
`,
`ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_status TEXT;
ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_attempts INTEGER;
ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_delay_seconds INTEGER;
ALTER TABLE filter_external
ADD COLUMN external_webhook_retry_max_jitter_seconds INTEGER;
`,
}

8
internal/domain/filter.go
View file

@ -151,6 +151,10 @@ type FilterExternal struct {
WebhookData string `json:"webhook_data,omitempty"`
WebhookHeaders string `json:"webhook_headers,omitempty"`
WebhookExpectStatus int `json:"webhook_expect_status,omitempty"`
WebhookRetryStatus string `json:"webhook_retry_status,omitempty"`
WebhookRetryAttempts int `json:"webhook_retry_attempts,omitempty"`
WebhookRetryDelaySeconds int `json:"webhook_retry_delay_seconds,omitempty"`
WebhookRetryMaxJitterSeconds int `json:"webhook_retry_max_jitter_seconds,omitempty"`
FilterId int `json:"-"`
}
@ -232,6 +236,10 @@ type FilterUpdate struct {
ExternalWebhookHost *string `json:"external_webhook_host,omitempty"`
ExternalWebhookData *string `json:"external_webhook_data,omitempty"`
ExternalWebhookExpectStatus *int `json:"external_webhook_expect_status,omitempty"`
ExternalWebhookRetryStatus *string `json:"external_webhook_retry_status,omitempty"`
ExternalWebhookRetryAttempts *int `json:"external_webhook_retry_attempts,omitempty"`
ExternalWebhookRetryDelaySeconds *int `json:"external_webhook_retry_delay_seconds,omitempty"`
ExternalWebhookRetryMaxJitterSeconds *int `json:"external_webhook_retry_max_jitter_seconds,omitempty"`
Actions []*Action `json:"actions,omitempty"`
External []FilterExternal `json:"external,omitempty"`
Indexers []Indexer `json:"indexers,omitempty"`

33
internal/filter/service.go
View file

@ -13,14 +13,17 @@ import (
"os"
"os/exec"
"sort"
"strconv"
"strings"
"time"
"github.com/autobrr/autobrr/internal/domain"
"github.com/autobrr/autobrr/internal/indexer"
"github.com/autobrr/autobrr/internal/logger"
"github.com/autobrr/autobrr/internal/utils"
"github.com/autobrr/autobrr/pkg/errors"
"github.com/avast/retry-go/v4"
"github.com/dustin/go-humanize"
"github.com/mattn/go-shellwords"
"github.com/rs/zerolog"
@ -730,8 +733,25 @@ func (s *service) webhook(ctx context.Context, external domain.FilterExternal, r
}
}
var opts []retry.Option
if external.WebhookRetryAttempts > 0 {
option := retry.Attempts(uint(external.WebhookRetryAttempts))
opts = append(opts, option)
}
if external.WebhookRetryDelaySeconds > 0 {
option := retry.Delay(time.Duration(external.WebhookRetryDelaySeconds) * time.Second)
opts = append(opts, option)
}
if external.WebhookRetryMaxJitterSeconds > 0 {
option := retry.MaxJitter(time.Duration(external.WebhookRetryMaxJitterSeconds) * time.Second)
opts = append(opts, option)
}
start := time.Now()
statusCode, err := retry.DoWithData(
func() (int, error) {
res, err := client.Do(req)
if err != nil {
return 0, errors.Wrap(err, "could not make request for webhook")
@ -748,7 +768,18 @@ func (s *service) webhook(ctx context.Context, external domain.FilterExternal, r
s.log.Debug().Msgf("filter external webhook response status: %d body: %s", res.StatusCode, body)
}
s.log.Debug().Msgf("successfully ran external webhook filter to: (%s) payload: (%s) finished in %s", external.WebhookHost, dataArgs, time.Since(start))
if external.WebhookRetryStatus != "" {
retryStatusCodes := strings.Split(strings.ReplaceAll(external.WebhookRetryStatus, " ", ""), ",")
if utils.StrSliceContains(retryStatusCodes, strconv.Itoa(res.StatusCode)) {
return 0, errors.New("retrying webhook request, got status code: %d", res.StatusCode)
}
}
return res.StatusCode, nil
},
opts...)
s.log.Debug().Msgf("successfully ran external webhook filter to: (%s) payload: (%s) finished in %s", external.WebhookHost, dataArgs, time.Since(start))
return statusCode, err
}

4
web/src/screens/filters/Details.tsx
View file

@ -218,6 +218,10 @@ const externalFilterSchema = z.object({
webhook_method: z.string().optional(),
webhook_data: z.string().optional(),
webhook_expect_status: z.number().optional(),
webhook_retry_status: z.string().optional(),
webhook_retry_attempts: z.number().optional(),
webhook_retry_delay_seconds: z.number().optional(),
webhook_retry_max_jitter_seconds: z.number().optional(),
});
const indexerSchema = z.object({

23
web/src/screens/filters/External.tsx
View file

@ -312,12 +312,31 @@ const TypeForm = ({ external, idx }: TypeFormProps) => {
rows={5}
placeholder={"Request data: { \"key\": \"value\" }"}
/>
<NumberField
name={`external.${idx}.webhook_expect_status`}
label="Expected http status"
label="Expected http status code"
placeholder="200"
/>
<TextField
name={`external.${idx}.webhook_retry_status`}
label="Retry http status code(s)"
placeholder="Retry on status eg. 202, 204"
/>
<NumberField
name={`external.${idx}.webhook_retry_attempts`}
label="Maximum retry attempts"
placeholder="10"
/>
<NumberField
name={`external.${idx}.webhook_retry_delay_seconds`}
label="Retry delay in seconds"
placeholder="1"
/>
<NumberField
name={`external.${idx}.webhook_retry_max_jitter_seconds`}
label="Max jitter in seconds"
placeholder="1"
/>
</div>
);

8
web/src/screens/filters/List.tsx
View file

@ -292,6 +292,10 @@ const FilterItemDropdown = ({ filter, onToggle }: FilterItemDropdownProps) => {
external_webhook_host: any;
external_webhook_data: any;
external_webhook_expect_status: any;
external_webhook_retry_status: any;
external_webhook_retry_attempts: any;
external_webhook_retry_delay_seconds: any;
external_webhook_retry_max_jitter_seconds: any;
};
const completeFilter = await APIClient.filters.getByID(filter.id) as Partial<CompleteFilterType>;
@ -313,6 +317,10 @@ const FilterItemDropdown = ({ filter, onToggle }: FilterItemDropdownProps) => {
delete completeFilter.external_webhook_host;
delete completeFilter.external_webhook_data;
delete completeFilter.external_webhook_expect_status;
delete completeFilter.external_webhook_retry_status;
delete completeFilter.external_webhook_retry_attempts;
delete completeFilter.external_webhook_retry_delay_seconds;
delete completeFilter.external_webhook_retry_max_jitter_seconds;
// Remove properties with default values from the exported filter to minimize the size of the JSON string
["enabled", "priority", "smart_episode", "resolutions", "sources", "codecs", "containers", "tags_match_logic", "except_tags_match_logic"].forEach((key) => {

4
web/src/types/Filter.d.ts vendored
View file

@ -130,5 +130,9 @@ interface ExternalFilter {
webhook_data?: string,
webhook_headers?: string;
webhook_expect_status?: number;
webhook_retry_status?: string,
webhook_retry_attempts?: number;
webhook_retry_delay_seconds?: number;
webhook_retry_max_jitter_seconds?: number;
filter_id?: number;
}