feat: add support for proxies to use with IRC and Indexers (#1421)

* feat: add support for proxies

* fix(http): release handler

* fix(migrations): define proxy early

* fix(migrations): pg proxy

* fix(proxy): list update delete

* fix(proxy): remove log and imports

* feat(irc): use proxy

* feat(irc): tests

* fix(web): update imports for ProxyForms.tsx

* fix(database): migration

* feat(proxy): test

* feat(proxy): validate proxy type

* feat(proxy): validate and test

* feat(proxy): improve validate and test

* feat(proxy): fix db schema

* feat(proxy): add db tests

* feat(proxy): handle http errors

* fix(http): imports

* feat(proxy): use proxy for indexer downloads

* feat(proxy): indexerforms select proxy

* feat(proxy): handle torrent download

* feat(proxy): skip if disabled

* feat(proxy): imports

* feat(proxy): implement in Feeds

* feat(proxy): update helper text indexer proxy

* feat(proxy): add internal cache

internal/domain/action.go

@@ -5,7 +5,6 @@ package domain
 
 import (
 	"context"
-	"os"
 	"strings"
 
 	"github.com/autobrr/autobrr/pkg/errors"
@@ -60,43 +59,69 @@ type Action struct {
 	Client                   *DownloadClient     `json:"client,omitempty"`
 }
 
-// ParseMacros parse all macros on action
-func (a *Action) ParseMacros(release *Release) error {
-	var err error
-
+// CheckMacrosNeedTorrentTmpFile check if macros needs torrent downloaded
+func (a *Action) CheckMacrosNeedTorrentTmpFile(release *Release) bool {
 	if release.TorrentTmpFile == "" &&
-		(strings.Contains(a.ExecArgs, "TorrentPathName") || strings.Contains(a.ExecArgs, "TorrentDataRawBytes") ||
-			strings.Contains(a.WebhookData, "TorrentPathName") || strings.Contains(a.WebhookData, "TorrentDataRawBytes") ||
-			strings.Contains(a.SavePath, "TorrentPathName") || a.Type == ActionTypeWatchFolder) {
-		if err := release.DownloadTorrentFile(); err != nil {
-			return errors.Wrap(err, "webhook: could not download torrent file for release: %v", release.TorrentName)
-		}
+		(strings.Contains(a.ExecArgs, "TorrentPathName") ||
+			strings.Contains(a.ExecArgs, "TorrentDataRawBytes") ||
+			strings.Contains(a.WebhookData, "TorrentPathName") ||
+			strings.Contains(a.WebhookData, "TorrentDataRawBytes") ||
+			strings.Contains(a.SavePath, "TorrentPathName") ||
+			a.Type == ActionTypeWatchFolder) {
+		return true
 	}
 
+	return false
+}
+
+func (a *Action) CheckMacrosNeedRawDataBytes(release *Release) bool {
 	// if webhook data contains TorrentDataRawBytes, lets read the file into bytes we can then use in the macro
 	if len(release.TorrentDataRawBytes) == 0 &&
 		(strings.Contains(a.ExecArgs, "TorrentDataRawBytes") || strings.Contains(a.WebhookData, "TorrentDataRawBytes") ||
 			a.Type == ActionTypeWatchFolder) {
-		t, err := os.ReadFile(release.TorrentTmpFile)
-		if err != nil {
-			return errors.Wrap(err, "could not read torrent file: %v", release.TorrentTmpFile)
-		}
-
-		release.TorrentDataRawBytes = t
+		return true
 	}
 
+	return false
+}
+
+// ParseMacros parse all macros on action
+func (a *Action) ParseMacros(release *Release) error {
+	var err error
+
 	m := NewMacro(*release)
 
 	a.ExecArgs, err = m.Parse(a.ExecArgs)
-	a.WatchFolder, err = m.Parse(a.WatchFolder)
-	a.Category, err = m.Parse(a.Category)
-	a.Tags, err = m.Parse(a.Tags)
-	a.Label, err = m.Parse(a.Label)
-	a.SavePath, err = m.Parse(a.SavePath)
-	a.WebhookData, err = m.Parse(a.WebhookData)
-
 	if err != nil {
-		return errors.Wrap(err, "could not parse macros for action: %v", a.Name)
+		return errors.Wrap(err, "could not parse exec args")
+	}
+
+	a.WatchFolder, err = m.Parse(a.WatchFolder)
+	if err != nil {
+		return errors.Wrap(err, "could not parse watch folder")
+	}
+
+	a.Category, err = m.Parse(a.Category)
+	if err != nil {
+		return errors.Wrap(err, "could not parse category")
+	}
+
+	a.Tags, err = m.Parse(a.Tags)
+	if err != nil {
+		return errors.Wrap(err, "could not parse tags")
+	}
+
+	a.Label, err = m.Parse(a.Label)
+	if err != nil {
+		return errors.Wrap(err, "could not parse label")
+	}
+	a.SavePath, err = m.Parse(a.SavePath)
+	if err != nil {
+		return errors.Wrap(err, "could not parse save_path")
+	}
+	a.WebhookData, err = m.Parse(a.WebhookData)
+	if err != nil {
+		return errors.Wrap(err, "could not parse webhook_data")
 	}
 
 	return nil

internal/domain/error.go

@@ -3,8 +3,14 @@
 
 package domain
 
-import "database/sql"
+import (
+	"database/sql"
+
+	"github.com/autobrr/autobrr/pkg/errors"
+)
 
 var (
 	ErrRecordNotFound = sql.ErrNoRows
+	ErrUpdateFailed   = errors.New("update failed")
+	ErrDeleteFailed   = errors.New("delete failed")
 )

internal/domain/feed.go

@@ -53,6 +53,11 @@ type Feed struct {
 	LastRun      time.Time         `json:"last_run"`
 	LastRunData  string            `json:"last_run_data"`
 	NextRun      time.Time         `json:"next_run"`
+
+	// belongs to Indexer
+	ProxyID  int64
+	UseProxy bool
+	Proxy    *Proxy
 }
 
 type FeedSettingsJSON struct {

internal/domain/indexer.go

@@ -34,6 +34,9 @@ type Indexer struct {
 	Enabled            bool              `json:"enabled"`
 	Implementation     string            `json:"implementation"`
 	BaseURL            string            `json:"base_url,omitempty"`
+	UseProxy           bool              `json:"use_proxy"`
+	Proxy              *Proxy            `json:"proxy"`
+	ProxyID            int64             `json:"proxy_id"`
 	Settings           map[string]string `json:"settings,omitempty"`
 }
 
@@ -66,6 +69,8 @@ type IndexerDefinition struct {
 	Protocol           string            `json:"protocol"`
 	URLS               []string          `json:"urls"`
 	Supports           []string          `json:"supports"`
+	UseProxy           bool              `json:"use_proxy"`
+	ProxyID            int64             `json:"proxy_id"`
 	Settings           []IndexerSetting  `json:"settings,omitempty"`
 	SettingsMap        map[string]string `json:"-"`
 	IRC                *IndexerIRC       `json:"irc,omitempty"`

internal/domain/irc.go

@@ -48,6 +48,9 @@ type IrcNetwork struct {
 	InviteCommand  string       `json:"invite_command"`
 	UseBouncer     bool         `json:"use_bouncer"`
 	BouncerAddr    string       `json:"bouncer_addr"`
+	UseProxy       bool         `json:"use_proxy"`
+	ProxyId        int64        `json:"proxy_id"`
+	Proxy          *Proxy       `json:"proxy"`
 	BotMode        bool         `json:"bot_mode"`
 	Channels       []IrcChannel `json:"channels"`
 	Connected      bool         `json:"connected"`
@@ -70,6 +73,9 @@ type IrcNetworkWithHealth struct {
 	BotMode          bool                `json:"bot_mode"`
 	CurrentNick      string              `json:"current_nick"`
 	PreferredNick    string              `json:"preferred_nick"`
+	UseProxy         bool                `json:"use_proxy"`
+	ProxyId          int64               `json:"proxy_id"`
+	Proxy            *Proxy              `json:"proxy"`
 	Channels         []ChannelWithHealth `json:"channels"`
 	Connected        bool                `json:"connected"`
 	ConnectedSince   time.Time           `json:"connected_since"`

internal/domain/macros.go

@@ -163,6 +163,8 @@ func (m Macro) Parse(text string) (string, error) {
 		return "", nil
 	}
 
+	// TODO implement template cache
+
 	// setup template
 	tmpl, err := template.New("macro").Funcs(sprig.TxtFuncMap()).Parse(text)
 	if err != nil {

internal/domain/proxy.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2021 - 2024, Ludvig Lundgren and the autobrr contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+package domain
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/autobrr/autobrr/pkg/errors"
+)
+
+type ProxyRepo interface {
+	Store(ctx context.Context, p *Proxy) error
+	Update(ctx context.Context, p *Proxy) error
+	List(ctx context.Context) ([]Proxy, error)
+	Delete(ctx context.Context, id int64) error
+	FindByID(ctx context.Context, id int64) (*Proxy, error)
+	ToggleEnabled(ctx context.Context, id int64, enabled bool) error
+}
+
+type Proxy struct {
+	ID      int64     `json:"id"`
+	Name    string    `json:"name"`
+	Enabled bool      `json:"enabled"`
+	Type    ProxyType `json:"type"`
+	Addr    string    `json:"addr"`
+	User    string    `json:"user"`
+	Pass    string    `json:"pass"`
+	Timeout int       `json:"timeout"`
+}
+
+type ProxyType string
+
+const (
+	ProxyTypeSocks5 = "SOCKS5"
+)
+
+func (p Proxy) ValidProxyType() bool {
+	if p.Type == ProxyTypeSocks5 {
+		return true
+	}
+
+	return false
+}
+
+func (p Proxy) Validate() error {
+	if !p.ValidProxyType() {
+		return errors.New("invalid proxy type: %s", p.Type)
+	}
+
+	if err := ValidateProxyAddr(p.Addr); err != nil {
+		return err
+	}
+
+	if p.Name == "" {
+		return errors.New("name is required")
+	}
+
+	return nil
+}
+
+func ValidateProxyAddr(addr string) error {
+	if addr == "" {
+		return errors.New("addr is required")
+	}
+
+	proxyUrl, err := url.Parse(addr)
+	if err != nil {
+		return errors.Wrap(err, "could not parse proxy url: %s", addr)
+	}
+
+	if proxyUrl.Scheme != "socks5" && proxyUrl.Scheme != "socks5h" {
+		return errors.New("proxy url scheme must be socks5 or socks5h")
+	}
+
+	return nil
+}

internal/domain/release.go

@@ -356,8 +356,6 @@ func (r *Release) ParseString(title string) {
 	r.ParseReleaseTagsString(r.ReleaseTags)
 }
 
-var ErrUnrecoverableError = errors.New("unrecoverable error")
-
 func (r *Release) ParseReleaseTagsString(tags string) {
 	cleanTags := CleanReleaseTags(tags)
 	t := ParseReleaseTagString(cleanTags)
@@ -432,10 +430,6 @@ func (r *Release) DownloadTorrentFileCtx(ctx context.Context) error {
 	return r.downloadTorrentFile(ctx)
 }
 
-func (r *Release) DownloadTorrentFile() error {
-	return r.downloadTorrentFile(context.Background())
-}
-
 func (r *Release) downloadTorrentFile(ctx context.Context) error {
 	if r.HasMagnetUri() {
 		return errors.New("downloading magnet links is not supported: %s", r.MagnetURI)
@@ -592,7 +586,7 @@ func (r *Release) downloadTorrentFile(ctx context.Context) error {
 }
 
 func (r *Release) CleanupTemporaryFiles() {
-	if len(r.TorrentTmpFile) == 0 {
+	if r.TorrentTmpFile == "" {
 		return
 	}
 
@@ -600,54 +594,15 @@ func (r *Release) CleanupTemporaryFiles() {
 	r.TorrentTmpFile = ""
 }
 
-// HasMagnetUri check uf MagnetURI is set or empty
+// HasMagnetUri check uf MagnetURI is set and valid or empty
 func (r *Release) HasMagnetUri() bool {
-	return r.MagnetURI != ""
+	if r.MagnetURI != "" && strings.HasPrefix(r.MagnetURI, MagnetURIPrefix) {
+		return true
+	}
+	return false
 }
 
-func (r *Release) ResolveMagnetUri(ctx context.Context) error {
-	if r.MagnetURI == "" {
-		return nil
-	} else if strings.HasPrefix(r.MagnetURI, "magnet:?") {
-		return nil
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.MagnetURI, nil)
-	if err != nil {
-		return errors.Wrap(err, "could not build request to resolve magnet uri")
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("User-Agent", "autobrr")
-
-	client := &http.Client{
-		Timeout:   time.Second * 45,
-		Transport: sharedhttp.MagnetTransport,
-	}
-
-	res, err := client.Do(req)
-	if err != nil {
-		return errors.Wrap(err, "could not make request to resolve magnet uri")
-	}
-
-	defer res.Body.Close()
-
-	if res.StatusCode != http.StatusOK {
-		return errors.New("unexpected status code: %d", res.StatusCode)
-	}
-
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		return errors.Wrap(err, "could not read response body")
-	}
-
-	magnet := string(body)
-	if magnet != "" {
-		r.MagnetURI = magnet
-	}
-
-	return nil
-}
+const MagnetURIPrefix = "magnet:?"
 
 func (r *Release) addRejection(reason string) {
 	r.Rejections = append(r.Rejections, reason)

internal/domain/release_download_test.go

@@ -6,6 +6,7 @@
 package domain
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -290,7 +291,7 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
 				Filter:                      tt.fields.Filter,
 				ActionStatus:                tt.fields.ActionStatus,
 			}
-			err := r.DownloadTorrentFile()
+			err := r.DownloadTorrentFileCtx(context.Background())
 			if err == nil && tt.wantErr {
 				fmt.Println("error")
 			}