refactor(http): implement shared transport and clients (#1288)

* fix(http): flip to a shared transport and clients

* nice threads

* that is terrible

* fake uri for magnet

* lazy locking

* why bother with r's

* flip magic params to struct

* refactor(http-clients): use separate clients with shared transport

* refactor(http-clients): add missing license header

* refactor(http-clients): defer and fix errors

---------

Co-authored-by: ze0s <ze0s@riseup.net>

internal/action/run.go

@@ -6,7 +6,6 @@ package action
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
@@ -197,14 +196,6 @@ func (s *service) webhook(ctx context.Context, action *domain.Action, release do
 		s.log.Trace().Msgf("webhook action '%s' - host: %s data: %s", action.Name, action.WebhookHost, action.WebhookData)
 	}
 
-	t := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-	}
-
-	client := http.Client{Transport: t, Timeout: 120 * time.Second}
-
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, action.WebhookHost, bytes.NewBufferString(action.WebhookData))
 	if err != nil {
 		return errors.Wrap(err, "could not build request for webhook")
@@ -214,8 +205,7 @@ func (s *service) webhook(ctx context.Context, action *domain.Action, release do
 	req.Header.Set("User-Agent", "autobrr")
 
 	start := time.Now()
-
+	res, err := s.httpClient.Do(req)
-	res, err := client.Do(req)
 	if err != nil {
 		return errors.Wrap(err, "could not make request for webhook")
 	}

internal/action/service.go

@@ -6,10 +6,13 @@ package action
 import (
 	"context"
 	"log"
+	"net/http"
+	"time"
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/internal/download_client"
 	"github.com/autobrr/autobrr/internal/logger"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/asaskevich/EventBus"
 	"github.com/dcarbone/zadapters/zstdlog"
@@ -34,6 +37,8 @@ type service struct {
 	repo      domain.ActionRepo
 	clientSvc download_client.Service
 	bus       EventBus.Bus
+
+	httpClient *http.Client
 }
 
 func NewService(log logger.Logger, repo domain.ActionRepo, clientSvc download_client.Service, bus EventBus.Bus) Service {
@@ -42,6 +47,11 @@ func NewService(log logger.Logger, repo domain.ActionRepo, clientSvc download_cl
 		repo:      repo,
 		clientSvc: clientSvc,
 		bus:       bus,
+
+		httpClient: &http.Client{
+			Timeout:   time.Second * 120,
+			Transport: sharedhttp.TransportTLSInsecure,
+		},
 	}
 
 	s.subLogger = zstdlog.NewStdLoggerWithLevel(s.log.With().Logger(), zerolog.TraceLevel)

internal/client/http.go

@@ -1,114 +0,0 @@
-// Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-package client
-
-import (
-	"io"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/autobrr/autobrr/pkg/errors"
-
-	"github.com/anacrolix/torrent/metainfo"
-	"github.com/avast/retry-go"
-	"github.com/rs/zerolog/log"
-)
-
-type DownloadFileResponse struct {
-	Body     *io.ReadCloser
-	FileName string
-}
-
-type DownloadTorrentFileResponse struct {
-	MetaInfo    *metainfo.MetaInfo
-	TmpFileName string
-}
-
-type HttpClient struct {
-	http *http.Client
-}
-
-func NewHttpClient() *HttpClient {
-	httpClient := &http.Client{
-		Timeout: time.Second * 10,
-	}
-	return &HttpClient{
-		http: httpClient,
-	}
-}
-
-func (c *HttpClient) DownloadTorrentFile(url string, opts map[string]string) (*DownloadTorrentFileResponse, error) {
-	if url == "" {
-		return nil, errors.New("download_file: url can't be empty")
-	}
-
-	// Create tmp file
-	tmpFile, err := os.CreateTemp("", "autobrr-")
-	if err != nil {
-		log.Error().Stack().Err(err).Msg("error creating temp file")
-		return nil, err
-	}
-	defer tmpFile.Close()
-
-	res := &DownloadTorrentFileResponse{}
-	// try request and if fail run 3 retries
-	err = retry.Do(func() error {
-		resp, err := http.Get(url)
-		if err != nil {
-			return errors.New("error downloading file: %q", err)
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode != http.StatusOK {
-			return errors.New("error downloading file bad status: %d", resp.StatusCode)
-		}
-
-		nuke := func() {
-			tmpFile.Seek(0, io.SeekStart)
-			tmpFile.Truncate(0)
-		}
-
-		// Write the body to file
-		_, err = io.Copy(tmpFile, resp.Body)
-		if err != nil {
-			nuke()
-			return errors.New("error writing downloaded file: %v | %q", tmpFile.Name(), err)
-		}
-
-		meta, err := metainfo.Load(resp.Body)
-		if err != nil {
-			nuke()
-			return errors.New("metainfo could not load file contents: %v | %q", tmpFile.Name(), err)
-		}
-
-		res = &DownloadTorrentFileResponse{
-			MetaInfo:    meta,
-			TmpFileName: tmpFile.Name(),
-		}
-
-		if res.TmpFileName == "" || res.MetaInfo == nil {
-			nuke()
-			return errors.New("tmp file error - empty body")
-		}
-
-		if len(res.MetaInfo.InfoBytes) < 1 {
-			nuke()
-			return errors.New("could not read infohash")
-		}
-
-		log.Debug().Msgf("successfully downloaded file: %v", tmpFile.Name())
-		return nil
-	},
-		//retry.OnRetry(func(n uint, err error) { c.log.Printf("%q: attempt %d - %v\n", err, n, url) }),
-		retry.Delay(time.Second*5),
-		retry.Attempts(3),
-		retry.MaxJitter(time.Second*1))
-
-	if err != nil {
-		res = nil
-	}
-
-	return res, err
-}

internal/domain/release.go

@@ -6,7 +6,6 @@ package domain
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"html"
 	"io"
@@ -19,6 +18,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/anacrolix/torrent/bencode"
 	"github.com/anacrolix/torrent/metainfo"
@@ -397,13 +397,6 @@ func (r *Release) downloadTorrentFile(ctx context.Context) error {
 		return nil
 	}
 
-	customTransport := http.DefaultTransport.(*http.Transport).Clone()
-	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	client := &http.Client{
-		Transport: customTransport,
-		Timeout:   time.Second * 45,
-	}
-
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.DownloadURL, nil)
 	if err != nil {
 		return errors.Wrap(err, "error downloading file")
@@ -411,6 +404,11 @@ func (r *Release) downloadTorrentFile(ctx context.Context) error {
 
 	req.Header.Set("User-Agent", "autobrr")
 
+	client := http.Client{
+		Timeout:   time.Second * 60,
+		Transport: sharedhttp.TransportTLSInsecure,
+	}
+
 	if r.RawCookie != "" {
 		jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
 		if err != nil {
@@ -573,11 +571,6 @@ func (r *Release) ResolveMagnetUri(ctx context.Context) error {
 		return nil
 	}
 
-	client := http.Client{
-		Transport: &magnetRoundTripper{},
-		Timeout:   time.Second * 60,
-	}
-
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.MagnetURI, nil)
 	if err != nil {
 		return errors.Wrap(err, "could not build request to resolve magnet uri")
@@ -586,6 +579,11 @@ func (r *Release) ResolveMagnetUri(ctx context.Context) error {
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("User-Agent", "autobrr")
 
+	client := &http.Client{
+		Timeout:   time.Second * 45,
+		Transport: sharedhttp.MagnetTransport,
+	}
+
 	res, err := client.Do(req)
 	if err != nil {
 		return errors.Wrap(err, "could not make request to resolve magnet uri")

internal/feed/client.go

@@ -5,11 +5,12 @@ package feed
 
 import (
 	"context"
-	"crypto/tls"
 	"net/http"
 	"net/http/cookiejar"
 	"time"
 
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
+
 	"github.com/mmcdole/gofeed"
 	"golang.org/x/net/publicsuffix"
 )
@@ -22,16 +23,16 @@ type RSSParser struct {
 
 // NewFeedParser wraps the gofeed.Parser using our own http client for full control
 func NewFeedParser(timeout time.Duration, cookie string) *RSSParser {
-	//store cookies in jar
-	jarOptions := &cookiejar.Options{PublicSuffixList: publicsuffix.List}
-	jar, _ := cookiejar.New(jarOptions)
-
-	customTransport := http.DefaultTransport.(*http.Transport).Clone()
-	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	httpClient := &http.Client{
 		Timeout:   time.Second * 60,
-		Transport: customTransport,
+		Transport: sharedhttp.TransportTLSInsecure,
-		Jar:       jar,
+	}
+
+	if cookie != "" {
+		//store cookies in jar
+		jarOptions := &cookiejar.Options{PublicSuffixList: publicsuffix.List}
+		jar, _ := cookiejar.New(jarOptions)
+		httpClient.Jar = jar
 	}
 
 	c := &RSSParser{

internal/filter/service.go

@@ -6,7 +6,6 @@ package filter
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
@@ -22,6 +21,7 @@ import (
 	"github.com/autobrr/autobrr/internal/logger"
 	"github.com/autobrr/autobrr/internal/utils"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/avast/retry-go/v4"
 	"github.com/mattn/go-shellwords"
@@ -52,6 +52,8 @@ type service struct {
 	releaseRepo domain.ReleaseRepo
 	indexerSvc  indexer.Service
 	apiService  indexer.APIService
+
+	httpClient *http.Client
 }
 
 func NewService(log logger.Logger, repo domain.FilterRepo, actionRepo domain.ActionRepo, releaseRepo domain.ReleaseRepo, apiService indexer.APIService, indexerSvc indexer.Service) Service {
@@ -62,6 +64,10 @@ func NewService(log logger.Logger, repo domain.FilterRepo, actionRepo domain.Act
 		releaseRepo: releaseRepo,
 		apiService:  apiService,
 		indexerSvc:  indexerSvc,
+		httpClient: &http.Client{
+			Timeout:   time.Second * 120,
+			Transport: sharedhttp.TransportTLSInsecure,
+		},
 	}
 }
 
@@ -659,14 +665,6 @@ func (s *service) webhook(ctx context.Context, external domain.FilterExternal, r
 
 	s.log.Trace().Msgf("sending %s to external webhook filter: (%s) payload: (%s)", external.WebhookMethod, external.WebhookHost, external.WebhookData)
 
-	t := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-	}
-
-	client := http.Client{Transport: t, Timeout: 120 * time.Second}
-
 	method := http.MethodPost
 	if external.WebhookMethod != "" {
 		method = external.WebhookMethod
@@ -720,7 +718,7 @@ func (s *service) webhook(ctx context.Context, external domain.FilterExternal, r
 			if external.WebhookData != "" && dataArgs != "" {
 				clonereq.Body = io.NopCloser(bytes.NewBufferString(dataArgs))
 			}
-			res, err := client.Do(clonereq)
+			res, err := s.httpClient.Do(clonereq)
 			if err != nil {
 				return 0, errors.Wrap(err, "could not make request for webhook")
 			}

internal/http/auth_test.go

@@ -91,6 +91,8 @@ func setupAuthHandler() {
 }
 
 func TestAuthHandlerLogin(t *testing.T) {
+	t.Parallel()
+
 	logger := zerolog.Nop()
 	encoder := encoder{}
 	cookieStore := sessions.NewCookieStore([]byte("test"))
@@ -141,6 +143,8 @@ func TestAuthHandlerLogin(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	// check for response, here we'll just check for 204 NoContent
 	if status := resp.StatusCode; status != http.StatusNoContent {
 		t.Errorf("login: handler returned wrong status code: got %v want %v", status, http.StatusNoContent)
@@ -152,6 +156,8 @@ func TestAuthHandlerLogin(t *testing.T) {
 }
 
 func TestAuthHandlerValidateOK(t *testing.T) {
+	t.Parallel()
+
 	logger := zerolog.Nop()
 	encoder := encoder{}
 	cookieStore := sessions.NewCookieStore([]byte("test"))
@@ -202,6 +208,8 @@ func TestAuthHandlerValidateOK(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	// check for response, here we'll just check for 204 NoContent
 	if status := resp.StatusCode; status != http.StatusNoContent {
 		t.Errorf("login: handler returned wrong status code: got %v want %v", status, http.StatusNoContent)
@@ -217,12 +225,16 @@ func TestAuthHandlerValidateOK(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	if status := resp.StatusCode; status != http.StatusNoContent {
 		t.Errorf("validate: handler returned wrong status code: got %v want %v", status, http.StatusNoContent)
 	}
 }
 
 func TestAuthHandlerValidateBad(t *testing.T) {
+	t.Parallel()
+
 	logger := zerolog.Nop()
 	encoder := encoder{}
 	cookieStore := sessions.NewCookieStore([]byte("test"))
@@ -264,12 +276,16 @@ func TestAuthHandlerValidateBad(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	if status := resp.StatusCode; status != http.StatusUnauthorized {
 		t.Errorf("validate: handler returned wrong status code: got %v want %v", status, http.StatusUnauthorized)
 	}
 }
 
 func TestAuthHandlerLoginBad(t *testing.T) {
+	t.Parallel()
+
 	logger := zerolog.Nop()
 	encoder := encoder{}
 	cookieStore := sessions.NewCookieStore([]byte("test"))
@@ -310,6 +326,8 @@ func TestAuthHandlerLoginBad(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	// check for response, here we'll just check for 204 NoContent
 	if status := resp.StatusCode; status != http.StatusUnauthorized {
 		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusUnauthorized)
@@ -317,6 +335,8 @@ func TestAuthHandlerLoginBad(t *testing.T) {
 }
 
 func TestAuthHandlerLogout(t *testing.T) {
+	t.Parallel()
+
 	logger := zerolog.Nop()
 	encoder := encoder{}
 	cookieStore := sessions.NewCookieStore([]byte("test"))
@@ -367,6 +387,8 @@ func TestAuthHandlerLogout(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	// check for response, here we'll just check for 204 NoContent
 	if status := resp.StatusCode; status != http.StatusNoContent {
 		t.Errorf("login: handler returned wrong status code: got %v want %v", status, http.StatusNoContent)
@@ -382,6 +404,8 @@ func TestAuthHandlerLogout(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	if status := resp.StatusCode; status != http.StatusNoContent {
 		t.Errorf("validate: handler returned wrong status code: got %v want %v", status, http.StatusNoContent)
 	}
@@ -392,6 +416,8 @@ func TestAuthHandlerLogout(t *testing.T) {
 		log.Fatalf("Error occurred: %v", err)
 	}
 
+	defer resp.Body.Close()
+
 	if status := resp.StatusCode; status != http.StatusNoContent {
 		t.Errorf("validate: handler returned wrong status code: got %v want %v", status, http.StatusNoContent)
 	}

internal/indexer/api.go

@@ -92,7 +92,7 @@ func (s *apiService) AddClient(indexer string, settings map[string]string) error
 		if !ok || key == "" {
 			return errors.New("api.Service.AddClient: could not initialize btn client: missing var 'api_key'")
 		}
-		s.apiClients[indexer] = btn.NewClient("", key)
+		s.apiClients[indexer] = btn.NewClient(key)
 
 	case "ptp":
 		user, ok := settings["api_user"]
@@ -128,7 +128,7 @@ func (s *apiService) AddClient(indexer string, settings map[string]string) error
 		s.apiClients[indexer] = ops.NewClient(key)
 
 	case "mock":
-		s.apiClients[indexer] = mock.NewMockClient("", "mock")
+		s.apiClients[indexer] = mock.NewMockClient("mock")
 
 	default:
 		return errors.New("api.Service.AddClient: could not initialize client: unsupported indexer: %s", indexer)
@@ -154,7 +154,7 @@ func (s *apiService) getClientForTest(req domain.IndexerTestApiRequest) (apiClie
 		if req.ApiKey == "" {
 			return nil, errors.New("api.Service.AddClient: could not initialize btn client: missing var 'api_key'")
 		}
-		return btn.NewClient("", req.ApiKey), nil
+		return btn.NewClient(req.ApiKey), nil
 
 	case "ptp":
 		if req.ApiUser == "" {
@@ -185,7 +185,7 @@ func (s *apiService) getClientForTest(req domain.IndexerTestApiRequest) (apiClie
 		return ops.NewClient(req.ApiKey), nil
 
 	case "mock":
-		return mock.NewMockClient("", "mock"), nil
+		return mock.NewMockClient("mock"), nil
 
 	default:
 		return nil, errors.New("api.Service.AddClient: could not initialize client: unsupported indexer: %s", req.Identifier)

internal/mock/indexer_api.go

@@ -16,16 +16,28 @@ type IndexerApiClient interface {
 }
 
 type IndexerClient struct {
-	URL    string
+	url    string
 	APIKey string
 }
 
-func NewMockClient(url string, apiKey string) IndexerApiClient {
+type OptFunc func(client *IndexerClient)
+
+func WithUrl(url string) OptFunc {
+	return func(c *IndexerClient) {
+		c.url = url
+	}
+}
+
+func NewMockClient(apiKey string, opts ...OptFunc) IndexerApiClient {
 	c := &IndexerClient{
-		URL:    url,
+		url:    "",
 		APIKey: apiKey,
 	}
 
+	for _, opt := range opts {
+		opt(c)
+	}
+
 	return c
 }
 

internal/notification/discord.go

@@ -5,7 +5,6 @@ package notification
 
 import (
 	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -15,6 +14,7 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/dustin/go-humanize"
 	"github.com/rs/zerolog"
@@ -50,12 +50,18 @@ const (
 type discordSender struct {
 	log      zerolog.Logger
 	Settings domain.Notification
+
+	httpClient *http.Client
 }
 
 func NewDiscordSender(log zerolog.Logger, settings domain.Notification) domain.NotificationSender {
 	return &discordSender{
 		log:      log.With().Str("sender", "discord").Logger(),
 		Settings: settings,
+		httpClient: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
+		},
 	}
 }
 
@@ -80,27 +86,20 @@ func (a *discordSender) Send(event domain.NotificationEvent, payload domain.Noti
 	req.Header.Set("Content-Type", "application/json")
 	//req.Header.Set("User-Agent", "autobrr")
 
-	t := &http.Transport{
+	res, err := a.httpClient.Do(req)
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-	}
-
-	client := http.Client{Transport: t, Timeout: 30 * time.Second}
-	res, err := client.Do(req)
 	if err != nil {
 		a.log.Error().Err(err).Msgf("discord client request error: %v", event)
 		return errors.Wrap(err, "could not make request: %+v", req)
 	}
 
+	defer res.Body.Close()
+
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		a.log.Error().Err(err).Msgf("discord client request error: %v", event)
 		return errors.Wrap(err, "could not read data")
 	}
 
-	defer res.Body.Close()
-
 	a.log.Trace().Msgf("discord status: %v response: %v", res.StatusCode, string(body))
 
 	// discord responds with 204, Notifiarr with 204 so lets take all 200 as ok

internal/notification/gotify.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/rs/zerolog"
 )
@@ -26,6 +27,8 @@ type gotifySender struct {
 	log      zerolog.Logger
 	Settings domain.Notification
 	builder  NotificationBuilderPlainText
+
+	httpClient *http.Client
 }
 
 func NewGotifySender(log zerolog.Logger, settings domain.Notification) domain.NotificationSender {
@@ -33,6 +36,10 @@ func NewGotifySender(log zerolog.Logger, settings domain.Notification) domain.No
 		log:      log.With().Str("sender", "gotify").Logger(),
 		Settings: settings,
 		builder:  NotificationBuilderPlainText{},
+		httpClient: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
+		},
 	}
 }
 
@@ -56,21 +63,20 @@ func (s *gotifySender) Send(event domain.NotificationEvent, payload domain.Notif
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 	req.Header.Set("User-Agent", "autobrr")
 
-	client := http.Client{Timeout: 30 * time.Second}
+	res, err := s.httpClient.Do(req)
-	res, err := client.Do(req)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("gotify client request error: %v", event)
 		return errors.Wrap(err, "could not make request: %+v", req)
 	}
 
+	defer res.Body.Close()
+
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("gotify client request error: %v", event)
 		return errors.Wrap(err, "could not read data")
 	}
 
-	defer res.Body.Close()
-
 	s.log.Trace().Msgf("gotify status: %v response: %v", res.StatusCode, string(body))
 
 	if res.StatusCode != http.StatusOK {

internal/notification/lunasea.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/rs/zerolog"
 )
@@ -26,6 +27,8 @@ type lunaSeaSender struct {
 	log      zerolog.Logger
 	Settings domain.Notification
 	builder  NotificationBuilderPlainText
+
+	httpClient *http.Client
 }
 
 func (s *lunaSeaSender) rewriteWebhookURL(url string) string {
@@ -38,6 +41,10 @@ func NewLunaSeaSender(log zerolog.Logger, settings domain.Notification) domain.N
 		log:      log.With().Str("sender", "lunasea").Logger(),
 		Settings: settings,
 		builder:  NotificationBuilderPlainText{},
+		httpClient: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
+		},
 	}
 }
 
@@ -64,8 +71,7 @@ func (s *lunaSeaSender) Send(event domain.NotificationEvent, payload domain.Noti
 
 	req.Header.Set("Content-Type", "application/json")
 
-	client := &http.Client{Timeout: 30 * time.Second}
+	res, err := s.httpClient.Do(req)
-	res, err := client.Do(req)
 	if err != nil {
 		s.log.Error().Err(err).Msg("lunasea client request error")
 		return errors.Wrap(err, "could not make request")

internal/notification/notifiarr.go

@@ -5,7 +5,6 @@ package notification
 
 import (
 	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -13,6 +12,7 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/rs/zerolog"
 )
@@ -45,6 +45,8 @@ type notifiarrSender struct {
 	log      zerolog.Logger
 	Settings domain.Notification
 	baseUrl  string
+
+	httpClient *http.Client
 }
 
 func NewNotifiarrSender(log zerolog.Logger, settings domain.Notification) domain.NotificationSender {
@@ -52,6 +54,10 @@ func NewNotifiarrSender(log zerolog.Logger, settings domain.Notification) domain
 		log:      log.With().Str("sender", "notifiarr").Logger(),
 		Settings: settings,
 		baseUrl:  "https://notifiarr.com/api/v1/notification/autobrr",
+		httpClient: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
+		},
 	}
 }
 
@@ -77,27 +83,20 @@ func (s *notifiarrSender) Send(event domain.NotificationEvent, payload domain.No
 	req.Header.Set("User-Agent", "autobrr")
 	req.Header.Set("X-API-Key", s.Settings.APIKey)
 
-	t := &http.Transport{
+	res, err := s.httpClient.Do(req)
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-	}
-
-	client := http.Client{Transport: t, Timeout: 30 * time.Second}
-	res, err := client.Do(req)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("notifiarr client request error: %v", event)
 		return errors.Wrap(err, "could not make request: %+v", req)
 	}
 
+	defer res.Body.Close()
+
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("notifiarr client request error: %v", event)
 		return errors.Wrap(err, "could not read data")
 	}
 
-	defer res.Body.Close()
-
 	s.log.Trace().Msgf("notifiarr status: %v response: %v", res.StatusCode, string(body))
 
 	if res.StatusCode != http.StatusOK {

internal/notification/pushover.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/rs/zerolog"
 )
@@ -33,6 +34,8 @@ type pushoverSender struct {
 	Settings domain.Notification
 	baseUrl  string
 	builder  NotificationBuilderPlainText
+
+	httpClient *http.Client
 }
 
 func NewPushoverSender(log zerolog.Logger, settings domain.Notification) domain.NotificationSender {
@@ -40,6 +43,11 @@ func NewPushoverSender(log zerolog.Logger, settings domain.Notification) domain.
 		log:      log.With().Str("sender", "pushover").Logger(),
 		Settings: settings,
 		baseUrl:  "https://api.pushover.net/1/messages.json",
+		builder:  NotificationBuilderPlainText{},
+		httpClient: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
+		},
 	}
 }
 
@@ -81,21 +89,20 @@ func (s *pushoverSender) Send(event domain.NotificationEvent, payload domain.Not
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 	req.Header.Set("User-Agent", "autobrr")
 
-	client := http.Client{Timeout: 30 * time.Second}
+	res, err := s.httpClient.Do(req)
-	res, err := client.Do(req)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("pushover client request error: %v", event)
 		return errors.Wrap(err, "could not make request: %+v", req)
 	}
 
+	defer res.Body.Close()
+
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("pushover client request error: %v", event)
 		return errors.Wrap(err, "could not read data")
 	}
 
-	defer res.Body.Close()
-
 	s.log.Trace().Msgf("pushover status: %v response: %v", res.StatusCode, string(body))
 
 	if res.StatusCode != http.StatusOK {

internal/notification/telegram.go

@@ -14,11 +14,12 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"github.com/rs/zerolog"
 )
 
-// Reference: https://core.telegram.org/bots/api#sendmessage
+// TelegramMessage Reference: https://core.telegram.org/bots/api#sendmessage
 type TelegramMessage struct {
 	ChatID          string `json:"chat_id"`
 	Text            string `json:"text"`
@@ -31,6 +32,8 @@ type telegramSender struct {
 	Settings domain.Notification
 	ThreadID int
 	builder  NotificationBuilderPlainText
+
+	httpClient *http.Client
 }
 
 func NewTelegramSender(log zerolog.Logger, settings domain.Notification) domain.NotificationSender {
@@ -46,6 +49,11 @@ func NewTelegramSender(log zerolog.Logger, settings domain.Notification) domain.
 		log:      log.With().Str("sender", "telegram").Logger(),
 		Settings: settings,
 		ThreadID: threadID,
+		builder:  NotificationBuilderPlainText{},
+		httpClient: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
+		},
 	}
 }
 
@@ -76,21 +84,20 @@ func (s *telegramSender) Send(event domain.NotificationEvent, payload domain.Not
 	req.Header.Set("Content-Type", "application/json")
 	//req.Header.Set("User-Agent", "autobrr")
 
-	client := http.Client{Timeout: 30 * time.Second}
+	res, err := s.httpClient.Do(req)
-	res, err := client.Do(req)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("telegram client request error: %v", event)
 		return errors.Wrap(err, "could not make request: %+v", req)
 	}
 
+	defer res.Body.Close()
+
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		s.log.Error().Err(err).Msgf("telegram client request error: %v", event)
 		return errors.Wrap(err, "could not read data")
 	}
 
-	defer res.Body.Close()
-
 	s.log.Trace().Msgf("telegram status: %v response: %v", res.StatusCode, string(body))
 
 	if res.StatusCode != http.StatusOK {

pkg/btn/btn_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package btn
 
 import (
@@ -14,9 +16,8 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestAPI(t *testing.T) {
@@ -67,7 +68,7 @@ func TestAPI(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c := NewClient(tt.fields.Url, tt.fields.APIKey)
+			c := NewClient(tt.fields.APIKey, WithUrl(ts.URL))
 
 			got, err := c.TestAPI(context.Background())
 			if tt.wantErr && assert.Error(t, err) {
@@ -166,8 +167,7 @@ func TestClient_GetTorrentByID(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-
+			c := NewClient(tt.fields.APIKey, WithUrl(ts.URL))
-			c := NewClient(tt.fields.Url, tt.fields.APIKey)
 
 			got, err := c.GetTorrentByID(context.Background(), tt.args.torrentID)
 			if tt.wantErr && assert.Error(t, err) {

pkg/btn/client.go

@@ -7,42 +7,61 @@ import (
 	"context"
 	"io"
 	"log"
+	"net/http"
 	"time"
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/jsonrpc"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"golang.org/x/time/rate"
 )
 
+const DefaultURL = "https://api.broadcasthe.net/"
+
 type ApiClient interface {
 	GetTorrentByID(ctx context.Context, torrentID string) (*domain.TorrentBasic, error)
 	TestAPI(ctx context.Context) (bool, error)
 }
 
+type OptFunc func(*Client)
+
+func WithUrl(url string) OptFunc {
+	return func(c *Client) {
+		c.url = url
+	}
+}
+
 type Client struct {
 	rpcClient   jsonrpc.Client
-	Ratelimiter *rate.Limiter
+	rateLimiter *rate.Limiter
 	APIKey      string
+	url         string
 
 	Log *log.Logger
 }
 
-func NewClient(url string, apiKey string) ApiClient {
+func NewClient(apiKey string, opts ...OptFunc) ApiClient {
-	if url == "" {
-		url = "https://api.broadcasthe.net/"
-	}
-
 	c := &Client{
-		rpcClient: jsonrpc.NewClientWithOpts(url, &jsonrpc.ClientOpts{
+		url:         DefaultURL,
-			Headers: map[string]string{
+		rateLimiter: rate.NewLimiter(rate.Every(150*time.Hour), 1), // 150 rpcRequest every 1 hour
-				"User-Agent": "autobrr",
-			},
-		}),
-		Ratelimiter: rate.NewLimiter(rate.Every(150*time.Hour), 1), // 150 rpcRequest every 1 hour
 		APIKey:      apiKey,
 	}
 
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	c.rpcClient = jsonrpc.NewClientWithOpts(c.url, &jsonrpc.ClientOpts{
+		Headers: map[string]string{
+			"User-Agent": "autobrr",
+		},
+		HTTPClient: &http.Client{
+			Timeout:   time.Second * 60,
+			Transport: sharedhttp.Transport,
+		},
+	})
+
 	if c.Log == nil {
 		c.Log = log.New(io.Discard, "", log.LstdFlags)
 	}

pkg/ggn/ggn.go

@@ -15,38 +15,53 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"golang.org/x/time/rate"
 )
 
+const DefaultURL = "https://gazellegames.net/api.php"
+
+var ErrUnauthorized = errors.New("unauthorized: bad credentials")
+var ErrForbidden = errors.New("forbidden")
+var ErrTooManyRequests = errors.New("too many requests: rate-limit reached")
+
 type ApiClient interface {
 	GetTorrentByID(ctx context.Context, torrentID string) (*domain.TorrentBasic, error)
 	TestAPI(ctx context.Context) (bool, error)
-	UseURL(url string)
 }
 
 type Client struct {
-	Url         string
+	url         string
 	client      *http.Client
-	Ratelimiter *rate.Limiter
+	rateLimiter *rate.Limiter
 	APIKey      string
 }
 
-func NewClient(apiKey string) ApiClient {
+type OptFunc func(*Client)
+
+func WithUrl(url string) OptFunc {
+	return func(c *Client) {
+		c.url = url
+	}
+}
+
+func NewClient(apiKey string, opts ...OptFunc) ApiClient {
 	c := &Client{
-		Url: "https://gazellegames.net/api.php",
+		url: DefaultURL,
 		client: &http.Client{
-			Timeout: time.Second * 30,
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
 		},
-		Ratelimiter: rate.NewLimiter(rate.Every(5*time.Second), 1), // 5 request every 10 seconds
+		rateLimiter: rate.NewLimiter(rate.Every(5*time.Second), 1), // 5 request every 10 seconds
 		APIKey:      apiKey,
 	}
 
-	return c
+	for _, opt := range opts {
-}
+		opt(c)
+	}
 
-func (c *Client) UseURL(url string) {
+	return c
-	c.Url = url
 }
 
 type Group struct {
@@ -150,13 +165,13 @@ type Response struct {
 
 func (c *Client) Do(req *http.Request) (*http.Response, error) {
 	ctx := context.Background()
-	err := c.Ratelimiter.Wait(ctx) // This is a blocking call. Honors the rate limit
+	err := c.rateLimiter.Wait(ctx) // This is a blocking call. Honors the rate limit
 	if err != nil {
 		return nil, errors.Wrap(err, "error waiting for ratelimiter")
 	}
 	resp, err := c.client.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "error making request")
+		return resp, errors.Wrap(err, "error making request")
 	}
 	return resp, nil
 }
@@ -172,15 +187,15 @@ func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
 
 	res, err := c.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "ggn client request error : %s", url)
+		return res, errors.Wrap(err, "ggn client request error : %s", url)
 	}
 
 	if res.StatusCode == http.StatusUnauthorized {
-		return nil, errors.New("unauthorized: bad credentials")
+		return res, ErrUnauthorized
 	} else if res.StatusCode == http.StatusForbidden {
-		return nil, nil
+		return res, ErrForbidden
 	} else if res.StatusCode == http.StatusTooManyRequests {
-		return nil, nil
+		return res, ErrTooManyRequests
 	}
 
 	return res, nil
@@ -197,7 +212,7 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 	v.Add("id", torrentID)
 	params := v.Encode()
 
-	reqUrl := fmt.Sprintf("%s?%s&%s", c.Url, "request=torrent", params)
+	reqUrl := fmt.Sprintf("%s?%s&%s", c.url, "request=torrent", params)
 
 	resp, err := c.get(ctx, reqUrl)
 	if err != nil {
@@ -231,7 +246,7 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 
 // TestAPI try api access against torrents page
 func (c *Client) TestAPI(ctx context.Context) (bool, error) {
-	resp, err := c.get(ctx, c.Url)
+	resp, err := c.get(ctx, c.url)
 	if err != nil {
 		return false, errors.Wrap(err, "error getting data")
 	}

pkg/ggn/ggn_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package ggn
 
 import (
@@ -8,13 +10,12 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"strings"
 	"testing"
 
+	"github.com/autobrr/autobrr/internal/domain"
+
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
-
-	"github.com/autobrr/autobrr/internal/domain"
 )
 
 func Test_client_GetTorrentByID(t *testing.T) {
@@ -32,18 +33,26 @@ func Test_client_GetTorrentByID(t *testing.T) {
 			return
 		}
 
-		if !strings.Contains(r.RequestURI, "422368") {
+		id := r.URL.Query().Get("id")
-			jsonPayload, _ := os.ReadFile("testdata/ggn_get_torrent_by_id_not_found.json")
+		var jsonPayload []byte
+		switch id {
+		case "422368":
+			jsonPayload, _ = os.ReadFile("testdata/ggn_get_torrent_by_id.json")
 			w.Header().Set("Content-Type", "application/json")
 			w.WriteHeader(http.StatusOK)
-			w.Write(jsonPayload)
+			break
-			return
+
+		case "100002":
+			jsonPayload, _ = os.ReadFile("testdata/ggn_get_torrent_by_id_not_found.json")
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			break
 		}
 
 		// read json response
-		jsonPayload, _ := os.ReadFile("testdata/ggn_get_torrent_by_id.json")
+		//jsonPayload, _ := os.ReadFile("testdata/ggn_get_torrent_by_id.json")
-		w.Header().Set("Content-Type", "application/json")
+		//w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusOK)
+		//w.WriteHeader(http.StatusOK)
 		w.Write(jsonPayload)
 	}))
 	defer ts.Close()
@@ -77,7 +86,7 @@ func Test_client_GetTorrentByID(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name: "get_by_id_2",
+			name: "get_by_invalid_id",
 			fields: fields{
 				Url:    ts.URL,
 				APIKey: key,
@@ -89,9 +98,7 @@ func Test_client_GetTorrentByID(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-
+			c := NewClient(tt.fields.APIKey, WithUrl(ts.URL))
-			c := NewClient(tt.fields.APIKey)
-			c.UseURL(tt.fields.Url)
 
 			got, err := c.GetTorrentByID(context.Background(), tt.args.torrentID)
 			if tt.wantErr && assert.Error(t, err) {

pkg/jsonrpc/jsonrpc.go

@@ -166,7 +166,6 @@ func (c *rpcClient) newRequest(ctx context.Context, req interface{}) (*http.Requ
 }
 
 func (c *rpcClient) doCall(ctx context.Context, request RPCRequest) (*RPCResponse, error) {
-
 	httpRequest, err := c.newRequest(ctx, request)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create rpc http request")

pkg/lidarr/client.go

@@ -71,14 +71,14 @@ func (c *client) post(ctx context.Context, endpoint string, data interface{}) (*
 
 	res, err := c.http.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "lidarr client request error: %v", reqUrl)
+		return res, errors.Wrap(err, "lidarr client request error: %v", reqUrl)
 	}
 
 	// validate response
 	if res.StatusCode == http.StatusUnauthorized {
-		return nil, errors.New("lidarr: unauthorized: bad credentials")
+		return res, errors.New("lidarr: unauthorized: bad credentials")
 	} else if res.StatusCode != http.StatusOK {
-		return nil, errors.New("lidarr: bad request")
+		return res, errors.New("lidarr: bad request")
 	}
 
 	// return raw response and let the caller handle json unmarshal of body

pkg/lidarr/lidarr.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 type Config struct {
@@ -42,19 +43,19 @@ type client struct {
 
 // New create new lidarr client
 func New(config Config) Client {
-
 	httpClient := &http.Client{
-		Timeout: time.Second * 120,
+		Timeout:   time.Second * 120,
+		Transport: sharedhttp.Transport,
 	}
 
 	c := &client{
 		config: config,
 		http:   httpClient,
-		Log:    config.Log,
+		Log:    log.New(io.Discard, "", log.LstdFlags),
 	}
 
-	if config.Log == nil {
+	if config.Log != nil {
-		c.Log = log.New(io.Discard, "", log.LstdFlags)
+		c.Log = config.Log
 	}
 
 	return c

pkg/lidarr/lidarr_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package lidarr
 
 import (

pkg/newznab/newznab.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 const DefaultTimeout = 60
@@ -63,7 +64,8 @@ type Capabilities struct {
 
 func NewClient(config Config) Client {
 	httpClient := &http.Client{
-		Timeout: time.Second * DefaultTimeout,
+		Timeout:   time.Second * DefaultTimeout,
+		Transport: sharedhttp.Transport,
 	}
 
 	if config.Timeout > 0 {
@@ -176,7 +178,7 @@ func (c *client) getData(ctx context.Context, endpoint string, queryParams map[s
 
 	resp, err := c.http.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not make request. %+v", req)
+		return resp, errors.Wrap(err, "could not make request. %+v", req)
 	}
 
 	return resp, nil

pkg/ops/ops.go

@@ -1,3 +1,6 @@
+// Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+
 package ops
 
 import (
@@ -12,38 +15,49 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"golang.org/x/time/rate"
 )
 
+const DefaultURL = "https://orpheus.network/ajax.php"
+
 type ApiClient interface {
 	GetTorrentByID(ctx context.Context, torrentID string) (*domain.TorrentBasic, error)
 	TestAPI(ctx context.Context) (bool, error)
-	UseURL(url string)
 }
 
 type Client struct {
-	Url         string
+	url         string
 	client      *http.Client
 	RateLimiter *rate.Limiter
 	APIKey      string
 }
 
-func NewClient(apiKey string) ApiClient {
+type OptFunc func(*Client)
+
+func WithUrl(url string) OptFunc {
+	return func(c *Client) {
+		c.url = url
+	}
+}
+
+func NewClient(apiKey string, opts ...OptFunc) ApiClient {
 	c := &Client{
-		Url: "https://orpheus.network/ajax.php",
+		url: DefaultURL,
 		client: &http.Client{
-			Timeout: time.Second * 30,
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
 		},
 		RateLimiter: rate.NewLimiter(rate.Every(10*time.Second), 5),
 		APIKey:      apiKey,
 	}
 
-	return c
+	for _, opt := range opts {
-}
+		opt(c)
+	}
 
-func (c *Client) UseURL(url string) {
+	return c
-	c.Url = url
 }
 
 type ErrorResponse struct {
@@ -127,7 +141,7 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 	}
 	resp, err := c.client.Do(req)
 	if err != nil {
-		return nil, err
+		return resp, err
 	}
 	return resp, nil
 }
@@ -147,13 +161,15 @@ func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
 
 	res, err := c.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not make request: %+v", req)
+		return res, errors.Wrap(err, "could not make request: %+v", req)
 	}
 
 	// return early if not OK
 	if res.StatusCode != http.StatusOK {
 		var r ErrorResponse
 
+		defer res.Body.Close()
+
 		body, readErr := io.ReadAll(res.Body)
 		if readErr != nil {
 			return nil, errors.Wrap(readErr, "could not read body")
@@ -163,8 +179,6 @@ func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
 			return nil, errors.Wrap(err, "could not unmarshal body")
 		}
 
-		res.Body.Close()
-
 		return nil, errors.New("status code: %d status: %s error: %s", res.StatusCode, r.Status, r.Error)
 	}
 
@@ -182,7 +196,7 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 	v.Add("id", torrentID)
 	params := v.Encode()
 
-	reqUrl := fmt.Sprintf("%s?action=torrent&%s", c.Url, params)
+	reqUrl := fmt.Sprintf("%s?action=torrent&%s", c.url, params)
 
 	resp, err := c.get(ctx, reqUrl)
 	if err != nil {
@@ -212,7 +226,7 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 
 // TestAPI try api access against torrents page
 func (c *Client) TestAPI(ctx context.Context) (bool, error) {
-	resp, err := c.get(ctx, c.Url+"?action=index")
+	resp, err := c.get(ctx, c.url+"?action=index")
 	if err != nil {
 		return false, errors.Wrap(err, "test api error")
 	}

pkg/ops/ops_test.go

@@ -1,3 +1,8 @@
+// Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+//go:build integration
+
 package ops
 
 import (
@@ -9,6 +14,7 @@ import (
 	"testing"
 
 	"github.com/autobrr/autobrr/internal/domain"
+
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
@@ -28,18 +34,18 @@ func TestOrpheusClient_GetTorrentByID(t *testing.T) {
 			return
 		}
 
-		if !strings.Contains(r.RequestURI, "2156788") {
+		if strings.Contains(r.RequestURI, "2156788") {
-			jsonPayload, _ := os.ReadFile("testdata/get_torrent_by_id_not_found.json")
+			jsonPayload, _ := os.ReadFile("testdata/get_torrent_by_id.json")
 			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusBadRequest)
+			w.WriteHeader(http.StatusOK)
 			w.Write(jsonPayload)
 			return
 		}
 
 		// read json response
-		jsonPayload, _ := os.ReadFile("testdata/get_torrent_by_id.json")
+		jsonPayload, _ := os.ReadFile("testdata/get_torrent_by_id_not_found.json")
 		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusOK)
+		w.WriteHeader(http.StatusBadRequest)
 		w.Write(jsonPayload)
 	}))
 	defer ts.Close()
@@ -74,7 +80,7 @@ func TestOrpheusClient_GetTorrentByID(t *testing.T) {
 			wantErr: "",
 		},
 		{
-			name: "get_by_id_2",
+			name: "invalid_torrent_id",
 			fields: fields{
 				Url:    ts.URL,
 				APIKey: key,
@@ -84,7 +90,7 @@ func TestOrpheusClient_GetTorrentByID(t *testing.T) {
 			wantErr: "could not get torrent by id: 100002: status code: 400 status: failure error: bad id parameter",
 		},
 		{
-			name: "get_by_id_3",
+			name: "missing_api_key",
 			fields: fields{
 				Url:    ts.URL,
 				APIKey: "",
@@ -96,8 +102,7 @@ func TestOrpheusClient_GetTorrentByID(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c := NewClient(tt.fields.APIKey)
+			c := NewClient(tt.fields.APIKey, WithUrl(ts.URL))
-			c.UseURL(tt.fields.Url)
 
 			got, err := c.GetTorrentByID(context.Background(), tt.args.torrentID)
 			if tt.wantErr != "" && assert.Error(t, err) {

pkg/porla/client.go

@@ -4,7 +4,6 @@
 package porla
 
 import (
-	"crypto/tls"
 	"io"
 	"log"
 	"net/http"
@@ -12,6 +11,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/jsonrpc"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 var (
@@ -62,20 +62,17 @@ func NewClient(cfg Config) *Client {
 		c.timeout = time.Duration(cfg.Timeout) * time.Second
 	}
 
-	c.http = &http.Client{
-		Timeout: c.timeout,
-	}
-
-	customTransport := http.DefaultTransport.(*http.Transport).Clone()
-	if cfg.TLSSkipVerify {
-		customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	}
-
 	httpClient := &http.Client{
 		Timeout:   c.timeout,
-		Transport: customTransport,
+		Transport: sharedhttp.Transport,
 	}
 
+	if cfg.TLSSkipVerify {
+		httpClient.Transport = sharedhttp.TransportTLSInsecure
+	}
+
+	c.http = httpClient
+
 	token := cfg.AuthToken
 
 	if !strings.HasPrefix(token, "Bearer ") {

pkg/ptp/ptp.go

@@ -14,40 +14,55 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"golang.org/x/time/rate"
 )
 
+const DefaultURL = "https://passthepopcorn.me/torrents.php"
+
+var ErrUnauthorized = errors.New("unauthorized: bad credentials")
+var ErrForbidden = errors.New("forbidden")
+var ErrTooManyRequests = errors.New("too many requests: rate-limit reached")
+
 type ApiClient interface {
 	GetTorrentByID(ctx context.Context, torrentID string) (*domain.TorrentBasic, error)
 	TestAPI(ctx context.Context) (bool, error)
-	UseURL(url string)
 }
 
 type Client struct {
-	Url         string
+	url         string
 	client      *http.Client
-	Ratelimiter *rate.Limiter
+	rateLimiter *rate.Limiter
 	APIUser     string
 	APIKey      string
 }
 
-func NewClient(apiUser, apiKey string) ApiClient {
+type OptFunc func(*Client)
+
+func WithUrl(url string) OptFunc {
+	return func(c *Client) {
+		c.url = url
+	}
+}
+
+func NewClient(apiUser, apiKey string, opts ...OptFunc) ApiClient {
 	c := &Client{
-		Url: "https://passthepopcorn.me/torrents.php",
+		url: DefaultURL,
 		client: &http.Client{
-			Timeout: time.Second * 30,
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
 		},
-		Ratelimiter: rate.NewLimiter(rate.Every(1*time.Second), 1), // 10 request every 10 seconds
+		rateLimiter: rate.NewLimiter(rate.Every(1*time.Second), 1), // 10 request every 10 seconds
 		APIUser:     apiUser,
 		APIKey:      apiKey,
 	}
 
-	return c
+	for _, opt := range opts {
-}
+		opt(c)
+	}
 
-func (c *Client) UseURL(url string) {
+	return c
-	c.Url = url
 }
 
 type TorrentResponse struct {
@@ -88,13 +103,13 @@ type Torrent struct {
 
 func (c *Client) Do(req *http.Request) (*http.Response, error) {
 	ctx := context.Background()
-	err := c.Ratelimiter.Wait(ctx) // This is a blocking call. Honors the rate limit
+	err := c.rateLimiter.Wait(ctx) // This is a blocking call. Honors the rate limit
 	if err != nil {
 		return nil, errors.Wrap(err, "error waiting for ratelimiter")
 	}
 	resp, err := c.client.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "error making request")
+		return resp, errors.Wrap(err, "error making request")
 	}
 	return resp, nil
 }
@@ -111,15 +126,15 @@ func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
 
 	res, err := c.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "ptp client request error : %v", url)
+		return res, errors.Wrap(err, "ptp client request error : %v", url)
 	}
 
 	if res.StatusCode == http.StatusUnauthorized {
-		return nil, errors.New("unauthorized: bad credentials")
+		return res, ErrUnauthorized
 	} else if res.StatusCode == http.StatusForbidden {
-		return nil, nil
+		return res, ErrForbidden
 	} else if res.StatusCode == http.StatusTooManyRequests {
-		return nil, nil
+		return res, ErrTooManyRequests
 	}
 
 	return res, nil
@@ -136,14 +151,16 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 	v.Add("torrentid", torrentID)
 	params := v.Encode()
 
-	reqUrl := fmt.Sprintf("%v?%v", c.Url, params)
+	reqUrl := fmt.Sprintf("%v?%v", c.url, params)
 
 	resp, err := c.get(ctx, reqUrl)
 	if err != nil {
 		return nil, errors.Wrap(err, "error requesting data")
 	}
 
-	defer resp.Body.Close()
+	defer func() {
+		resp.Body.Close()
+	}()
 
 	body, readErr := io.ReadAll(resp.Body)
 	if readErr != nil {
@@ -169,7 +186,7 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 
 // TestAPI try api access against torrents page
 func (c *Client) TestAPI(ctx context.Context) (bool, error) {
-	resp, err := c.get(ctx, c.Url)
+	resp, err := c.get(ctx, c.url)
 	if err != nil {
 		return false, errors.Wrap(err, "error requesting data")
 	}

pkg/ptp/ptp_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package ptp
 
 import (
@@ -91,8 +93,7 @@ func TestPTPClient_GetTorrentByID(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c := NewClient(tt.fields.APIUser, tt.fields.APIKey)
+			c := NewClient(tt.fields.APIUser, tt.fields.APIKey, WithUrl(ts.URL))
-			c.UseURL(tt.fields.Url)
 
 			got, err := c.GetTorrentByID(context.Background(), tt.args.torrentID)
 			if tt.wantErr && assert.Error(t, err) {
@@ -168,8 +169,7 @@ func Test(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c := NewClient(tt.fields.APIUser, tt.fields.APIKey)
+			c := NewClient(tt.fields.APIUser, tt.fields.APIKey, WithUrl(ts.URL))
-			c.UseURL(tt.fields.Url)
 
 			got, err := c.TestAPI(context.Background())
 

pkg/radarr/client.go

@@ -71,16 +71,16 @@ func (c *client) post(ctx context.Context, endpoint string, data interface{}) (*
 
 	res, err := c.http.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not make request: %+v", req)
+		return res, errors.Wrap(err, "could not make request: %+v", req)
 	}
 
 	// validate response
 	if res.StatusCode == http.StatusUnauthorized {
-		return nil, errors.New("unauthorized: bad credentials")
+		return res, errors.New("unauthorized: bad credentials")
 	} else if res.StatusCode == http.StatusBadRequest {
-		return nil, errors.New("radarr: bad request")
+		return res, errors.New("radarr: bad request")
 	} else if res.StatusCode != http.StatusOK {
-		return nil, errors.New("radarr: bad request")
+		return res, errors.New("radarr: bad request")
 	}
 
 	// return raw response and let the caller handle json unmarshal of body

pkg/radarr/radarr.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 type Config struct {
@@ -41,19 +42,19 @@ type client struct {
 }
 
 func New(config Config) Client {
-
 	httpClient := &http.Client{
-		Timeout: time.Second * 120,
+		Timeout:   time.Second * 120,
+		Transport: sharedhttp.Transport,
 	}
 
 	c := &client{
 		config: config,
 		http:   httpClient,
-		Log:    config.Log,
+		Log:    log.New(io.Discard, "", log.LstdFlags),
 	}
 
-	if config.Log == nil {
+	if config.Log != nil {
-		c.Log = log.New(io.Discard, "", log.LstdFlags)
+		c.Log = config.Log
 	}
 
 	return c

pkg/radarr/radarr_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package radarr
 
 import (
@@ -12,9 +14,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_client_Push(t *testing.T) {

pkg/readarr/client.go

@@ -76,9 +76,9 @@ func (c *client) post(ctx context.Context, endpoint string, data interface{}) (*
 
 	// validate response
 	if res.StatusCode == http.StatusUnauthorized {
-		return nil, errors.New("unauthorized: bad credentials")
+		return res, errors.New("unauthorized: bad credentials")
 	} else if res.StatusCode != http.StatusOK {
-		return nil, errors.New("readarr: bad request")
+		return res, errors.New("readarr: bad request")
 	}
 
 	// return raw response and let the caller handle json unmarshal of body

pkg/readarr/readarr.go

@@ -15,6 +15,7 @@ import (
 	"log"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 type Config struct {
@@ -43,20 +44,19 @@ type client struct {
 
 // New create new readarr client
 func New(config Config) Client {
-
 	httpClient := &http.Client{
-		Timeout: time.Second * 120,
+		Timeout:   time.Second * 120,
+		Transport: sharedhttp.Transport,
 	}
 
 	c := &client{
 		config: config,
 		http:   httpClient,
-		Log:    config.Log,
+		Log:    log.New(io.Discard, "", log.LstdFlags),
 	}
 
-	if config.Log == nil {
+	if config.Log != nil {
-		// if no provided logger then use io.Discard
+		c.Log = config.Log
-		c.Log = log.New(io.Discard, "", log.LstdFlags)
 	}
 
 	return c

pkg/readarr/readarr_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package readarr
 
 import (

pkg/red/red.go

@@ -15,38 +15,49 @@ import (
 
 	"github.com/autobrr/autobrr/internal/domain"
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	"golang.org/x/time/rate"
 )
 
+const DefaultURL = "https://redacted.ch/ajax.php"
+
 type ApiClient interface {
 	GetTorrentByID(ctx context.Context, torrentID string) (*domain.TorrentBasic, error)
 	TestAPI(ctx context.Context) (bool, error)
-	UseURL(url string)
 }
 
 type Client struct {
-	Url         string
+	url         string
 	client      *http.Client
-	RateLimiter *rate.Limiter
+	rateLimiter *rate.Limiter
 	APIKey      string
 }
 
-func NewClient(apiKey string) ApiClient {
+type OptFunc func(*Client)
+
+func WithUrl(url string) OptFunc {
+	return func(c *Client) {
+		c.url = url
+	}
+}
+
+func NewClient(apiKey string, opts ...OptFunc) ApiClient {
 	c := &Client{
-		Url: "https://redacted.ch/ajax.php",
+		url: DefaultURL,
 		client: &http.Client{
-			Timeout: time.Second * 30,
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
 		},
-		RateLimiter: rate.NewLimiter(rate.Every(10*time.Second), 10),
+		rateLimiter: rate.NewLimiter(rate.Every(10*time.Second), 10),
 		APIKey:      apiKey,
 	}
 
-	return c
+	for _, opt := range opts {
-}
+		opt(c)
+	}
 
-func (c *Client) UseURL(url string) {
+	return c
-	c.Url = url
 }
 
 type ErrorResponse struct {
@@ -126,13 +137,13 @@ type Torrent struct {
 
 func (c *Client) Do(req *http.Request) (*http.Response, error) {
 	//ctx := context.Background()
-	err := c.RateLimiter.Wait(req.Context()) // This is a blocking call. Honors the rate limit
+	err := c.rateLimiter.Wait(req.Context()) // This is a blocking call. Honors the rate limit
 	if err != nil {
 		return nil, err
 	}
 	resp, err := c.client.Do(req)
 	if err != nil {
-		return nil, err
+		return resp, err
 	}
 	return resp, nil
 }
@@ -152,7 +163,7 @@ func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
 
 	res, err := c.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not make request: %+v", req)
+		return res, errors.Wrap(err, "could not make request: %+v", req)
 	}
 
 	// return early if not OK
@@ -161,16 +172,16 @@ func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
 
 		body, readErr := io.ReadAll(res.Body)
 		if readErr != nil {
-			return nil, errors.Wrap(readErr, "could not read body")
+			return res, errors.Wrap(readErr, "could not read body")
 		}
 
 		if err = json.Unmarshal(body, &r); err != nil {
-			return nil, errors.Wrap(readErr, "could not unmarshal body")
+			return res, errors.Wrap(readErr, "could not unmarshal body")
 		}
 
 		res.Body.Close()
 
-		return nil, errors.New("status code: %d status: %s error: %s", res.StatusCode, r.Status, r.Error)
+		return res, errors.New("status code: %d status: %s error: %s", res.StatusCode, r.Status, r.Error)
 	}
 
 	return res, nil
@@ -187,7 +198,7 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 	v.Add("id", torrentID)
 	params := v.Encode()
 
-	reqUrl := fmt.Sprintf("%s?action=torrent&%s", c.Url, params)
+	reqUrl := fmt.Sprintf("%s?action=torrent&%s", c.url, params)
 
 	resp, err := c.get(ctx, reqUrl)
 	if err != nil {
@@ -215,7 +226,7 @@ func (c *Client) GetTorrentByID(ctx context.Context, torrentID string) (*domain.
 
 // TestAPI try api access against torrents page
 func (c *Client) TestAPI(ctx context.Context) (bool, error) {
-	resp, err := c.get(ctx, c.Url+"?action=index")
+	resp, err := c.get(ctx, c.url+"?action=index")
 	if err != nil {
 		return false, errors.Wrap(err, "test api error")
 	}

pkg/red/red_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package red
 
 import (
@@ -12,6 +14,7 @@ import (
 	"testing"
 
 	"github.com/autobrr/autobrr/internal/domain"
+
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
@@ -98,8 +101,7 @@ func TestREDClient_GetTorrentByID(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c := NewClient(tt.fields.APIKey)
+			c := NewClient(tt.fields.APIKey, WithUrl(ts.URL))
-			c.UseURL(tt.fields.Url)
 
 			got, err := c.GetTorrentByID(context.Background(), tt.args.torrentID)
 			if tt.wantErr != "" && assert.Error(t, err) {

pkg/sabnzbd/sabnzbd.go

@@ -12,6 +12,8 @@ import (
 	"net/http"
 	"net/url"
 	"time"
+
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 type Client struct {
@@ -23,7 +25,7 @@ type Client struct {
 
 	log *log.Logger
 
-	Http *http.Client
+	http *http.Client
 }
 
 type Options struct {
@@ -43,8 +45,9 @@ func New(opts Options) *Client {
 		basicUser: opts.BasicUser,
 		basicPass: opts.BasicPass,
 		log:       log.New(io.Discard, "", log.LstdFlags),
-		Http: &http.Client{
+		http: &http.Client{
-			Timeout: time.Second * 60,
+			Timeout:   time.Second * 60,
+			Transport: sharedhttp.Transport,
 		},
 	}
 
@@ -88,7 +91,7 @@ func (c *Client) AddFromUrl(ctx context.Context, r AddNzbRequest) (*AddFileRespo
 		req.SetBasicAuth(c.basicUser, c.basicPass)
 	}
 
-	res, err := c.Http.Do(req)
+	res, err := c.http.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -137,7 +140,7 @@ func (c *Client) Version(ctx context.Context) (*VersionResponse, error) {
 		req.SetBasicAuth(c.basicUser, c.basicPass)
 	}
 
-	res, err := c.Http.Do(req)
+	res, err := c.http.Do(req)
 	if err != nil {
 		return nil, err
 	}

pkg/sharedhttp/http.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+package sharedhttp
+
+import (
+	"crypto/tls"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+var Transport = &http.Transport{
+	Proxy: http.ProxyFromEnvironment,
+	DialContext: (&net.Dialer{
+		Timeout:   30 * time.Second, // default transport value
+		KeepAlive: 30 * time.Second, // default transport value
+	}).DialContext,
+	ForceAttemptHTTP2:     true,             // default is true; since HTTP/2 multiplexes a single TCP connection. we'd want to use HTTP/1, which would use multiple TCP connections.
+	MaxIdleConns:          100,              // default transport value
+	MaxIdleConnsPerHost:   10,               // default is 2, so we want to increase the number to use establish more connections.
+	IdleConnTimeout:       90 * time.Second, // default transport value
+	TLSHandshakeTimeout:   10 * time.Second, // default transport value
+	ExpectContinueTimeout: 1 * time.Second,  // default transport value
+	ReadBufferSize:        65536,
+	WriteBufferSize:       65536,
+	TLSClientConfig: &tls.Config{
+		MinVersion: tls.VersionTLS12,
+	},
+}
+
+var TransportTLSInsecure = &http.Transport{
+	Proxy: http.ProxyFromEnvironment,
+	DialContext: (&net.Dialer{
+		Timeout:   30 * time.Second, // default transport value
+		KeepAlive: 30 * time.Second, // default transport value
+	}).DialContext,
+	ForceAttemptHTTP2:     true,             // default is true; since HTTP/2 multiplexes a single TCP connection. we'd want to use HTTP/1, which would use multiple TCP connections.
+	MaxIdleConns:          100,              // default transport value
+	MaxIdleConnsPerHost:   10,               // default is 2, so we want to increase the number to use establish more connections.
+	IdleConnTimeout:       90 * time.Second, // default transport value
+	TLSHandshakeTimeout:   10 * time.Second, // default transport value
+	ExpectContinueTimeout: 1 * time.Second,  // default transport value
+	ReadBufferSize:        65536,
+	WriteBufferSize:       65536,
+	TLSClientConfig: &tls.Config{
+		InsecureSkipVerify: true,
+	},
+}
+
+var Client = &http.Client{
+	Timeout:   60 * time.Second,
+	Transport: Transport,
+}
+
+type MagnetRoundTripper struct{}
+
+func (rt *MagnetRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	if r.URL.Scheme == "magnet" {
+		responseBody := r.URL.String()
+		respReader := io.NopCloser(strings.NewReader(responseBody))
+
+		resp := &http.Response{
+			Status:        http.StatusText(http.StatusOK),
+			StatusCode:    http.StatusOK,
+			Body:          respReader,
+			ContentLength: int64(len(responseBody)),
+			Header: map[string][]string{
+				"Content-Type": {"text/plain"},
+				"Location":     {responseBody},
+			},
+			Proto:      "HTTP/2.0",
+			ProtoMajor: 2,
+		}
+
+		return resp, nil
+	}
+
+	return Transport.RoundTrip(r)
+}
+
+var MagnetTransport = &MagnetRoundTripper{}

pkg/sonarr/client.go

@@ -71,14 +71,14 @@ func (c *client) post(ctx context.Context, endpoint string, data interface{}) (*
 
 	res, err := c.http.Do(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not make request: %+v", req)
+		return res, errors.Wrap(err, "could not make request: %+v", req)
 	}
 
 	// validate response
 	if res.StatusCode == http.StatusUnauthorized {
-		return nil, errors.New("unauthorized: bad credentials")
+		return res, errors.New("unauthorized: bad credentials")
 	} else if res.StatusCode != http.StatusOK {
-		return nil, errors.New("sonarr: bad request")
+		return res, errors.New("sonarr: bad request")
 	}
 
 	// return raw response and let the caller handle json unmarshal of body

pkg/sonarr/sonarr.go

@@ -15,6 +15,7 @@ import (
 	"log"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 type Config struct {
@@ -43,20 +44,19 @@ type client struct {
 
 // New create new sonarr client
 func New(config Config) Client {
-
 	httpClient := &http.Client{
-		Timeout: time.Second * 120,
+		Timeout:   time.Second * 120,
+		Transport: sharedhttp.Transport,
 	}
 
 	c := &client{
 		config: config,
 		http:   httpClient,
-		Log:    config.Log,
+		Log:    log.New(io.Discard, "", log.LstdFlags),
 	}
 
-	if config.Log == nil {
+	if config.Log != nil {
-		// if no provided logger then use io.Discard
+		c.Log = config.Log
-		c.Log = log.New(io.Discard, "", log.LstdFlags)
 	}
 
 	return c

pkg/sonarr/sonarr_test.go

@@ -1,26 +1,27 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package sonarr
 
 import (
 	"context"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_client_Push(t *testing.T) {
 	// disable logger
 	zerolog.SetGlobalLevel(zerolog.Disabled)
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 
 	mux := http.NewServeMux()
 	ts := httptest.NewServer(mux)
@@ -125,7 +126,7 @@ func Test_client_Push(t *testing.T) {
 func Test_client_Test(t *testing.T) {
 	// disable logger
 	zerolog.SetGlobalLevel(zerolog.Disabled)
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 
 	key := "mock-key"
 

pkg/torznab/torznab.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 type Client interface {
@@ -61,7 +62,8 @@ type Capabilities struct {
 
 func NewClient(config Config) Client {
 	httpClient := &http.Client{
-		Timeout: config.Timeout,
+		Timeout:   config.Timeout,
+		Transport: sharedhttp.Transport,
 	}
 
 	c := &client{

pkg/torznab/torznab_test.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+//go:build integration
+
 package torznab
 
 import (
@@ -51,7 +53,7 @@ import (
 //		{
 //			name: "get feed",
 //			fields: fields{
-//				Host:      srv.URL + "/api",
+//				Host:      srv.url + "/api",
 //				ApiKey:    key,
 //				BasicAuth: BasicAuth{},
 //			},

pkg/transmission/transmission.go

@@ -6,6 +6,8 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
+
 	"github.com/hekmon/transmissionrpc/v3"
 )
 
@@ -43,7 +45,7 @@ type customTransport struct {
 }
 
 func (t *customTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	dt := http.DefaultTransport.(*http.Transport).Clone()
+	dt := sharedhttp.Transport
 	if t.TLSSkipVerify {
 		dt.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}

pkg/version/version.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 
 	goversion "github.com/hashicorp/go-version"
 )
@@ -72,6 +73,8 @@ type Checker struct {
 	Owner          string
 	Repo           string
 	CurrentVersion string
+
+	httpClient *http.Client
 }
 
 func NewChecker(owner, repo, currentVersion string) *Checker {
@@ -79,6 +82,10 @@ func NewChecker(owner, repo, currentVersion string) *Checker {
 		Owner:          owner,
 		Repo:           repo,
 		CurrentVersion: currentVersion,
+		httpClient: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: sharedhttp.Transport,
+		},
 	}
 }
 
@@ -93,12 +100,11 @@ func (c *Checker) get(ctx context.Context) (*Release, error) {
 	req.Header.Set("Accept", "application/vnd.github.v3+json")
 	req.Header.Set("User-Agent", c.buildUserAgent())
 
-	client := http.DefaultClient
+	resp, err := c.httpClient.Do(req)
-
-	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err
 	}
+
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {

pkg/whisparr/whisparr.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/autobrr/autobrr/pkg/errors"
+	"github.com/autobrr/autobrr/pkg/sharedhttp"
 )
 
 type Config struct {
@@ -40,19 +41,19 @@ type client struct {
 }
 
 func New(config Config) Client {
-
 	httpClient := &http.Client{
-		Timeout: time.Second * 120,
+		Timeout:   time.Second * 120,
+		Transport: sharedhttp.Transport,
 	}
 
 	c := &client{
 		config: config,
 		http:   httpClient,
-		Log:    config.Log,
+		Log:    log.New(io.Discard, "", log.LstdFlags),
 	}
 
-	if config.Log == nil {
+	if config.Log != nil {
-		c.Log = log.New(io.Discard, "", log.LstdFlags)
+		c.Log = config.Log
 	}
 
 	return c

test/docs/docs_test.go

@@ -1,3 +1,8 @@
+// Copyright (c) 2021 - 2023, Ludvig Lundgren and the autobrr contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+//go:build integration
+
 package http
 
 import (