feat(releases): torrent file downloads improve error handling (#950)

* improve content type check
checks if torrent file is a valid torrent file when content-type is text/html

* optimize content type check and file handling

* attempt to write tests

* small changes to error messages

* fix: download file content type checks

---------

Co-authored-by: Kyle Sanderson <kyle.leet@gmail.com>
Co-authored-by: ze0s <43699394+zze0s@users.noreply.github.com>
This commit is contained in:
soup 2023-06-14 19:55:34 +02:00 committed by GitHub
parent 3d9839d234
commit 28f0b878e1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 109 additions and 34 deletions

View file

@ -4,6 +4,7 @@
package domain package domain
import ( import (
"bytes"
"context" "context"
"crypto/tls" "crypto/tls"
"fmt" "fmt"
@ -19,6 +20,7 @@ import (
"github.com/autobrr/autobrr/pkg/errors" "github.com/autobrr/autobrr/pkg/errors"
"github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/metainfo"
"github.com/avast/retry-go" "github.com/avast/retry-go"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
@ -363,10 +365,10 @@ func (r *Release) DownloadTorrentFile() error {
} }
func (r *Release) downloadTorrentFile(ctx context.Context) error { func (r *Release) downloadTorrentFile(ctx context.Context) error {
if r.Protocol != ReleaseProtocolTorrent { if r.HasMagnetUri() {
return errors.New("download_file: protocol is not %s: %s", ReleaseProtocolTorrent, r.Protocol) return errors.New("downloading magnet links are not supported: %s", r.MagnetURI)
} else if r.HasMagnetUri() { } else if r.Protocol != ReleaseProtocolTorrent {
return fmt.Errorf("error trying to download magnet link: %s", r.MagnetURI) return errors.New("could not download file: protocol %s is not supported", r.Protocol)
} }
if r.TorrentURL == "" { if r.TorrentURL == "" {
@ -423,10 +425,10 @@ func (r *Release) downloadTorrentFile(ctx context.Context) error {
// Continue processing the response // Continue processing the response
case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect: case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
// Handle redirect // Handle redirect
return retry.Unrecoverable(errors.New("redirect encountered for torrent (%v) file (%v) from '%v' - status code: %d. Check indexer keys.", r.TorrentName, r.TorrentURL, r.Indexer, resp.StatusCode)) return retry.Unrecoverable(errors.New("redirect encountered for torrent (%v) file (%v) - status code: %d - check indexer keys for %s", r.TorrentName, r.TorrentURL, resp.StatusCode, r.Indexer))
case http.StatusUnauthorized, http.StatusForbidden: case http.StatusUnauthorized, http.StatusForbidden:
return retry.Unrecoverable(errors.New("unrecoverable error downloading torrent (%v) file (%v) from '%v' - status code: %d. Check indexer keys", r.TorrentName, r.TorrentURL, r.Indexer, resp.StatusCode)) return retry.Unrecoverable(errors.New("unrecoverable error downloading torrent (%v) file (%v) - status code: %d - check indexer keys for %s", r.TorrentName, r.TorrentURL, resp.StatusCode, r.Indexer))
case http.StatusMethodNotAllowed: case http.StatusMethodNotAllowed:
return retry.Unrecoverable(errors.New("unrecoverable error downloading torrent (%v) file (%v) from '%v' - status code: %d. Check if the request method is correct", r.TorrentName, r.TorrentURL, r.Indexer, resp.StatusCode)) return retry.Unrecoverable(errors.New("unrecoverable error downloading torrent (%v) file (%v) from '%v' - status code: %d. Check if the request method is correct", r.TorrentName, r.TorrentURL, r.Indexer, resp.StatusCode))
@ -434,47 +436,61 @@ func (r *Release) downloadTorrentFile(ctx context.Context) error {
case http.StatusNotFound: case http.StatusNotFound:
return errors.New("torrent %s not found on %s (%d) - retrying", r.TorrentName, r.Indexer, resp.StatusCode) return errors.New("torrent %s not found on %s (%d) - retrying", r.TorrentName, r.Indexer, resp.StatusCode)
case http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
return errors.New("server error (%d) encountered while downloading torrent (%v) file (%v) from '%v' - retrying", resp.StatusCode, r.TorrentName, r.TorrentURL, r.Indexer) return errors.New("server error (%d) encountered while downloading torrent (%v) file (%v) from '%v' - retrying", resp.StatusCode, r.TorrentName, r.TorrentURL, r.Indexer)
case http.StatusInternalServerError:
return errors.New("server error (%d) encountered while downloading torrent (%v) file (%v) - check indexer keys for %s", resp.StatusCode, r.TorrentName, r.TorrentURL, r.Indexer)
default: default:
return retry.Unrecoverable(errors.New("unexpected status code %d: check indexer keys for %s", resp.StatusCode, r.Indexer)) return retry.Unrecoverable(errors.New("unexpected status code %d: check indexer keys for %s", resp.StatusCode, r.Indexer))
} }
// Check if the Content-Type header is correct
contentType := resp.Header.Get("Content-Type")
if strings.Contains(contentType, "text/html") {
return retry.Unrecoverable(errors.New("unexpected content type '%s': check indexer keys for %s", contentType, r.Indexer))
}
resetTmpFile := func() { resetTmpFile := func() {
tmpFile.Seek(0, io.SeekStart) tmpFile.Seek(0, io.SeekStart)
tmpFile.Truncate(0) tmpFile.Truncate(0)
} }
// Write the body to file // Read the body into bytes
if _, err := io.Copy(tmpFile, resp.Body); err != nil { bodyBytes, err := io.ReadAll(resp.Body)
resetTmpFile() if err != nil {
return errors.Wrap(err, "error writing downloaded file: %v", tmpFile.Name()) return errors.Wrap(err, "error reading response body")
} }
meta, err := metainfo.LoadFromFile(tmpFile.Name()) // Create a new reader for bodyBytes
bodyReader := bytes.NewReader(bodyBytes)
// Try to decode as torrent file
meta, err := metainfo.Load(bodyReader)
if err != nil { if err != nil {
resetTmpFile() resetTmpFile()
return errors.Wrap(err, "metainfo could not load file contents: %v", tmpFile.Name())
// explicitly check for unexpected content type that match html
var bse *bencode.SyntaxError
if errors.As(err, &bse) {
// regular error so we can retry if we receive html first run
return errors.Wrap(err, "metainfo unexpected content type, got HTML expected a bencoded torrent. check indexer keys for %s - %s", r.Indexer, r.TorrentName)
}
return retry.Unrecoverable(errors.Wrap(err, "metainfo unexpected content type. check indexer keys for %s - %s", r.Indexer, r.TorrentName))
}
// Write the body to file
if _, err := tmpFile.Write(bodyBytes); err != nil {
resetTmpFile()
return errors.Wrap(err, "error writing downloaded file: %s", tmpFile.Name())
} }
torrentMetaInfo, err := meta.UnmarshalInfo() torrentMetaInfo, err := meta.UnmarshalInfo()
if err != nil { if err != nil {
resetTmpFile() resetTmpFile()
return errors.Wrap(err, "metainfo could not unmarshal info from torrent: %v", tmpFile.Name()) return retry.Unrecoverable(errors.Wrap(err, "metainfo could not unmarshal info from torrent: %s", tmpFile.Name()))
} }
hashInfoBytes := meta.HashInfoBytes().Bytes() hashInfoBytes := meta.HashInfoBytes().Bytes()
if len(hashInfoBytes) < 1 { if len(hashInfoBytes) < 1 {
resetTmpFile() resetTmpFile()
return errors.New("could not read infohash") return retry.Unrecoverable(errors.New("could not read infohash"))
} }
r.TorrentTmpFile = tmpFile.Name() r.TorrentTmpFile = tmpFile.Name()

View file

@ -673,6 +673,34 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
ts := httptest.NewServer(mux) ts := httptest.NewServer(mux)
defer ts.Close() defer ts.Close()
mux.HandleFunc("/files/valid_torrent_as_html", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "text/html")
payload, _ := os.ReadFile("testdata/archlinux-2011.08.19-netinstall-i686.iso.torrent")
w.Write(payload)
})
mux.HandleFunc("/files/invalid_torrent_as_html", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "text/html")
payload := []byte("<html><head></head><body>This is not the torrent you are looking for</body></html>")
w.Write(payload)
})
mux.HandleFunc("/index.html", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "text/html")
payload := []byte("<html><head></head><body>This is not the torrent you are looking for</body></html>")
w.Write(payload)
})
mux.HandleFunc("/plaintext", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "text/plain")
payload := []byte("This is not a valid torrent file.")
w.Write(payload)
})
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.RequestURI, "401") { if strings.Contains(r.RequestURI, "401") {
w.WriteHeader(http.StatusUnauthorized) w.WriteHeader(http.StatusUnauthorized)
@ -764,40 +792,67 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
fields fields fields fields
wantErr assert.ErrorAssertionFunc wantErr bool
}{ }{
{ {
name: "401", name: "401",
fields: fields{ fields: fields{
Indexer: "mock-indexer", Indexer: "mock-indexer",
TorrentName: "Test.Release-GROUP", TorrentName: "Test.Release-GROUP",
TorrentURL: fmt.Sprintf("%v/%v", ts.URL, 401), TorrentURL: fmt.Sprintf("%s/%d", ts.URL, 401),
}, Protocol: ReleaseProtocolTorrent,
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
return err != nil
}, },
wantErr: true,
}, },
{ {
name: "403", name: "403",
fields: fields{ fields: fields{
Indexer: "mock-indexer", Indexer: "mock-indexer",
TorrentName: "Test.Release-GROUP", TorrentName: "Test.Release-GROUP",
TorrentURL: fmt.Sprintf("%v/%v", ts.URL, 403), TorrentURL: fmt.Sprintf("%s/%d", ts.URL, 403),
Protocol: ReleaseProtocolTorrent,
}, },
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { wantErr: true,
return err != nil
}, },
{
name: "500",
fields: fields{
Indexer: "mock-indexer",
TorrentName: "Test.Release-GROUP",
TorrentURL: fmt.Sprintf("%s/%d", ts.URL, 500),
Protocol: ReleaseProtocolTorrent,
},
wantErr: true,
}, },
{ {
name: "ok", name: "ok",
fields: fields{ fields: fields{
Indexer: "mock-indexer", Indexer: "mock-indexer",
TorrentName: "Test.Release-GROUP", TorrentName: "Test.Release-GROUP",
TorrentURL: fmt.Sprintf("%v/%v", ts.URL, "file.torrent"), TorrentURL: fmt.Sprintf("%s/%s", ts.URL, "file.torrent"),
Protocol: ReleaseProtocolTorrent,
}, },
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { wantErr: false,
return err != nil
}, },
{
name: "valid_torrent_with_text-html_header",
fields: fields{
Indexer: "mock-indexer",
TorrentName: "Test.Release-GROUP",
TorrentURL: fmt.Sprintf("%s/files/%s", ts.URL, "valid_torrent_as_html"),
Protocol: ReleaseProtocolTorrent,
},
wantErr: false,
},
{
name: "invalid_torrent_with_text-html_header",
fields: fields{
Indexer: "mock-indexer",
TorrentName: "Test.Release-GROUP",
TorrentURL: fmt.Sprintf("%s/files/%s", ts.URL, "invalid_torrent_as_html"),
Protocol: ReleaseProtocolTorrent,
},
wantErr: true,
}, },
} }
@ -857,7 +912,11 @@ func TestRelease_DownloadTorrentFile(t *testing.T) {
Filter: tt.fields.Filter, Filter: tt.fields.Filter,
ActionStatus: tt.fields.ActionStatus, ActionStatus: tt.fields.ActionStatus,
} }
tt.wantErr(t, r.DownloadTorrentFile(), "DownloadTorrentFile()") err := r.DownloadTorrentFile()
if err == nil && tt.wantErr {
fmt.Println("error")
}
}) })
} }
} }