autobrr/internal/domain/release.go

package domain

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"golang.org/x/net/publicsuffix"
	"html"
	"io"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/autobrr/autobrr/pkg/wildcard"

	"github.com/anacrolix/torrent/metainfo"
	"github.com/dustin/go-humanize"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
)

type ReleaseRepo interface {
	Store(ctx context.Context, release *Release) (*Release, error)
	Find(ctx context.Context, params ReleaseQueryParams) (res []Release, nextCursor int64, count int64, err error)
	GetIndexerOptions(ctx context.Context) ([]string, error)
	GetActionStatusByReleaseID(ctx context.Context, releaseID int64) ([]ReleaseActionStatus, error)
	Stats(ctx context.Context) (*ReleaseStats, error)
	StoreReleaseActionStatus(ctx context.Context, actionStatus *ReleaseActionStatus) error
	Delete(ctx context.Context) error
}

type Release struct {
	ID                          int64                 `json:"id"`
	FilterStatus                ReleaseFilterStatus   `json:"filter_status"`
	Rejections                  []string              `json:"rejections"`
	Indexer                     string                `json:"indexer"`
	FilterName                  string                `json:"filter"`
	Protocol                    ReleaseProtocol       `json:"protocol"`
	Implementation              ReleaseImplementation `json:"implementation"` // irc, rss, api
	Timestamp                   time.Time             `json:"timestamp"`
	GroupID                     string                `json:"group_id"`
	TorrentID                   string                `json:"torrent_id"`
	TorrentURL                  string                `json:"-"`
	TorrentTmpFile              string                `json:"-"`
	TorrentHash                 string                `json:"-"`
	TorrentName                 string                `json:"torrent_name"` // full release name
	Size                        uint64                `json:"size"`
	Raw                         string                `json:"raw"`   // Raw release
	Clean                       string                `json:"clean"` // cleaned release name
	Title                       string                `json:"title"` // Parsed title
	Category                    string                `json:"category"`
	Season                      int                   `json:"season"`
	Episode                     int                   `json:"episode"`
	Year                        int                   `json:"year"`
	Resolution                  string                `json:"resolution"`
	Source                      string                `json:"source"` // CD, DVD, Vinyl, DAT, Cassette, WEB, Other
	Codec                       string                `json:"codec"`
	Container                   string                `json:"container"`
	HDR                         string                `json:"hdr"`
	Audio                       string                `json:"audio"`
	Group                       string                `json:"group"`
	Region                      string                `json:"region"`
	Language                    string                `json:"language"`
	Edition                     string                `json:"edition"` // Extended, directors cut
	Unrated                     bool                  `json:"unrated"`
	Hybrid                      bool                  `json:"hybrid"`
	Proper                      bool                  `json:"proper"`
	Repack                      bool                  `json:"repack"`
	Website                     string                `json:"website"`
	ThreeD                      bool                  `json:"-"`
	Artists                     []string              `json:"artists"`
	Type                        string                `json:"type"`    // Album,Single,EP
	Format                      string                `json:"format"`  // music only
	Quality                     string                `json:"quality"` // quality
	LogScore                    int                   `json:"log_score"`
	HasLog                      bool                  `json:"has_log"`
	HasCue                      bool                  `json:"has_cue"`
	IsScene                     bool                  `json:"is_scene"`
	Origin                      string                `json:"origin"` // P2P, Internal
	Tags                        []string              `json:"tags"`
	ReleaseTags                 string                `json:"-"`
	Freeleech                   bool                  `json:"freeleech"`
	FreeleechPercent            int                   `json:"freeleech_percent"`
	Uploader                    string                `json:"uploader"`
	PreTime                     string                `json:"pre_time"`
	RawCookie                   string                `json:"-"`
	AdditionalSizeCheckRequired bool                  `json:"-"`
	FilterID                    int                   `json:"-"`
	Filter                      *Filter               `json:"-"`
	ActionStatus                []ReleaseActionStatus `json:"action_status"`
}

type ReleaseActionStatus struct {
	ID         int64             `json:"id"`
	Status     ReleasePushStatus `json:"status"`
	Action     string            `json:"action"`
	Type       ActionType        `json:"type"`
	Rejections []string          `json:"rejections"`
	Timestamp  time.Time         `json:"timestamp"`
	ReleaseID  int64             `json:"-"`
}

func NewRelease(indexer string, line string) (*Release, error) {
	r := &Release{
		Indexer:        indexer,
		Raw:            line,
		FilterStatus:   ReleaseStatusFilterPending,
		Rejections:     []string{},
		Protocol:       ReleaseProtocolTorrent,
		Implementation: ReleaseImplementationIRC,
		Timestamp:      time.Now(),
		Artists:        []string{},
		Tags:           []string{},
	}

	return r, nil
}

func (r *Release) Parse() error {
	var err error

	err = r.extractYear()
	err = r.extractSeason()
	err = r.extractEpisode()
	err = r.extractResolution()
	err = r.extractSource()
	err = r.extractCodec()
	err = r.extractContainer()
	err = r.extractHDR()
	err = r.extractAudio()
	err = r.extractGroup()
	err = r.extractRegion()
	err = r.extractLanguage()
	err = r.extractEdition()
	err = r.extractUnrated()
	err = r.extractHybrid()
	err = r.extractProper()
	err = r.extractRepack()
	err = r.extractWebsite()
	err = r.extractReleaseTags()

	r.Clean = cleanReleaseName(r.TorrentName)

	if err != nil {
		log.Trace().Msgf("could not parse release: %v", r.TorrentName)
		return err
	}

	return nil
}

func (r *Release) extractYear() error {
	if r.Year > 0 {
		return nil
	}

	y, err := findLastInt(r.TorrentName, `\b(((?:19[0-9]|20[0-9])[0-9]))\b`)
	if err != nil {
		return err
	}
	r.Year = y

	return nil
}

func (r *Release) extractSeason() error {
	s, err := findLastInt(r.TorrentName, `(?i)(?:S|Season\s*)(\d{1,3})`)
	if err != nil {
		return err
	}
	r.Season = s

	return nil
}

func (r *Release) extractEpisode() error {
	e, err := findLastInt(r.TorrentName, `(?i)[ex]([0-9]{2})(?:[^0-9]|$)`)
	if err != nil {
		return err
	}
	r.Episode = e

	return nil
}

func (r *Release) extractResolution() error {
	v, err := findLast(r.TorrentName, `\b(?i)(([0-9]{3,4}p|i))\b`)
	if err != nil {
		return err
	}
	r.Resolution = v

	return nil
}

func (r *Release) extractResolutionFromTags(tag string) error {
	if r.Resolution != "" {
		return nil
	}
	v, err := findLast(tag, `\b(?i)(([0-9]{3,4}p|i))\b`)
	if err != nil {
		return err
	}
	r.Resolution = v

	return nil
}

func (r *Release) extractSource() error {
	v, err := findLast(r.TorrentName, `(?i)\b(((?:PPV\.)?[HP]DTV|(?:HD)?CAM|B[DR]Rip|(?:HD-?)?TS|(?:PPV )?WEB-?DL(?: DVDRip)?|HDRip|DVDRip|DVDRIP|CamRip|WEB|W[EB]BRip|Blu-?Ray|DvDScr|telesync|CD|DVD|Vinyl|DAT|Cassette))\b`)
	if err != nil {
		return err
	}
	r.Source = v

	return nil
}

func (r *Release) extractSourceFromTags(tag string) error {
	if r.Source != "" {
		return nil
	}
	v, err := findLast(tag, `(?i)\b(((?:PPV\.)?[HP]DTV|(?:HD)?CAM|B[DR]Rip|(?:HD-?)?TS|(?:PPV )?WEB-?DL(?: DVDRip)?|HDRip|DVDRip|DVDRIP|CamRip|WEB|W[EB]BRip|Blu-?Ray|DvDScr|telesync|CD|DVD|Vinyl|DAT|Cassette))\b`)
	if err != nil {
		return err
	}
	r.Source = v

	return nil
}

func (r *Release) extractCodec() error {
	v, err := findLast(r.TorrentName, `(?i)\b(HEVC|[hx]\.?26[45] 10-bit|[hx]\.?26[45]|xvid|divx|AVC|MPEG-?2|AV1|VC-?1|VP9|WebP)\b`)
	if err != nil {
		return err
	}
	r.Codec = v

	return nil
}

func (r *Release) extractCodecFromTags(tag string) error {
	if r.Codec != "" {
		return nil
	}

	v, err := findLast(tag, `(?i)\b(HEVC|[hx]\.?26[45] 10-bit|[hx]\.?26[45]|xvid|divx|AVC|MPEG-?2|AV1|VC-?1|VP9|WebP)\b`)
	if err != nil {
		return err
	}
	r.Codec = v

	return nil
}

func (r *Release) extractContainer() error {
	v, err := findLast(r.TorrentName, `(?i)\b(AVI|MPG|MKV|MP4|VOB|m2ts|ISO|IMG)\b`)
	if err != nil {
		return err
	}
	r.Container = v

	return nil
}

func (r *Release) extractContainerFromTags(tag string) error {
	if r.Container != "" {
		return nil
	}

	v, err := findLast(tag, `(?i)\b(AVI|MPG|MKV|MP4|VOB|m2ts|ISO|IMG)\b`)
	if err != nil {
		return err
	}
	r.Container = v

	return nil
}

func (r *Release) extractHDR() error {
	v, err := findLast(r.TorrentName, `(?i)[\. ](HDR10\+|HDR10|DoVi[\. ]HDR|DV[\. ]HDR10\+|DV[\. ]HDR10|DV[\. ]HDR|HDR|DV|DoVi|Dolby[\. ]Vision[\. ]\+[\. ]HDR10|Dolby[\. ]Vision)[\. ]`)
	if err != nil {
		return err
	}
	r.HDR = v

	return nil
}

func (r *Release) extractAudio() error {
	v, err := findLast(r.TorrentName, `(?i)(FLAC[\. ][1-7][\. ][0-2]|FLAC|Opus|DD-EX|DDP[\. ]?[124567][\. ][012] Atmos|DDP[\. ]?[124567][\. ][012]|DDP|DD[1-7][\. ][0-2]|Dual[\- ]Audio|LiNE|PCM|Dolby TrueHD [0-9][\. ][0-4]|TrueHD [0-9][\. ][0-4] Atmos|TrueHD [0-9][\. ][0-4]|DTS X|DTS-HD MA [0-9][\. ][0-4]|DTS-HD MA|DTS-ES|DTS [1-7][\. ][0-2]|DTS|DD|DD[12][\. ]0|Dolby Atmos|TrueHD ATMOS|TrueHD|Atmos|Dolby Digital Plus|Dolby Digital Audio|Dolby Digital|AAC[.-]LC|AAC (?:\.?[1-7]\.[0-2])?|AAC|eac3|AC3(?:\.5\.1)?)`)
	if err != nil {
		return err
	}
	r.Audio = v

	return nil
}

func (r *Release) extractAudioFromTags(tag string) error {
	if r.Audio != "" {
		return nil
	}

	v, err := findLast(tag, `(?i)(FLAC[\. ][1-7][\. ][0-2]|FLAC|Opus|DD-EX|DDP[\. ]?[124567][\. ][012] Atmos|DDP[\. ]?[124567][\. ][012]|DDP|DD[1-7][\. ][0-2]|Dual[\- ]Audio|LiNE|PCM|Dolby TrueHD [0-9][\. ][0-4]|TrueHD [0-9][\. ][0-4] Atmos|TrueHD [0-9][\. ][0-4]|DTS X|DTS-HD MA [0-9][\. ][0-4]|DTS-HD MA|DTS-ES|DTS [1-7][\. ][0-2]|DTS|DD|DD[12][\. ]0|Dolby Atmos|TrueHD ATMOS|TrueHD|Atmos|Dolby Digital Plus|Dolby Digital Audio|Dolby Digital|AAC[.-]LC|AAC (?:\.?[1-7]\.[0-2])?|AAC|eac3|AC3(?:\.5\.1)?)`)
	if err != nil {
		return err
	}
	r.Audio = v

	return nil
}

func (r *Release) extractFormatsFromTags(tag string) error {
	if r.Format != "" {
		return nil
	}

	v, err := findLast(tag, `(?i)(?:MP3|FLAC|Ogg Vorbis|AAC|AC3|DTS)`)
	if err != nil {
		return err
	}
	r.Format = v

	return nil
}

//func (r *Release) extractCueFromTags(tag string) error {
//	v, err := findLast(tag, `Cue`)
//	if err != nil {
//		return err
//	}
//	r.HasCue = v
//
//	return nil
//}

func (r *Release) extractGroup() error {
	// try first for wierd anime group names [group] show name, or in brackets at the end

	//g, err := findLast(r.Clean, `\[(.*?)\]`)
	group, err := findLast(r.TorrentName, `\-([a-zA-Z0-9_\.]+)$`)
	if err != nil {
		return err
	}

	r.Group = group

	return nil
}

func (r *Release) extractAnimeGroupFromTags(tag string) error {
	if r.Group != "" {
		return nil
	}
	v, err := findLast(tag, `(?:RAW|Softsubs|Hardsubs)\s\((.+)\)`)
	if err != nil {
		return err
	}
	r.Group = v

	return nil
}

func (r *Release) extractRegion() error {
	v, err := findLast(r.TorrentName, `(?i)\b(R([0-9]))\b`)
	if err != nil {
		return err
	}
	r.Region = v

	return nil
}

func (r *Release) extractLanguage() error {
	v, err := findLast(r.TorrentName, `(?i)\b((DK|DKSUBS|DANiSH|DUTCH|NL|NLSUBBED|ENG|FI|FLEMiSH|FiNNiSH|DE|FRENCH|GERMAN|HE|HEBREW|HebSub|HiNDi|iCELANDiC|KOR|MULTi|MULTiSUBS|NORWEGiAN|NO|NORDiC|PL|PO|POLiSH|PLDUB|RO|ROMANiAN|RUS|SPANiSH|SE|SWEDiSH|SWESUB||))\b`)
	if err != nil {
		return err
	}
	r.Language = v

	return nil
}

func (r *Release) extractEdition() error {
	v, err := findLast(r.TorrentName, `(?i)\b((?:DIRECTOR'?S|EXTENDED|INTERNATIONAL|THEATRICAL|ORIGINAL|FINAL|BOOTLEG)(?:.CUT)?)\b`)
	if err != nil {
		return err
	}
	r.Edition = v

	return nil
}

func (r *Release) extractUnrated() error {
	v, err := findLastBool(r.TorrentName, `(?i)\b((UNRATED))\b`)
	if err != nil {
		return err
	}
	r.Unrated = v

	return nil
}

func (r *Release) extractHybrid() error {
	v, err := findLastBool(r.TorrentName, `(?i)\b((HYBRID))\b`)
	if err != nil {
		return err
	}
	r.Hybrid = v

	return nil
}

func (r *Release) extractProper() error {
	v, err := findLastBool(r.TorrentName, `(?i)\b((PROPER))\b`)
	if err != nil {
		return err
	}
	r.Proper = v

	return nil
}

func (r *Release) extractRepack() error {
	v, err := findLastBool(r.TorrentName, `(?i)\b((REPACK))\b`)
	if err != nil {
		return err
	}
	r.Repack = v

	return nil
}

func (r *Release) extractWebsite() error {
	// Start with the basic most common ones
	v, err := findLast(r.TorrentName, `(?i)\b((AMBC|AS|AMZN|AMC|ANPL|ATVP|iP|CORE|BCORE|CMOR|CN|CBC|CBS|CMAX|CNBC|CC|CRIT|CR|CSPN|CW|DAZN|DCU|DISC|DSCP|DSNY|DSNP|DPLY|ESPN|FOX|FUNI|PLAY|HBO|HMAX|HIST|HS|HOTSTAR|HULU|iT|MNBC|MTV|NATG|NBC|NF|NICK|NRK|PMNT|PMNP|PCOK|PBS|PBSK|PSN|QIBI|SBS|SHO|STAN|STZ|SVT|SYFY|TLC|TRVL|TUBI|TV3|TV4|TVL|VH1|VICE|VMEO|UFC|USAN|VIAP|VIAPLAY|VL|WWEN|XBOX|YHOO|YT|RED))\b`)
	if err != nil {
		return err
	}
	r.Website = v

	return nil
}

func (r *Release) extractFreeleechFromTags(tag string) error {
	if r.Freeleech == true {
		return nil
	}

	// Start with the basic most common ones
	v, err := findLast(tag, `(?i)(Freeleech!|Freeleech)`)
	if err != nil {
		return err
	}
	if v != "" {
		r.Freeleech = true
		return nil
	}

	r.Freeleech = false

	return nil
}

func (r *Release) extractLogScoreFromTags(tag string) error {
	if r.LogScore > 0 {
		return nil
	}

	// Start with the basic most common ones

	rxp, err := regexp.Compile(`([\d\.]+)%`)
	if err != nil {
		return err
		//return errors.Wrapf(err, "invalid regex: %s", value)
	}

	matches := rxp.FindStringSubmatch(tag)
	if matches != nil {
		// first value is the match, second value is the text
		if len(matches) >= 1 {
			last := matches[len(matches)-1]
			score, err := strconv.ParseInt(last, 10, 32)
			if err != nil {
				return err
			}

			r.LogScore = int(score)
			return nil
		}
	}

	return nil
}

func (r *Release) extractQualityFromTags(tag string) error {
	if r.Quality != "" {
		return nil
	}

	// Start with the basic most common ones

	rxp, err := regexp.Compile(`(?i)(Lossless|24bit Lossless|V0 \(VBR\)|V1 \(VBR\)|V2 \(VBR\)|APS \(VBR\)|APX \(VBR\)|320|256|192)`)
	if err != nil {
		return err
		//return errors.Wrapf(err, "invalid regex: %s", value)
	}

	matches := rxp.FindStringSubmatch(tag)
	if matches != nil {
		// first value is the match, second value is the text
		if len(matches) >= 1 {
			last := matches[len(matches)-1]

			r.Quality = last
			return nil
		}
	}

	return nil
}

func (r *Release) extractReleaseTags() error {
	if r.ReleaseTags == "" {
		return nil
	}

	tags := SplitAny(r.ReleaseTags, ",|/")

	for _, t := range tags {
		t = strings.Trim(t, " ")

		var err error
		err = r.extractAudioFromTags(t)
		err = r.extractFormatsFromTags(t)
		err = r.extractResolutionFromTags(t)
		err = r.extractCodecFromTags(t)
		err = r.extractContainerFromTags(t)
		err = r.extractSourceFromTags(t)
		err = r.extractFreeleechFromTags(t)
		err = r.extractLogScoreFromTags(t)
		err = r.extractQualityFromTags(t)
		err = r.extractAnimeGroupFromTags(t)

		if err != nil {
			continue
		}

		switch t {
		case "Cue":
			r.HasCue = true
		case "Log":
			r.HasLog = true
			// check percent
		}
	}

	return nil
}

func (r *Release) ParseTorrentUrl(match string, vars map[string]string, extraVars map[string]string, encode []string) error {
	tmpVars := map[string]string{}

	// copy vars to new tmp map
	for k, v := range vars {
		tmpVars[k] = v
	}

	// merge extra vars with vars
	if extraVars != nil {
		for k, v := range extraVars {
			tmpVars[k] = v
		}
	}

	// handle url encode of values
	if encode != nil {
		for _, e := range encode {
			if v, ok := tmpVars[e]; ok {
				// url encode  value
				t := url.QueryEscape(v)
				tmpVars[e] = t
			}
		}
	}

	// setup text template to inject variables into
	tmpl, err := template.New("torrenturl").Parse(match)
	if err != nil {
		log.Error().Err(err).Msg("could not create torrent url template")
		return err
	}

	var urlBytes bytes.Buffer
	err = tmpl.Execute(&urlBytes, &tmpVars)
	if err != nil {
		log.Error().Err(err).Msg("could not write torrent url template output")
		return err
	}

	r.TorrentURL = urlBytes.String()

	// handle cookies
	if v, ok := extraVars["cookie"]; ok {
		r.RawCookie = v
	}

	return nil
}

func (r *Release) DownloadTorrentFile() error {
	if r.TorrentURL == "" {
		return errors.New("download_file: url can't be empty")
	} else if r.TorrentTmpFile != "" {
		// already downloaded
		return nil
	}

	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
	if err != nil {
		return err
	}

	customTransport := http.DefaultTransport.(*http.Transport).Clone()
	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	client := &http.Client{
		Transport: customTransport,
		Jar:       jar,
	}

	req, err := http.NewRequest("GET", r.TorrentURL, nil)
	if err != nil {
		log.Error().Stack().Err(err).Msg("error downloading file")
		return err
	}

	if r.RawCookie != "" {
		// set the cookie on the header instead of req.AddCookie
		// since we have a raw cookie like "uid=10; pass=000"
		req.Header.Set("Cookie", r.RawCookie)
	}

	// Get the data
	resp, err := client.Do(req)
	if err != nil {
		log.Error().Stack().Err(err).Msg("error downloading file")
		return err
	}
	defer resp.Body.Close()

	// retry logic

	if resp.StatusCode != http.StatusOK {
		log.Error().Stack().Err(err).Msgf("error downloading file from: %v - bad status: %d", r.TorrentURL, resp.StatusCode)
		return fmt.Errorf("error downloading torrent (%v) file (%v) from '%v' - status code: %d", r.TorrentName, r.TorrentURL, r.Indexer, resp.StatusCode)
	}

	// Create tmp file
	tmpFile, err := os.CreateTemp("", "autobrr-")
	if err != nil {
		log.Error().Stack().Err(err).Msg("error creating temp file")
		return err
	}
	defer tmpFile.Close()

	// Write the body to file
	_, err = io.Copy(tmpFile, resp.Body)
	if err != nil {
		log.Error().Stack().Err(err).Msgf("error writing downloaded file: %v", tmpFile.Name())
		return err
	}

	meta, err := metainfo.LoadFromFile(tmpFile.Name())
	if err != nil {
		log.Error().Stack().Err(err).Msgf("metainfo could not load file contents: %v", tmpFile.Name())
		return err
	}

	torrentMetaInfo, err := meta.UnmarshalInfo()
	if err != nil {
		log.Error().Stack().Err(err).Msgf("metainfo could not unmarshal info from torrent: %v", tmpFile.Name())
		return err
	}

	r.TorrentTmpFile = tmpFile.Name()
	r.TorrentHash = meta.HashInfoBytes().String()
	r.Size = uint64(torrentMetaInfo.TotalLength())

	// remove file if fail

	log.Debug().Msgf("successfully downloaded file: %v", tmpFile.Name())

	return nil
}

func (r *Release) addRejection(reason string) {
	r.Rejections = append(r.Rejections, reason)
}

// ResetRejections reset rejections between filter checks
func (r *Release) resetRejections() {
	r.Rejections = []string{}
}

func (r *Release) CheckFilter(filter Filter) bool {
	// reset rejections first to clean previous checks
	r.resetRejections()

	if !filter.Enabled {
		return false
	}

	// FIXME what if someone explicitly doesnt want scene, or toggles in filter. Make enum? 0,1,2? Yes, No, Dont care
	if filter.Scene && r.IsScene != filter.Scene {
		r.addRejection("wanted: scene")
		return false
	}

	if filter.Freeleech && r.Freeleech != filter.Freeleech {
		r.addRejection("wanted: freeleech")
		return false
	}

	if filter.FreeleechPercent != "" && !checkFreeleechPercent(r.FreeleechPercent, filter.FreeleechPercent) {
		r.addRejection("freeleech percent not matching")
		return false
	}

	// check against TorrentName and Clean which is a cleaned name without (. _ -)
	if filter.Shows != "" && !checkMultipleFilterStrings(filter.Shows, r.TorrentName, r.Clean) {
		r.addRejection("shows not matching")
		return false
	}

	if filter.Seasons != "" && !checkFilterIntStrings(r.Season, filter.Seasons) {
		r.addRejection("season not matching")
		return false
	}

	if filter.Episodes != "" && !checkFilterIntStrings(r.Episode, filter.Episodes) {
		r.addRejection("episode not matching")
		return false
	}

	// matchRelease
	// TODO allow to match against regex
	if filter.MatchReleases != "" && !checkMultipleFilterStrings(filter.MatchReleases, r.TorrentName, r.Clean) {
		r.addRejection("match release not matching")
		return false
	}

	if filter.ExceptReleases != "" && checkMultipleFilterStrings(filter.ExceptReleases, r.TorrentName, r.Clean) {
		r.addRejection("except_releases: unwanted release")
		return false
	}

	if filter.MatchReleaseGroups != "" && !checkMultipleFilterGroups(filter.MatchReleaseGroups, r.Group, r.Clean) {
		r.addRejection("release groups not matching")
		return false
	}

	if filter.ExceptReleaseGroups != "" && checkMultipleFilterGroups(filter.ExceptReleaseGroups, r.Group, r.Clean) {
		r.addRejection("unwanted release group")
		return false
	}

	if filter.MatchUploaders != "" && !checkFilterStrings(r.Uploader, filter.MatchUploaders) {
		r.addRejection("uploaders not matching")
		return false
	}

	if filter.ExceptUploaders != "" && checkFilterStrings(r.Uploader, filter.ExceptUploaders) {
		r.addRejection("unwanted uploaders")
		return false
	}

	if len(filter.Resolutions) > 0 && !checkFilterSlice(r.Resolution, filter.Resolutions) {
		r.addRejection("resolution not matching")
		return false
	}

	if len(filter.Codecs) > 0 && !checkFilterSlice(r.Codec, filter.Codecs) {
		r.addRejection("codec not matching")
		return false
	}

	if len(filter.Sources) > 0 && !checkFilterSource(r.Source, filter.Sources) {
		r.addRejection("source not matching")
		return false
	}

	if len(filter.Containers) > 0 && !checkFilterSlice(r.Container, filter.Containers) {
		r.addRejection("container not matching")
		return false
	}

	if len(filter.MatchHDR) > 0 && !checkMultipleFilterHDR(filter.MatchHDR, r.HDR, r.TorrentName) {
		r.addRejection("hdr not matching")
		return false
	}

	if len(filter.ExceptHDR) > 0 && checkMultipleFilterHDR(filter.ExceptHDR, r.HDR, r.TorrentName) {
		r.addRejection("unwanted hdr")
		return false
	}

	if filter.Years != "" && !checkFilterIntStrings(r.Year, filter.Years) {
		r.addRejection("year not matching")
		return false
	}

	if filter.MatchCategories != "" && !checkFilterStrings(r.Category, filter.MatchCategories) {
		r.addRejection("category not matching")
		return false
	}

	if filter.ExceptCategories != "" && checkFilterStrings(r.Category, filter.ExceptCategories) {
		r.addRejection("unwanted category")
		return false
	}

	if len(filter.MatchReleaseTypes) > 0 && !checkFilterSlice(r.Category, filter.MatchReleaseTypes) {
		r.addRejection("release type not matching")
		return false
	}

	if (filter.MinSize != "" || filter.MaxSize != "") && !r.CheckSizeFilter(filter.MinSize, filter.MaxSize) {
		return false
	}

	if filter.Tags != "" && !checkFilterTags(r.Tags, filter.Tags) {
		r.addRejection("tags not matching")
		return false
	}

	if filter.ExceptTags != "" && checkFilterTags(r.Tags, filter.ExceptTags) {
		r.addRejection("unwanted tags")
		return false
	}

	if len(filter.Artists) > 0 && !checkFilterStrings(r.TorrentName, filter.Artists) {
		r.addRejection("artists not matching")
		return false
	}

	if len(filter.Albums) > 0 && !checkFilterStrings(r.TorrentName, filter.Albums) {
		r.addRejection("albums not matching")
		return false
	}

	// Perfect flac requires Cue, Log, Log Score 100, FLAC and 24bit Lossless
	if filter.PerfectFlac {
		if !r.HasLog || !r.HasCue || r.LogScore != 100 || r.Format != "FLAC" && !checkFilterSlice(r.Quality, []string{"Lossless", "24bit Lossless"}) {
			r.addRejection("wanted: log")
			return false
		}
	}

	if len(filter.Formats) > 0 && !checkFilterSlice(r.Format, filter.Formats) {
		r.addRejection("formats not matching")
		return false
	}

	if len(filter.Quality) > 0 && !checkFilterSlice(r.Quality, filter.Quality) {
		r.addRejection("formats not matching")
		return false
	}

	if len(filter.Media) > 0 && !checkFilterSource(r.Source, filter.Media) {
		r.addRejection("source not matching")
		return false
	}

	if filter.Log && r.HasLog != filter.Log {
		r.addRejection("wanted: log")
		return false
	}

	if filter.Log && filter.LogScore != 0 && r.LogScore != filter.LogScore {
		r.addRejection("wanted: log score")
		return false
	}

	if filter.Cue && r.HasCue != filter.Cue {
		r.addRejection("wanted: cue")
		return false
	}

	return true
}

// CheckSizeFilter additional size check
// for indexers that doesn't announce size, like some gazelle based
// set flag r.AdditionalSizeCheckRequired if there's a size in the filter, otherwise go a head
// implement API for ptp,btn,ggn to check for size if needed
// for others pull down torrent and do check
func (r *Release) CheckSizeFilter(minSize string, maxSize string) bool {

	if r.Size == 0 {
		r.AdditionalSizeCheckRequired = true

		return true
	} else {
		r.AdditionalSizeCheckRequired = false
	}

	// if r.Size parse filter to bytes and compare
	// handle both min and max
	if minSize != "" {
		// string to bytes
		minSizeBytes, err := humanize.ParseBytes(minSize)
		if err != nil {
			// log could not parse into bytes
		}

		if r.Size <= minSizeBytes {
			r.addRejection("size: smaller than min size")
			return false
		}

	}

	if maxSize != "" {
		// string to bytes
		maxSizeBytes, err := humanize.ParseBytes(maxSize)
		if err != nil {
			// log could not parse into bytes
		}

		if r.Size >= maxSizeBytes {
			r.addRejection("size: larger than max size")
			return false
		}
	}

	return true
}

// MapVars better name
func (r *Release) MapVars(def IndexerDefinition, varMap map[string]string) error {

	if torrentName, err := getStringMapValue(varMap, "torrentName"); err != nil {
		return errors.Wrap(err, "failed parsing required field")
	} else {
		r.TorrentName = html.UnescapeString(torrentName)
	}

	if torrentID, err := getStringMapValue(varMap, "torrentId"); err == nil {
		r.TorrentID = torrentID
	}

	if category, err := getStringMapValue(varMap, "category"); err == nil {
		r.Category = category
	}

	if freeleech, err := getStringMapValue(varMap, "freeleech"); err == nil {
		r.Freeleech = strings.EqualFold(freeleech, "freeleech") || strings.EqualFold(freeleech, "yes") || strings.EqualFold(freeleech, "1")
	}

	if freeleechPercent, err := getStringMapValue(varMap, "freeleechPercent"); err == nil {
		// remove % and trim spaces
		freeleechPercent = strings.Replace(freeleechPercent, "%", "", -1)
		freeleechPercent = strings.Trim(freeleechPercent, " ")

		freeleechPercentInt, err := strconv.Atoi(freeleechPercent)
		if err != nil {
			//log.Debug().Msgf("bad freeleechPercent var: %v", year)
		}

		r.FreeleechPercent = freeleechPercentInt
	}

	if uploader, err := getStringMapValue(varMap, "uploader"); err == nil {
		r.Uploader = uploader
	}

	if torrentSize, err := getStringMapValue(varMap, "torrentSize"); err == nil {
		// handling for indexer who doesn't explicitly set which size unit is used like (AR)
		if def.Parse.ForceSizeUnit != "" {
			torrentSize = fmt.Sprintf("%v %v", torrentSize, def.Parse.ForceSizeUnit)
		}

		size, err := humanize.ParseBytes(torrentSize)
		if err != nil {
			// log could not parse into bytes
		}
		r.Size = size
	}

	if scene, err := getStringMapValue(varMap, "scene"); err == nil {
		r.IsScene = strings.EqualFold(scene, "true") || strings.EqualFold(scene, "yes")
	}

	if yearVal, err := getStringMapValue(varMap, "year"); err == nil {
		year, err := strconv.Atoi(yearVal)
		if err != nil {
			//log.Debug().Msgf("bad year var: %v", year)
		}
		r.Year = year
	}

	if tags, err := getStringMapValue(varMap, "tags"); err == nil {
		tagArr := strings.Split(strings.ReplaceAll(tags, " ", ""), ",")
		r.Tags = tagArr
	}

	// handle releaseTags. Most of them are redundant but some are useful
	if releaseTags, err := getStringMapValue(varMap, "releaseTags"); err == nil {
		r.ReleaseTags = releaseTags
	}

	if resolution, err := getStringMapValue(varMap, "resolution"); err == nil {
		r.Resolution = resolution
	}

	return nil
}

func checkFilterSlice(name string, filterList []string) bool {
	name = strings.ToLower(name)

	for _, filter := range filterList {
		filter = strings.ToLower(filter)
		filter = strings.Trim(filter, " ")
		// check if line contains * or ?, if so try wildcard match, otherwise try substring match
		a := strings.ContainsAny(filter, "?|*")
		if a {
			match := wildcard.Match(filter, name)
			if match {
				return true
			}
		} else {
			b := strings.Contains(name, filter)
			if b {
				return true
			}
		}
	}

	return false
}

func checkFilterStrings(name string, filterList string) bool {
	filterSplit := strings.Split(filterList, ",")
	name = strings.ToLower(name)

	for _, s := range filterSplit {
		s = strings.ToLower(s)
		s = strings.Trim(s, " ")
		// check if line contains * or ?, if so try wildcard match, otherwise try substring match
		a := strings.ContainsAny(s, "?|*")
		if a {
			match := wildcard.Match(s, name)
			if match {
				return true
			}
		} else {
			b := strings.Contains(name, s)
			if b {
				return true
			}
		}

	}

	return false
}

// checkMultipleFilterStrings check against multiple vars of unknown length
func checkMultipleFilterStrings(filterList string, vars ...string) bool {
	filterSplit := strings.Split(filterList, ",")

	for _, name := range vars {
		name = strings.ToLower(name)

		for _, s := range filterSplit {
			s = strings.ToLower(s)
			s = strings.Trim(s, " ")
			// check if line contains * or ?, if so try wildcard match, otherwise try substring match
			a := strings.ContainsAny(s, "?|*")
			if a {
				match := wildcard.Match(s, name)
				if match {
					return true
				}
			} else {
				b := strings.Contains(name, s)
				if b {
					return true
				}
			}
		}
	}

	return false
}

// checkFilterIntStrings "1,2,3-20"
func checkFilterIntStrings(value int, filterList string) bool {
	filters := strings.Split(filterList, ",")

	for _, s := range filters {
		s = strings.Replace(s, "%", "", -1)
		s = strings.Trim(s, " ")

		if strings.Contains(s, "-") {
			minMax := strings.Split(s, "-")

			// to int
			min, err := strconv.ParseInt(minMax[0], 10, 32)
			if err != nil {
				return false
			}

			max, err := strconv.ParseInt(minMax[1], 10, 32)
			if err != nil {
				return false
			}

			if min > max {
				// handle error
				return false
			} else {
				// if announcePercent is greater than min and less than max return true
				if value >= int(min) && value <= int(max) {
					return true
				}
			}
		}

		filterInt, err := strconv.ParseInt(s, 10, 32)
		if err != nil {
			return false
		}

		if int(filterInt) == value {
			return true
		}
	}

	return false
}

func checkMultipleFilterGroups(filterList string, vars ...string) bool {
	filterSplit := strings.Split(filterList, ",")

	for _, name := range vars {
		name = strings.ToLower(name)

		for _, s := range filterSplit {
			s = strings.ToLower(strings.Trim(s, " "))
			// check if line contains * or ?, if so try wildcard match, otherwise try substring match
			a := strings.ContainsAny(s, "?|*")
			if a {
				match := wildcard.Match(s, name)
				if match {
					return true
				}
			} else {
				split := SplitAny(name, " .-")
				for _, c := range split {
					if c == s {
						return true
					}
				}
				continue
			}
		}
	}

	return false
}

func checkMultipleFilterHDR(filterList []string, vars ...string) bool {
	for _, name := range vars {
		name = strings.ToLower(name)

		for _, s := range filterList {
			s = strings.ToLower(strings.Trim(s, " "))
			// check if line contains * or ?, if so try wildcard match, otherwise try substring match
			a := strings.ContainsAny(s, "?|*")
			if a {
				match := wildcard.Match(s, name)
				if match {
					return true
				}
			} else {
				split := SplitAny(name, " .-")
				for _, c := range split {
					if c == s {
						return true
					}
				}
				continue
			}
		}
	}

	return false
}

func checkFilterSource(name string, filterList []string) bool {
	// remove dash (-) in blu-ray web-dl and make lowercase
	name = strings.ToLower(strings.ReplaceAll(name, "-", ""))

	for _, filter := range filterList {
		// remove dash (-) in blu-ray web-dl, trim spaces and make lowercase
		filter = strings.ToLower(strings.Trim(strings.ReplaceAll(filter, "-", ""), " "))

		b := strings.Contains(name, filter)
		if b {
			return true
		}
	}

	return false
}

func checkFilterTags(tags []string, filter string) bool {
	filterTags := strings.Split(filter, ",")

	for _, tag := range tags {
		tag = strings.ToLower(tag)

		for _, filter := range filterTags {
			filter = strings.ToLower(filter)
			filter = strings.Trim(filter, " ")
			// check if line contains * or ?, if so try wildcard match, otherwise try substring match
			a := strings.ContainsAny(filter, "?|*")
			if a {
				match := wildcard.Match(filter, tag)
				if match {
					return true
				}
			} else {
				b := strings.Contains(tag, filter)
				if b {
					return true
				}
			}
		}
	}

	return false
}

func checkFreeleechPercent(announcePercent int, filterPercent string) bool {
	filters := strings.Split(filterPercent, ",")

	// remove % and trim spaces
	//announcePercent = strings.Replace(announcePercent, "%", "", -1)
	//announcePercent = strings.Trim(announcePercent, " ")

	//announcePercentInt, err := strconv.ParseInt(announcePercent, 10, 32)
	//if err != nil {
	//	return false
	//}

	for _, s := range filters {
		s = strings.Replace(s, "%", "", -1)
		s = strings.Trim(s, " ")

		if strings.Contains(s, "-") {
			minMax := strings.Split(s, "-")

			// to int
			min, err := strconv.ParseInt(minMax[0], 10, 32)
			if err != nil {
				return false
			}

			max, err := strconv.ParseInt(minMax[1], 10, 32)
			if err != nil {
				return false
			}

			if min > max {
				// handle error
				return false
			} else {
				// if announcePercent is greater than min and less than max return true
				if announcePercent >= int(min) && announcePercent <= int(max) {
					return true
				}
			}
		}

		filterPercentInt, err := strconv.ParseInt(s, 10, 32)
		if err != nil {
			return false
		}

		if int(filterPercentInt) == announcePercent {
			return true
		}
	}

	return false
}

func getStringMapValue(stringMap map[string]string, key string) (string, error) {
	lowerKey := strings.ToLower(key)

	// case-sensitive match
	//if caseSensitive {
	//	v, ok := stringMap[key]
	//	if !ok {
	//		return "", fmt.Errorf("key was not found in map: %q", key)
	//	}
	//
	//	return v, nil
	//}

	// case-insensitive match
	for k, v := range stringMap {
		if strings.ToLower(k) == lowerKey {
			return v, nil
		}
	}

	return "", fmt.Errorf("key was not found in map: %q", lowerKey)
}

func getFirstStringMapValue(stringMap map[string]string, keys []string) (string, error) {
	for _, k := range keys {
		if val, err := getStringMapValue(stringMap, k); err == nil {
			return val, nil
		}
	}

	return "", fmt.Errorf("key were not found in map: %q", strings.Join(keys, ", "))
}

func findLast(input string, pattern string) (string, error) {
	matched := make([]string, 0)
	//for _, s := range arr {

	rxp, err := regexp.Compile(pattern)
	if err != nil {
		return "", err
		//return errors.Wrapf(err, "invalid regex: %s", value)
	}

	matches := rxp.FindStringSubmatch(input)
	if matches != nil {
		// first value is the match, second value is the text
		if len(matches) >= 1 {
			last := matches[len(matches)-1]

			// add to temp slice
			matched = append(matched, last)
		}
	}

	//}

	// check if multiple values in temp slice, if so get the last one
	if len(matched) >= 1 {
		last := matched[len(matched)-1]

		return last, nil
	}

	return "", nil
}

func findLastBool(input string, pattern string) (bool, error) {
	matched := make([]string, 0)

	rxp, err := regexp.Compile(pattern)
	if err != nil {
		return false, err
	}

	matches := rxp.FindStringSubmatch(input)
	if matches != nil {
		// first value is the match, second value is the text
		if len(matches) >= 1 {
			last := matches[len(matches)-1]

			// add to temp slice
			matched = append(matched, last)
		}
	}

	//}

	// check if multiple values in temp slice, if so get the last one
	if len(matched) >= 1 {
		//last := matched[len(matched)-1]

		return true, nil
	}

	return false, nil
}

func findLastInt(input string, pattern string) (int, error) {
	matched := make([]string, 0)
	//for _, s := range arr {

	rxp, err := regexp.Compile(pattern)
	if err != nil {
		return 0, err
		//return errors.Wrapf(err, "invalid regex: %s", value)
	}

	matches := rxp.FindStringSubmatch(input)
	if matches != nil {
		// first value is the match, second value is the text
		if len(matches) >= 1 {
			last := matches[len(matches)-1]

			// add to temp slice
			matched = append(matched, last)
		}
	}

	//}

	// check if multiple values in temp slice, if so get the last one
	if len(matched) >= 1 {
		last := matched[len(matched)-1]

		i, err := strconv.Atoi(last)
		if err != nil {
			return 0, err
		}

		return i, nil
	}

	return 0, nil
}

func SplitAny(s string, seps string) []string {
	splitter := func(r rune) bool {
		return strings.ContainsRune(seps, r)
	}
	return strings.FieldsFunc(s, splitter)
}

//func Splitter(s string, splits string) []string {
//	m := make(map[rune]int)
//	for _, r := range splits {
//		m[r] = 1
//	}
//
//	splitter := func(r rune) bool {
//		return m[r] == 1
//	}
//
//	return strings.FieldsFunc(s, splitter)
//}
//
//func canonicalizeString(s string) []string {
//	//a := strings.FieldsFunc(s, split)
//	a := Splitter(s, " .")
//
//	return a
//}

func cleanReleaseName(input string) string {
	// Make a Regex to say we only want letters and numbers
	reg, err := regexp.Compile(`[\x00-\x1F\x2D\x2E\x5F\x7F]`)
	if err != nil {
		return ""
	}
	processedString := reg.ReplaceAllString(input, " ")

	return processedString
}

type DownloadTorrentFileResponse struct {
	MetaInfo    *metainfo.MetaInfo
	TmpFileName string
}

type ReleaseStats struct {
	TotalCount          int64 `json:"total_count"`
	FilteredCount       int64 `json:"filtered_count"`
	FilterRejectedCount int64 `json:"filter_rejected_count"`
	PushApprovedCount   int64 `json:"push_approved_count"`
	PushRejectedCount   int64 `json:"push_rejected_count"`
}

type ReleasePushStatus string

const (
	ReleasePushStatusApproved ReleasePushStatus = "PUSH_APPROVED"
	ReleasePushStatusRejected ReleasePushStatus = "PUSH_REJECTED"
	ReleasePushStatusErr      ReleasePushStatus = "PUSH_ERROR"
	ReleasePushStatusPending  ReleasePushStatus = "PENDING" // Initial status
)

type ReleaseFilterStatus string

const (
	ReleaseStatusFilterApproved ReleaseFilterStatus = "FILTER_APPROVED"
	ReleaseStatusFilterRejected ReleaseFilterStatus = "FILTER_REJECTED"
	ReleaseStatusFilterPending  ReleaseFilterStatus = "PENDING"
)

type ReleaseProtocol string

const (
	ReleaseProtocolTorrent ReleaseProtocol = "torrent"
)

type ReleaseImplementation string

const (
	ReleaseImplementationIRC ReleaseImplementation = "IRC"
)

type ReleaseQueryParams struct {
	Limit   uint64
	Offset  uint64
	Cursor  uint64
	Sort    map[string]string
	Filters struct {
		Indexers   []string
		PushStatus string
	}
	Search string
}