Feature: Save releases (#36)

* chore: tidy deps

* refactor: database migration

* refactor: store release

* refactor: save release

* chore: add packages

* feat(web): show stats and recent releases

* refactor: simply filter struct

* feat: add eventbus

* chore: cleanup logging

* chore: update packages
This commit is contained in:
Ludvig Lundgren 2021-11-24 23:18:12 +01:00 committed by GitHub
parent d22dd2fe84
commit 7177e48c02
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 5859 additions and 3328 deletions

View file

@ -1,51 +0,0 @@
package domain
type Announce struct {
ReleaseType string
Freeleech bool
FreeleechPercent string
Origin string
ReleaseGroup string
Category string
TorrentName string
Uploader string
TorrentSize string
PreTime string
TorrentUrl string
TorrentUrlSSL string
Year int
Name1 string // artist, show, movie
Name2 string // album
Season int
Episode int
Resolution string
Source string
Encoder string
Container string
Format string
Bitrate string
Media string
Tags string
Scene bool
Log string
LogScore string
Cue bool
Line string
OrigLine string
Site string
HttpHeaders string
Filter *Filter
}
//type Announce struct {
// Channel string
// Announcer string
// Message string
// CreatedAt time.Time
//}
//
type AnnounceRepo interface {
Store(announce Announce) error
}

View file

@ -20,71 +20,49 @@ type FilterRepo interface {
}
type Filter struct {
ID int `json:"id"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
FilterGeneral
FilterP2P
FilterTVMovies
FilterMusic
FilterAdvanced
Actions []Action `json:"actions"`
Indexers []Indexer `json:"indexers"`
}
type FilterGeneral struct {
MinSize string `json:"min_size"`
MaxSize string `json:"max_size"`
Delay int `json:"delay"`
}
type FilterP2P struct {
MatchReleases string `json:"match_releases"`
ExceptReleases string `json:"except_releases"`
UseRegex bool `json:"use_regex"`
MatchReleaseGroups string `json:"match_release_groups"`
ExceptReleaseGroups string `json:"except_release_groups"`
Scene bool `json:"scene"`
Origins string `json:"origins"`
Freeleech bool `json:"freeleech"`
FreeleechPercent string `json:"freeleech_percent"`
}
type FilterTVMovies struct {
Shows string `json:"shows"`
Seasons string `json:"seasons"`
Episodes string `json:"episodes"`
Resolutions []string `json:"resolutions"` // SD, 480i, 480p, 576p, 720p, 810p, 1080i, 1080p.
Codecs []string `json:"codecs"` // XviD, DivX, x264, h.264 (or h264), mpeg2 (or mpeg-2), VC-1 (or VC1), WMV, Remux, h.264 Remux (or h264 Remux), VC-1 Remux (or VC1 Remux).
Sources []string `json:"sources"` // DSR, PDTV, HDTV, HR.PDTV, HR.HDTV, DVDRip, DVDScr, BDr, BD5, BD9, BDRip, BRRip, DVDR, MDVDR, HDDVD, HDDVDRip, BluRay, WEB-DL, TVRip, CAM, R5, TELESYNC, TS, TELECINE, TC. TELESYNC and TS are synonyms (you don't need both). Same for TELECINE and TC
Containers []string `json:"containers"`
Years string `json:"years"`
}
type FilterMusic struct {
Artists string `json:"artists"`
Albums string `json:"albums"`
MatchReleaseTypes string `json:"match_release_types"` // Album,Single,EP
ExceptReleaseTypes string `json:"except_release_types"`
Formats []string `json:"formats"` // MP3, FLAC, Ogg, AAC, AC3, DTS
Bitrates []string `json:"bitrates"` // 192, 320, APS (VBR), V2 (VBR), V1 (VBR), APX (VBR), V0 (VBR), q8.x (VBR), Lossless, 24bit Lossless, Other
Media []string `json:"media"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other
Cue bool `json:"cue"`
Log bool `json:"log"`
LogScores string `json:"log_scores"`
}
type FilterAdvanced struct {
MatchCategories string `json:"match_categories"`
ExceptCategories string `json:"except_categories"`
MatchUploaders string `json:"match_uploaders"`
ExceptUploaders string `json:"except_uploaders"`
Tags string `json:"tags"`
ExceptTags string `json:"except_tags"`
TagsAny string `json:"tags_any"`
ExceptTagsAny string `json:"except_tags_any"`
ID int `json:"id"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
MinSize string `json:"min_size"`
MaxSize string `json:"max_size"`
Delay int `json:"delay"`
MatchReleases string `json:"match_releases"`
ExceptReleases string `json:"except_releases"`
UseRegex bool `json:"use_regex"`
MatchReleaseGroups string `json:"match_release_groups"`
ExceptReleaseGroups string `json:"except_release_groups"`
Scene bool `json:"scene"`
Origins string `json:"origins"`
Freeleech bool `json:"freeleech"`
FreeleechPercent string `json:"freeleech_percent"`
Shows string `json:"shows"`
Seasons string `json:"seasons"`
Episodes string `json:"episodes"`
Resolutions []string `json:"resolutions"` // SD, 480i, 480p, 576p, 720p, 810p, 1080i, 1080p.
Codecs []string `json:"codecs"` // XviD, DivX, x264, h.264 (or h264), mpeg2 (or mpeg-2), VC-1 (or VC1), WMV, Remux, h.264 Remux (or h264 Remux), VC-1 Remux (or VC1 Remux).
Sources []string `json:"sources"` // DSR, PDTV, HDTV, HR.PDTV, HR.HDTV, DVDRip, DVDScr, BDr, BD5, BD9, BDRip, BRRip, DVDR, MDVDR, HDDVD, HDDVDRip, BluRay, WEB-DL, TVRip, CAM, R5, TELESYNC, TS, TELECINE, TC. TELESYNC and TS are synonyms (you don't need both). Same for TELECINE and TC
Containers []string `json:"containers"`
Years string `json:"years"`
Artists string `json:"artists"`
Albums string `json:"albums"`
MatchReleaseTypes string `json:"match_release_types"` // Album,Single,EP
ExceptReleaseTypes string `json:"except_release_types"`
Formats []string `json:"formats"` // MP3, FLAC, Ogg, AAC, AC3, DTS
Bitrates []string `json:"bitrates"` // 192, 320, APS (VBR), V2 (VBR), V1 (VBR), APX (VBR), V0 (VBR), q8.x (VBR), Lossless, 24bit Lossless, Other
Media []string `json:"media"` // CD, DVD, Vinyl, Soundboard, SACD, DAT, Cassette, WEB, Other
Cue bool `json:"cue"`
Log bool `json:"log"`
LogScores string `json:"log_scores"`
MatchCategories string `json:"match_categories"`
ExceptCategories string `json:"except_categories"`
MatchUploaders string `json:"match_uploaders"`
ExceptUploaders string `json:"except_uploaders"`
Tags string `json:"tags"`
ExceptTags string `json:"except_tags"`
TagsAny string `json:"tags_any"`
ExceptTagsAny string `json:"except_tags_any"`
Actions []Action `json:"actions"`
Indexers []Indexer `json:"indexers"`
}

View file

@ -35,7 +35,6 @@ type IrcNetwork struct {
}
type IrcRepo interface {
Store(announce Announce) error
StoreNetwork(network *IrcNetwork) error
StoreChannel(networkID int64, channel *IrcChannel) error
ListNetworks(ctx context.Context) ([]IrcNetwork, error)

920
internal/domain/release.go Normal file
View file

@ -0,0 +1,920 @@
package domain
import (
"context"
"fmt"
"html"
"regexp"
"strconv"
"strings"
"time"
"github.com/autobrr/autobrr/pkg/wildcard"
"github.com/dustin/go-humanize"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
type ReleaseRepo interface {
Store(ctx context.Context, release *Release) (*Release, error)
Find(ctx context.Context, params QueryParams) (res []Release, nextCursor int64, err error)
Stats(ctx context.Context) (*ReleaseStats, error)
UpdatePushStatus(ctx context.Context, id int64, status ReleasePushStatus) error
UpdatePushStatusRejected(ctx context.Context, id int64, rejections string) error
}
type Release struct {
ID int64 `json:"id"`
FilterStatus ReleaseFilterStatus `json:"filter_status"`
PushStatus ReleasePushStatus `json:"push_status"`
Rejections []string `json:"rejections"`
Indexer string `json:"indexer"`
FilterName string `json:"filter"`
Protocol ReleaseProtocol `json:"protocol"`
Implementation ReleaseImplementation `json:"implementation"` // irc, rss, api
Timestamp time.Time `json:"timestamp"`
GroupID string `json:"group_id"`
TorrentID string `json:"torrent_id"`
TorrentURL string `json:"-"`
TorrentName string `json:"torrent_name"` // full release name
Size uint64 `json:"size"`
Raw string `json:"raw"` // Raw release
Title string `json:"title"` // Parsed title
Category string `json:"category"`
Season int `json:"season"`
Episode int `json:"episode"`
Year int `json:"year"`
Resolution string `json:"resolution"`
Source string `json:"source"` // CD, DVD, Vinyl, DAT, Cassette, WEB, Other
Codec string `json:"codec"`
Container string `json:"container"`
HDR string `json:"hdr"`
Audio string `json:"audio"`
Group string `json:"group"`
Region string `json:"region"`
Language string `json:"language"`
Edition string `json:"edition"` // Extended, directors cut
Unrated bool `json:"unrated"`
Hybrid bool `json:"hybrid"`
Proper bool `json:"proper"`
Repack bool `json:"repack"`
Website string `json:"website"`
ThreeD bool `json:"-"`
Artists []string `json:"artists"`
Type string `json:"type"` // Album,Single,EP
Format string `json:"format"` // music only
Bitrate string `json:"bitrate"` // bitrate
LogScore int `json:"log_score"`
HasLog bool `json:"has_log"`
HasCue bool `json:"has_cue"`
IsScene bool `json:"is_scene"`
Origin string `json:"origin"` // P2P, Internal
Tags []string `json:"tags"`
Freeleech bool `json:"freeleech"`
FreeleechPercent int `json:"freeleech_percent"`
Uploader string `json:"uploader"`
PreTime string `json:"pre_time"`
AdditionalSizeCheckRequired bool `json:"-"`
FilterID int `json:"-"`
Filter *Filter `json:"-"`
}
func NewRelease(indexer string, line string) (*Release, error) {
r := &Release{
Indexer: indexer,
Raw: line,
FilterStatus: ReleaseStatusFilterPending,
PushStatus: ReleasePushStatusPending,
Rejections: []string{},
Protocol: ReleaseProtocolTorrent,
Implementation: ReleaseImplementationIRC,
Timestamp: time.Now(),
Artists: []string{},
Tags: []string{},
}
return r, nil
}
func (r *Release) Parse() error {
var err error
err = r.extractYear()
err = r.extractSeason()
err = r.extractEpisode()
err = r.extractResolution()
err = r.extractSource()
err = r.extractCodec()
err = r.extractContainer()
err = r.extractHDR()
err = r.extractAudio()
err = r.extractGroup()
err = r.extractRegion()
err = r.extractLanguage()
err = r.extractEdition()
err = r.extractUnrated()
err = r.extractHybrid()
err = r.extractProper()
err = r.extractRepack()
err = r.extractWebsite()
if err != nil {
log.Trace().Msgf("could not parse release: %v", r.TorrentName)
return err
}
return nil
}
func (r *Release) extractYear() error {
y, err := findLastInt(r.TorrentName, `\b(((?:19[0-9]|20[0-9])[0-9]))\b`)
if err != nil {
return err
}
r.Year = y
return nil
}
func (r *Release) extractSeason() error {
s, err := findLastInt(r.TorrentName, `(?:S|Season\s*)(\d{1,3})`)
if err != nil {
return err
}
r.Season = s
return nil
}
func (r *Release) extractEpisode() error {
e, err := findLastInt(r.TorrentName, `(?i)[ex]([0-9]{2})(?:[^0-9]|$)`)
if err != nil {
return err
}
r.Episode = e
return nil
}
func (r *Release) extractResolution() error {
v, err := findLast(r.TorrentName, `\b(([0-9]{3,4}p|i))\b`)
if err != nil {
return err
}
r.Resolution = v
return nil
}
func (r *Release) extractSource() error {
v, err := findLast(r.TorrentName, `(?i)\b(((?:PPV\.)?[HP]DTV|(?:HD)?CAM|B[DR]Rip|(?:HD-?)?TS|(?:PPV )?WEB-?DL(?: DVDRip)?|HDRip|DVDRip|DVDRIP|CamRip|WEB|W[EB]BRip|Blu-?Ray|DvDScr|telesync|CD|DVD|Vinyl|DAT|Cassette))\b`)
if err != nil {
return err
}
r.Source = v
return nil
}
func (r *Release) extractCodec() error {
v, err := findLast(r.TorrentName, `(?i)\b(HEVC|[hx]\.?26[45]|xvid|divx|AVC|MPEG-?2|AV1|VC-?1|VP9|WebP)\b`)
if err != nil {
return err
}
r.Codec = v
return nil
}
func (r *Release) extractContainer() error {
v, err := findLast(r.TorrentName, `(?i)\b(AVI|MPG|MKV|MP4|VOB|m2ts|ISO|IMG)\b`)
if err != nil {
return err
}
r.Container = v
return nil
}
func (r *Release) extractHDR() error {
v, err := findLast(r.TorrentName, `(?i)(HDR10\+|HDR10|DoVi HDR|DV HDR|HDR|DV|DoVi|Dolby Vision \+ HDR10|Dolby Vision)`)
if err != nil {
return err
}
r.HDR = v
return nil
}
func (r *Release) extractAudio() error {
v, err := findLast(r.TorrentName, `(?i)(MP3|FLAC[\. ][1-7][\. ][0-2]|FLAC|Opus|DD-EX|DDP[\. ]?[124567][\. ][012] Atmos|DDP[\. ]?[124567][\. ][012]|DDP|DD[1-7][\. ][0-2]|Dual[\- ]Audio|LiNE|PCM|Dolby TrueHD [0-9][\. ][0-4]|TrueHD [0-9][\. ][0-4] Atmos|TrueHD [0-9][\. ][0-4]|DTS X|DTS-HD MA [0-9][\. ][0-4]|DTS-HD MA|DTS-ES|DTS [1-7][\. ][0-2]|DTS|DD|DD[12][\. ]0|Dolby Atmos|TrueHD ATMOS|TrueHD|Atmos|Dolby Digital Plus|Dolby Digital Audio|Dolby Digital|AAC[.-]LC|AAC (?:\.?[1-7]\.[0-2])?|AAC|eac3|AC3(?:\.5\.1)?)`)
if err != nil {
return err
}
r.Audio = v
return nil
}
func (r *Release) extractGroup() error {
// try first for wierd anime group names [group] show name, or in brackets at the end
group := ""
g, err := findLast(r.TorrentName, `\[(.*?)\]`)
if err != nil {
return err
}
group = g
if group == "" {
g2, err := findLast(r.TorrentName, `(- ?([^-]+(?:-={[^-]+-?$)?))$`)
if err != nil {
return err
}
group = g2
}
r.Group = group
return nil
}
func (r *Release) extractRegion() error {
v, err := findLast(r.TorrentName, `(?i)\b(R([0-9]))\b`)
if err != nil {
return err
}
r.Region = v
return nil
}
func (r *Release) extractLanguage() error {
v, err := findLast(r.TorrentName, `(?i)\b((DK|DKSUBS|DANiSH|DUTCH|NL|NLSUBBED|ENG|FI|FLEMiSH|FiNNiSH|DE|FRENCH|GERMAN|HE|HEBREW|HebSub|HiNDi|iCELANDiC|KOR|MULTi|MULTiSUBS|NORWEGiAN|NO|NORDiC|PL|PO|POLiSH|PLDUB|RO|ROMANiAN|RUS|SPANiSH|SE|SWEDiSH|SWESUB||))\b`)
if err != nil {
return err
}
r.Language = v
return nil
}
func (r *Release) extractEdition() error {
v, err := findLast(r.TorrentName, `(?i)\b((?:DIRECTOR'?S|EXTENDED|INTERNATIONAL|THEATRICAL|ORIGINAL|FINAL|BOOTLEG)(?:.CUT)?)\b`)
if err != nil {
return err
}
r.Edition = v
return nil
}
func (r *Release) extractUnrated() error {
v, err := findLastBool(r.TorrentName, `(?i)\b((UNRATED))\b`)
if err != nil {
return err
}
r.Unrated = v
return nil
}
func (r *Release) extractHybrid() error {
v, err := findLastBool(r.TorrentName, `(?i)\b((HYBRID))\b`)
if err != nil {
return err
}
r.Hybrid = v
return nil
}
func (r *Release) extractProper() error {
v, err := findLastBool(r.TorrentName, `(?i)\b((PROPER))\b`)
if err != nil {
return err
}
r.Proper = v
return nil
}
func (r *Release) extractRepack() error {
v, err := findLastBool(r.TorrentName, `(?i)\b((REPACK))\b`)
if err != nil {
return err
}
r.Repack = v
return nil
}
func (r *Release) extractWebsite() error {
// Start with the basic most common ones
v, err := findLast(r.TorrentName, `(?i)\b((AMBC|AS|AMZN|AMC|ANPL|ATVP|iP|CORE|BCORE|CMOR|CN|CBC|CBS|CMAX|CNBC|CC|CRIT|CR|CSPN|CW|DAZN|DCU|DISC|DSCP|DSNY|DSNP|DPLY|ESPN|FOX|FUNI|PLAY|HBO|HMAX|HIST|HS|HOTSTAR|HULU|iT|MNBC|MTV|NATG|NBC|NF|NICK|NRK|PMNT|PMNP|PCOK|PBS|PBSK|PSN|QIBI|SBS|SHO|STAN|STZ|SVT|SYFY|TLC|TRVL|TUBI|TV3|TV4|TVL|VH1|VICE|VMEO|UFC|USAN|VIAP|VIAPLAY|VL|WWEN|XBOX|YHOO|YT|RED))\b`)
if err != nil {
return err
}
r.Website = v
return nil
}
func (r *Release) addRejection(reason string) {
r.Rejections = append(r.Rejections, reason)
}
// ResetRejections reset rejections between filter checks
func (r *Release) resetRejections() {
r.Rejections = []string{}
}
func (r *Release) CheckFilter(filter Filter) bool {
// reset rejections first to clean previous checks
r.resetRejections()
if !filter.Enabled {
return false
}
// FIXME what if someone explicitly doesnt want scene, or toggles in filter. Make enum? 0,1,2? Yes, No, Dont care
if filter.Scene && r.IsScene != filter.Scene {
r.addRejection("wanted: scene")
return false
}
if filter.Freeleech && r.Freeleech != filter.Freeleech {
r.addRejection("wanted: freeleech")
return false
}
if filter.FreeleechPercent != "" && !checkFreeleechPercent(r.FreeleechPercent, filter.FreeleechPercent) {
r.addRejection("freeleech percent not matching")
return false
}
// check against title when parsed correctly
if filter.Shows != "" && !checkFilterStrings(r.TorrentName, filter.Shows) {
r.addRejection("shows not matching")
return false
}
if filter.Seasons != "" && !checkFilterIntStrings(r.Season, filter.Seasons) {
r.addRejection("season not matching")
return false
}
if filter.Episodes != "" && !checkFilterIntStrings(r.Episode, filter.Episodes) {
r.addRejection("episode not matching")
return false
}
// matchRelease
// TODO allow to match against regex
if filter.MatchReleases != "" && !checkFilterStrings(r.TorrentName, filter.MatchReleases) {
r.addRejection("match release not matching")
return false
}
if filter.ExceptReleases != "" && checkFilterStrings(r.TorrentName, filter.ExceptReleases) {
r.addRejection("except_releases: unwanted release")
return false
}
if filter.MatchReleaseGroups != "" && !checkFilterStrings(r.Group, filter.MatchReleaseGroups) {
r.addRejection("release groups not matching")
return false
}
if filter.ExceptReleaseGroups != "" && checkFilterStrings(r.Group, filter.ExceptReleaseGroups) {
r.addRejection("unwanted release group")
return false
}
if filter.MatchUploaders != "" && !checkFilterStrings(r.Uploader, filter.MatchUploaders) {
r.addRejection("uploaders not matching")
return false
}
if filter.ExceptUploaders != "" && checkFilterStrings(r.Uploader, filter.ExceptUploaders) {
r.addRejection("unwanted uploaders")
return false
}
if len(filter.Resolutions) > 0 && !checkFilterSlice(r.Resolution, filter.Resolutions) {
r.addRejection("resolution not matching")
return false
}
if len(filter.Codecs) > 0 && !checkFilterSlice(r.Codec, filter.Codecs) {
r.addRejection("codec not matching")
return false
}
if len(filter.Sources) > 0 && !checkFilterSlice(r.Source, filter.Sources) {
r.addRejection("source not matching")
return false
}
if len(filter.Containers) > 0 && !checkFilterSlice(r.Container, filter.Containers) {
r.addRejection("container not matching")
return false
}
if filter.Years != "" && !checkFilterIntStrings(r.Year, filter.Years) {
r.addRejection("year not matching")
return false
}
if filter.MatchCategories != "" && !checkFilterStrings(r.Category, filter.MatchCategories) {
r.addRejection("category not matching")
return false
}
if filter.ExceptCategories != "" && checkFilterStrings(r.Category, filter.ExceptCategories) {
r.addRejection("unwanted category")
return false
}
if (filter.MinSize != "" || filter.MaxSize != "") && !r.CheckSizeFilter(filter.MinSize, filter.MaxSize) {
return false
}
//if filter.Tags != "" && !checkFilterStrings(r.Tags, filter.Tags) {
// r.addRejection("tags not matching")
// return false
//}
//
//if filter.ExceptTags != "" && checkFilterStrings(r.Tags, filter.ExceptTags) {
// r.addRejection("unwanted tags")
// return false
//}
return true
}
// CheckSizeFilter additional size check
// for indexers that doesn't announce size, like some cabals
// set flag r.AdditionalSizeCheckRequired if there's a size in the filter, otherwise go a head
// implement API for ptp,btn,bhd,ggn to check for size if needed
// for others pull down torrent and do check
func (r *Release) CheckSizeFilter(minSize string, maxSize string) bool {
if r.Size == 0 {
r.AdditionalSizeCheckRequired = true
return true
} else {
r.AdditionalSizeCheckRequired = false
}
// if r.Size parse filter to bytes and compare
// handle both min and max
if minSize != "" {
// string to bytes
minSizeBytes, err := humanize.ParseBytes(minSize)
if err != nil {
// log could not parse into bytes
}
if r.Size <= minSizeBytes {
r.addRejection("size: smaller than min size")
return false
}
}
if maxSize != "" {
// string to bytes
maxSizeBytes, err := humanize.ParseBytes(maxSize)
if err != nil {
// log could not parse into bytes
}
if r.Size >= maxSizeBytes {
r.addRejection("size: larger than max size")
return false
}
}
return true
}
// MapVars better name
func (r *Release) MapVars(varMap map[string]string) error {
if torrentName, err := getFirstStringMapValue(varMap, []string{"torrentName"}); err != nil {
return errors.Wrap(err, "failed parsing required field")
} else {
r.TorrentName = html.UnescapeString(torrentName)
}
if category, err := getFirstStringMapValue(varMap, []string{"category"}); err == nil {
r.Category = category
}
if freeleech, err := getFirstStringMapValue(varMap, []string{"freeleech"}); err == nil {
r.Freeleech = strings.EqualFold(freeleech, "freeleech") || strings.EqualFold(freeleech, "yes")
}
if freeleechPercent, err := getFirstStringMapValue(varMap, []string{"freeleechPercent"}); err == nil {
// remove % and trim spaces
freeleechPercent = strings.Replace(freeleechPercent, "%", "", -1)
freeleechPercent = strings.Trim(freeleechPercent, " ")
freeleechPercentInt, err := strconv.Atoi(freeleechPercent)
if err != nil {
//log.Debug().Msgf("bad freeleechPercent var: %v", year)
}
r.FreeleechPercent = freeleechPercentInt
}
if uploader, err := getFirstStringMapValue(varMap, []string{"uploader"}); err == nil {
r.Uploader = uploader
}
if torrentSize, err := getFirstStringMapValue(varMap, []string{"torrentSize"}); err == nil {
size, err := humanize.ParseBytes(torrentSize)
if err != nil {
// log could not parse into bytes
}
r.Size = size
// TODO implement other size checks in filter
}
if scene, err := getFirstStringMapValue(varMap, []string{"scene"}); err == nil {
r.IsScene = strings.EqualFold(scene, "true") || strings.EqualFold(scene, "yes")
}
//if year, err := getFirstStringMapValue(varMap, []string{"year"}); err == nil {
// yearI, err := strconv.Atoi(year)
// if err != nil {
// //log.Debug().Msgf("bad year var: %v", year)
// }
// r.Year = yearI
//}
// TODO split this into two
if tags, err := getFirstStringMapValue(varMap, []string{"releaseTags", "tags"}); err == nil {
r.Tags = []string{tags}
}
// TODO parse releaseType
//if releaseType, err := getFirstStringMapValue(varMap, []string{"releaseType", "$releaseType"}); err == nil {
// r.Type = releaseType
//}
//if cue, err := getFirstStringMapValue(varMap, []string{"cue", "$cue"}); err == nil {
// r.Cue = strings.EqualFold(cue, "true")
//}
//if logVar, err := getFirstStringMapValue(varMap, []string{"log", "$log"}); err == nil {
// r.Log = logVar
//}
//if media, err := getFirstStringMapValue(varMap, []string{"media", "$media"}); err == nil {
// r.Media = media
//}
//if format, err := getFirstStringMapValue(varMap, []string{"format", "$format"}); err == nil {
// r.Format = format
//}
//if bitRate, err := getFirstStringMapValue(varMap, []string{"bitrate", "$bitrate"}); err == nil {
// r.Bitrate = bitRate
//}
return nil
}
func checkFilterSlice(name string, filterList []string) bool {
name = strings.ToLower(name)
for _, filter := range filterList {
filter = strings.ToLower(filter)
filter = strings.Trim(filter, " ")
// check if line contains * or ?, if so try wildcard match, otherwise try substring match
a := strings.ContainsAny(filter, "?|*")
if a {
match := wildcard.Match(filter, name)
if match {
return true
}
} else {
b := strings.Contains(name, filter)
if b {
return true
}
}
}
return false
}
func checkFilterStrings(name string, filterList string) bool {
filterSplit := strings.Split(filterList, ",")
name = strings.ToLower(name)
for _, s := range filterSplit {
s = strings.ToLower(s)
s = strings.Trim(s, " ")
// check if line contains * or ?, if so try wildcard match, otherwise try substring match
a := strings.ContainsAny(s, "?|*")
if a {
match := wildcard.Match(s, name)
if match {
return true
}
} else {
b := strings.Contains(name, s)
if b {
return true
}
}
}
return false
}
// checkFilterIntStrings "1,2,3-20"
func checkFilterIntStrings(value int, filterList string) bool {
filters := strings.Split(filterList, ",")
for _, s := range filters {
s = strings.Replace(s, "%", "", -1)
s = strings.Trim(s, " ")
if strings.Contains(s, "-") {
minMax := strings.Split(s, "-")
// to int
min, err := strconv.ParseInt(minMax[0], 10, 32)
if err != nil {
return false
}
max, err := strconv.ParseInt(minMax[1], 10, 32)
if err != nil {
return false
}
if min > max {
// handle error
return false
} else {
// if announcePercent is greater than min and less than max return true
if value >= int(min) && value <= int(max) {
return true
}
}
}
filterInt, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return false
}
if int(filterInt) == value {
return true
}
}
return false
}
func checkFreeleechPercent(announcePercent int, filterPercent string) bool {
filters := strings.Split(filterPercent, ",")
// remove % and trim spaces
//announcePercent = strings.Replace(announcePercent, "%", "", -1)
//announcePercent = strings.Trim(announcePercent, " ")
//announcePercentInt, err := strconv.ParseInt(announcePercent, 10, 32)
//if err != nil {
// return false
//}
for _, s := range filters {
s = strings.Replace(s, "%", "", -1)
s = strings.Trim(s, " ")
if strings.Contains(s, "-") {
minMax := strings.Split(s, "-")
// to int
min, err := strconv.ParseInt(minMax[0], 10, 32)
if err != nil {
return false
}
max, err := strconv.ParseInt(minMax[1], 10, 32)
if err != nil {
return false
}
if min > max {
// handle error
return false
} else {
// if announcePercent is greater than min and less than max return true
if announcePercent >= int(min) && announcePercent <= int(max) {
return true
}
}
}
filterPercentInt, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return false
}
if int(filterPercentInt) == announcePercent {
return true
}
}
return false
}
func getStringMapValue(stringMap map[string]string, key string) (string, error) {
lowerKey := strings.ToLower(key)
// case-sensitive match
//if caseSensitive {
// v, ok := stringMap[key]
// if !ok {
// return "", fmt.Errorf("key was not found in map: %q", key)
// }
//
// return v, nil
//}
// case-insensitive match
for k, v := range stringMap {
if strings.ToLower(k) == lowerKey {
return v, nil
}
}
return "", fmt.Errorf("key was not found in map: %q", lowerKey)
}
func getFirstStringMapValue(stringMap map[string]string, keys []string) (string, error) {
for _, k := range keys {
if val, err := getStringMapValue(stringMap, k); err == nil {
return val, nil
}
}
return "", fmt.Errorf("key were not found in map: %q", strings.Join(keys, ", "))
}
func findLast(input string, pattern string) (string, error) {
matched := make([]string, 0)
//for _, s := range arr {
rxp, err := regexp.Compile(pattern)
if err != nil {
return "", err
//return errors.Wrapf(err, "invalid regex: %s", value)
}
matches := rxp.FindStringSubmatch(input)
if matches != nil {
// first value is the match, second value is the text
if len(matches) >= 1 {
last := matches[len(matches)-1]
// add to temp slice
matched = append(matched, last)
}
}
//}
// check if multiple values in temp slice, if so get the last one
if len(matched) >= 1 {
last := matched[len(matched)-1]
return last, nil
}
return "", nil
}
func findLastBool(input string, pattern string) (bool, error) {
matched := make([]string, 0)
rxp, err := regexp.Compile(pattern)
if err != nil {
return false, err
}
matches := rxp.FindStringSubmatch(input)
if matches != nil {
// first value is the match, second value is the text
if len(matches) >= 1 {
last := matches[len(matches)-1]
// add to temp slice
matched = append(matched, last)
}
}
//}
// check if multiple values in temp slice, if so get the last one
if len(matched) >= 1 {
//last := matched[len(matched)-1]
return true, nil
}
return false, nil
}
func findLastInt(input string, pattern string) (int, error) {
matched := make([]string, 0)
//for _, s := range arr {
rxp, err := regexp.Compile(pattern)
if err != nil {
return 0, err
//return errors.Wrapf(err, "invalid regex: %s", value)
}
matches := rxp.FindStringSubmatch(input)
if matches != nil {
// first value is the match, second value is the text
if len(matches) >= 1 {
last := matches[len(matches)-1]
// add to temp slice
matched = append(matched, last)
}
}
//}
// check if multiple values in temp slice, if so get the last one
if len(matched) >= 1 {
last := matched[len(matched)-1]
i, err := strconv.Atoi(last)
if err != nil {
return 0, err
}
return i, nil
}
return 0, nil
}
type ReleaseStats struct {
TotalCount int64 `json:"total_count"`
FilteredCount int64 `json:"filtered_count"`
FilterRejectedCount int64 `json:"filter_rejected_count"`
PushApprovedCount int64 `json:"push_approved_count"`
PushRejectedCount int64 `json:"push_rejected_count"`
}
type ReleasePushStatus string
const (
ReleasePushStatusApproved ReleasePushStatus = "PUSH_APPROVED"
ReleasePushStatusRejected ReleasePushStatus = "PUSH_REJECTED"
ReleasePushStatusMixed ReleasePushStatus = "MIXED" // For multiple actions, one might go and the other not
ReleasePushStatusPending ReleasePushStatus = "PENDING" // Initial status
)
type ReleaseFilterStatus string
const (
ReleaseStatusFilterApproved ReleaseFilterStatus = "FILTER_APPROVED"
ReleaseStatusFilterRejected ReleaseFilterStatus = "FILTER_REJECTED"
ReleaseStatusFilterPending ReleaseFilterStatus = "PENDING"
)
type ReleaseProtocol string
const (
ReleaseProtocolTorrent ReleaseProtocol = "torrent"
)
type ReleaseImplementation string
const (
ReleaseImplementationIRC ReleaseImplementation = "IRC"
)
type QueryParams struct {
Limit uint64
Cursor uint64
Sort map[string]string
Filter map[string]string
Search string
}

View file

@ -0,0 +1,364 @@
package domain
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestRelease_Parse(t *testing.T) {
tests := []struct {
name string
fields Release
wantErr bool
}{
{name: "parse_1", fields: Release{
ID: 0,
Rejections: nil,
Indexer: "",
FilterName: "",
Protocol: "",
Implementation: "",
Timestamp: time.Time{},
TorrentID: "",
GroupID: "",
TorrentName: "Servant S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-FLUX",
Raw: "",
Title: "",
Category: "",
Season: 0,
Episode: 0,
Year: 0,
Resolution: "",
Source: "",
Codec: "",
Container: "",
HDR: "",
Audio: "",
Group: "",
Region: "",
Edition: "",
Proper: false,
Repack: false,
Website: "",
Language: "",
Unrated: false,
Hybrid: false,
Size: 0,
ThreeD: false,
Artists: nil,
Type: "",
Format: "",
Bitrate: "",
LogScore: 0,
HasLog: false,
HasCue: false,
IsScene: false,
Origin: "",
Tags: nil,
Freeleech: false,
FreeleechPercent: 0,
Uploader: "",
PreTime: "",
TorrentURL: "",
Filter: nil,
}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := tt.fields
if err := r.Parse(); (err != nil) != tt.wantErr {
t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestRelease_CheckFilter(t *testing.T) {
type args struct {
filter Filter
}
tests := []struct {
name string
fields *Release
args args
want bool
}{
{
name: "size_between_max_min",
fields: &Release{Size: uint64(10000000001)},
args: args{
filter: Filter{
Enabled: true,
MinSize: "10 GB",
MaxSize: "20GB",
},
},
want: true,
},
{
name: "size_larger_than_max",
fields: &Release{Size: uint64(30000000001)},
args: args{
filter: Filter{
Enabled: true,
MinSize: "10 GB",
MaxSize: "20GB",
},
},
want: false,
},
//{
// name: "test_no_size",
// fields: &Release{Size: uint64(0)},
// args: args{
// filter: Filter{
// Enabled: true,
// FilterGeneral: FilterGeneral{MinSize: "10 GB", MaxSize: "20GB"},
// },
// },
// want: false, // additional checks
//},
{
name: "movie_parse_1",
fields: &Release{
TorrentName: "That Movie 2020 2160p BluRay DD5.1 x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001),
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2020",
MatchReleaseGroups: "GROUP1",
},
},
want: true,
},
{
name: "movie_parse_shows",
fields: &Release{
TorrentName: "That Movie 2020 2160p BluRay DD5.1 x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001),
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2020",
MatchReleaseGroups: "GROUP1",
Shows: "That Movie",
},
},
want: true,
},
{
name: "movie_parse_multiple_shows",
fields: &Release{
TorrentName: "That Movie 2020 2160p BluRay DD5.1 x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001),
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2020",
MatchReleaseGroups: "GROUP1",
Shows: "That Movie, good story, bad movie",
},
},
want: true,
},
{
name: "movie_parse_wildcard_shows",
fields: &Release{
TorrentName: "That Movie 2020 2160p BluRay DD5.1 x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001), // 30GB
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "Movies, tv",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"1080p", "2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2015,2018-2022",
MatchReleaseGroups: "GROUP1,BADGROUP",
Shows: "*Movie*, good story, bad movie",
},
},
want: true,
},
{
name: "movie_bad_category",
fields: &Release{
TorrentName: "That Movie 2020 2160p BluRay DD5.1 x264-GROUP1",
Category: "Movies",
Freeleech: true,
Size: uint64(30000000001), // 30GB
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
Freeleech: true,
MinSize: "10 GB",
MaxSize: "40GB",
Resolutions: []string{"1080p", "2160p"},
Sources: []string{"BluRay"},
Codecs: []string{"x264"},
Years: "2015,2018-2022",
MatchReleaseGroups: "GROUP1,BADGROUP",
Shows: "*Movie*, good story, bad movie",
},
},
want: false,
},
{
name: "tv_match_season_episode",
fields: &Release{
TorrentName: "Good show S01E01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
Resolutions: []string{"1080p", "2160p"},
Sources: []string{"WEB-DL"},
Codecs: []string{"HEVC"},
MatchReleaseGroups: "GROUP1,GROUP2",
Seasons: "1,2",
Episodes: "1",
},
},
want: true,
},
{
name: "tv_match_season",
fields: &Release{
TorrentName: "Good show S01 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
Resolutions: []string{"1080p", "2160p"},
Sources: []string{"WEB-DL"},
Codecs: []string{"HEVC"},
MatchReleaseGroups: "GROUP1,GROUP2",
Seasons: "1,2",
},
},
want: true,
},
{
name: "tv_bad_match_season",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
Resolutions: []string{"1080p", "2160p"},
Sources: []string{"WEB-DL"},
Codecs: []string{"HEVC"},
MatchReleaseGroups: "GROUP1,GROUP2",
Seasons: "1",
},
},
want: false,
},
{
name: "match_uploader",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
Uploader: "Uploader1",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1",
},
},
want: true,
},
{
name: "except_uploader",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
Uploader: "Anonymous",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
ExceptUploaders: "Anonymous",
},
},
want: false,
},
{
name: "match_except_uploader",
fields: &Release{
TorrentName: "Good show S02 2160p ATVP WEB-DL DDP 5.1 Atmos DV HEVC-GROUP2",
Category: "TV",
Uploader: "Uploader1",
},
args: args{
filter: Filter{
Enabled: true,
MatchCategories: "*tv*",
MatchUploaders: "Uploader1,Uploader2",
ExceptUploaders: "Anonymous",
},
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := tt.fields // Release
_ = r.Parse() // Parse TorrentName into struct
got := r.CheckFilter(tt.args.filter)
assert.Equal(t, tt.want, got)
})
}
}