mirror of
https://github.com/idanoo/autobrr
synced 2025-07-22 16:29:12 +00:00
fix(indexers): improve animebytes parsing (#477)
* Fixed typo for "Aweomse Raw Novel" Added one missing whitespace after "manga" tag for "Awesome Live Action Special" test Changed regex expression to fix AB parsing * Fixed regex to acknowledge "uploader" var in case there is no tags in the announce. Added test line for announces without tags. * chore: update rls package * fix(release): don't overwrite season and episode
This commit is contained in:
xoaaC
GitHub
parent
bc335ccf1f
commit
2777f8c3cd
6 changed files with 33 additions and 34 deletions
11
go.mod
11
go.mod
|
|
@ -22,7 +22,7 @@ require (
|
|||
github.com/lib/pq v1.10.7
|
||||
github.com/mattn/go-shellwords v1.0.12
|
||||
github.com/mmcdole/gofeed v1.1.3
|
||||
github.com/moistari/rls v0.3.0
|
||||
github.com/moistari/rls v0.5.5
|
||||
github.com/mrobinsn/go-rtorrent v1.8.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/r3labs/sse/v2 v2.8.1
|
||||
|
|
@ -35,7 +35,7 @@ require (
|
|||
github.com/stretchr/testify v1.8.0
|
||||
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
|
||||
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
|
|
@ -84,12 +84,11 @@ require (
|
|||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.1 // indirect
|
||||
golang.org/x/mod v0.4.1 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
|
||||
golang.org/x/text v0.3.8 // indirect
|
||||
golang.org/x/tools v0.1.12 // indirect
|
||||
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
|
|
|||
20
go.sum
20
go.sum
|
|
@ -317,8 +317,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
|
|||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/moistari/rls v0.3.0 h1:D+Io+h39ICo+xnzpn9Gvrm+aUl+B70ABWRrfJZUWUz8=
|
||||
github.com/moistari/rls v0.3.0/go.mod h1:+VmEKlV6nBXzhGxyO9NoHIp8ywSQlm89UUGPNi8vZVc=
|
||||
github.com/moistari/rls v0.5.5 h1:sVj1G5E2eIeqJQh9sdQ4THXCLEBlJElXVxoUwFBY9Uw=
|
||||
github.com/moistari/rls v0.5.5/go.mod h1:+VmEKlV6nBXzhGxyO9NoHIp8ywSQlm89UUGPNi8vZVc=
|
||||
github.com/mrobinsn/go-rtorrent v1.8.0 h1:+61aDIP0asy57lRD/uZtmxfE0/gjkHnt3uddOhMKUJ8=
|
||||
github.com/mrobinsn/go-rtorrent v1.8.0/go.mod h1:CdVq2IwM+JU9D6TnWiQSg9lqZWu6zUfK67YXET2LqIM=
|
||||
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
|
||||
|
|
@ -477,8 +477,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
|
|||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -537,8 +538,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY=
|
||||
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -596,8 +597,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
|
|||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -652,14 +653,13 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f
|
|||
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618=
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
|
|
|
|||
|
|
@ -102,8 +102,7 @@ func (a *announceProcessor) processQueue(queue chan string) {
|
|||
rls := domain.NewRelease(a.indexer.Identifier)
|
||||
|
||||
// on lines matched
|
||||
err := a.onLinesMatched(a.indexer, tmpVars, rls)
|
||||
if err != nil {
|
||||
if err := a.onLinesMatched(a.indexer, tmpVars, rls); err != nil {
|
||||
a.log.Debug().Msgf("error match line: %v", "")
|
||||
continue
|
||||
}
|
||||
|
|
@ -164,20 +163,18 @@ func (a *announceProcessor) parseExtract(pattern string, vars []string, tmpVars
|
|||
|
||||
// onLinesMatched process vars into release
|
||||
func (a *announceProcessor) onLinesMatched(def *domain.IndexerDefinition, vars map[string]string, rls *domain.Release) error {
|
||||
var err error
|
||||
|
||||
err = rls.MapVars(def, vars)
|
||||
if err != nil {
|
||||
if err := rls.MapVars(def, vars); err != nil {
|
||||
a.log.Error().Stack().Err(err).Msg("announce: could not map vars for release")
|
||||
return err
|
||||
}
|
||||
|
||||
// parse fields
|
||||
// run before ParseMatch to not potentially use a reconstructed TorrentName
|
||||
rls.ParseString(rls.TorrentName)
|
||||
|
||||
// parse torrentUrl
|
||||
err = def.Parse.ParseMatch(vars, def.SettingsMap, rls)
|
||||
if err != nil {
|
||||
if err := def.Parse.ParseMatch(vars, def.SettingsMap, rls); err != nil {
|
||||
a.log.Error().Stack().Err(err).Msgf("announce: %v", err)
|
||||
return err
|
||||
}
|
||||
|
|
@ -219,8 +216,7 @@ func (a *announceProcessor) processTorrentUrl(match string, vars map[string]stri
|
|||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
err = tmpl.Execute(&b, &tmpVars)
|
||||
if err != nil {
|
||||
if err := tmpl.Execute(&b, &tmpVars); err != nil {
|
||||
a.log.Error().Err(err).Msg("could not write torrent url template output")
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -162,8 +162,7 @@ func (p *IndexerParse) ParseMatch(vars map[string]string, extraVars map[string]s
|
|||
}
|
||||
|
||||
var urlBytes bytes.Buffer
|
||||
err = tmpl.Execute(&urlBytes, &tmpVars)
|
||||
if err != nil {
|
||||
if err := tmpl.Execute(&urlBytes, &tmpVars); err != nil {
|
||||
return errors.New("could not write torrent url template output")
|
||||
}
|
||||
|
||||
|
|
@ -178,8 +177,7 @@ func (p *IndexerParse) ParseMatch(vars map[string]string, extraVars map[string]s
|
|||
}
|
||||
|
||||
var nameBytes bytes.Buffer
|
||||
err = tmplName.Execute(&nameBytes, &tmpVars)
|
||||
if err != nil {
|
||||
if err := tmplName.Execute(&nameBytes, &tmpVars); err != nil {
|
||||
return errors.New("could not write torrent name template output")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -192,8 +192,6 @@ func (r *Release) ParseString(title string) {
|
|||
r.Title = rel.Title
|
||||
r.Source = rel.Source
|
||||
r.Resolution = rel.Resolution
|
||||
r.Season = rel.Series
|
||||
r.Episode = rel.Episode
|
||||
r.Region = rel.Region
|
||||
r.Audio = rel.Audio
|
||||
r.AudioChannels = rel.Channels
|
||||
|
|
@ -203,6 +201,13 @@ func (r *Release) ParseString(title string) {
|
|||
r.Other = rel.Other
|
||||
r.Artists = rel.Artist
|
||||
|
||||
if r.Season == 0 {
|
||||
r.Season = rel.Series
|
||||
}
|
||||
if r.Episode == 0 {
|
||||
r.Episode = rel.Episode
|
||||
}
|
||||
|
||||
if r.Year == 0 {
|
||||
r.Year = rel.Year
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,15 +49,16 @@ parse:
|
|||
type: single
|
||||
lines:
|
||||
- test:
|
||||
- "Aweomse Raw Novel - Light Novel [2005] :: Raw / EPUB || https://animebytes.tv/torrents.php?id=00000&torrentid=00000 || supernatural || Uploaded by: Test-Uploader"
|
||||
- "Awesome Raw Novel - Light Novel [2005] :: Raw / EPUB || https://animebytes.tv/torrents.php?id=00000&torrentid=00000 || supernatural || Uploaded by: Test-Uploader"
|
||||
- "Awesome Translated Novel - Light Novel [2018] :: Translated (Translation Group) / EPUB || https://animebytes.tv/torrents.php?id=00000&torrentid=000000 || adventure, comedy, fantasy, harem, school.life, magic, action || Uploaded by: UPLOADER"
|
||||
- "Great BluRay SoftSubbed Anime - TV Series [2020] :: Blu-ray / MKV / h264 10-bit / 1080p / FLAC 2.0 / Dual Audio / Softsubs (Sub Group) / Freeleech || https://animebytes.tv/torrents.php?id=00008&torrentid=000000 || comedy, drama, school.life, sports || Uploaded by: Uploader"
|
||||
- "Awesome Translated Manga - Manga [2019] :: Translated (Translation Group) / Digital / Ongoing || https://animebytes.tv/torrents.php?id=00000&torrentid=000000 || comedy, fantasy, school.life, shounen, slice.of.life"
|
||||
- "Cool Movie - Movie [2020] :: Blu-ray / MKV / h265 10-bit / 1929x804 / AC3 5.1 / Dual Audio / Softsubs (Sub Group) || https://animebytes.tv/torrents.php?id=000000&torrentid=0000000 || drama, romance, slice.of.life || Uploaded by: Anon-Uploader"
|
||||
- "Awesome Live Action Special - Live Action TV Special [2021] :: Web / MKV / h264 / 848x480 / AAC 2.0 / Softsubs (Sub Group) || https://animebytes.tv/torrents.php?id=00000&torrentid=00000 || manga|| Uploaded by: Some-Uploader"
|
||||
- "Awesome Live Action Special - Live Action TV Special [2021] :: Web / MKV / h264 / 848x480 / AAC 2.0 / Softsubs (Sub Group) || https://animebytes.tv/torrents.php?id=00000&torrentid=00000 || manga || Uploaded by: Some-Uploader"
|
||||
- "Best Visual Novel - Visual Novel [2006] :: Game / PC / Unarchived / Hentai (Censored) || https://animebytes.tv/torrents.php?id=00000&torrentid=00000 || nukige || Uploaded by: Uploader"
|
||||
- "Artist Name - Album of awesome Music [1991] :: MP3 / V0 (VBR) / CD || https://animebytes.tv/torrents2.php?id=00000&torrentid=000000 || ambient, folk || Uploaded by: Uploader"
|
||||
pattern: '(.*?)(?: - )?(Visual Novel|Light Novel|TV.*|Movie|Manga|OVA|ONA|DVD Special|BD Special|Oneshot|Anthology|Manhwa|Manhua|Artbook|Game|Live Action.*|)[\s\p{Zs}]{2,}\[(\d+)\] :: (.*?)(?: \/ (?:RAW|Softsubs|Hardsubs|Translated)\s\((.+)\)(?:.*Episode\s(\d+))?(?:.*(Freeleech))?.*)? \|\| (https.*)\/torrents.*\?id=\d+&torrentid=(\d+) \|\| (.+?(?:(?:\|\| Uploaded by|$))?) (?:\|\| Uploaded by: (.*))?$'
|
||||
- "Awesome Series - TV Series [2022] :: Web / MKV / h264 / 1080p / AAC 2.0 / Softsubs (Sub Group) / Episode 1 / Freeleech || https://animebytes.tv/torrents.php?id=00000&torrentid=000000 || || Uploaded by: Uploader"
|
||||
pattern: '(.*?)(?: - )?(Visual Novel|Light Novel|TV.*|Movie|Manga|OVA|ONA|DVD Special|BD Special|Oneshot|Anthology|Manhwa|Manhua|Artbook|Game|Live Action.*|)[\s\p{Zs}]{2,}\[(\d+)\] :: (.*?)(?: \/ (?:RAW|Softsubs|Hardsubs|Translated)\s\((.+)\)(?:.*Episode\s(\d+))?(?:.*(Freeleech))?.*)?(?:.?\|\|.?)?(https.*)\/torrents.*\?id=\d+&torrentid=(\d+)(?:.?\|\|.?)?([A-Za-z,. ]+\w)?(?:.?\|\|.?)?(?:Uploaded by: (.*))?'
|
||||
vars:
|
||||
- torrentName
|
||||
- category
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue