idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 08:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(filters): RED and OPS lossless parsing and filtering (#1373)

Browse source 
* fix(filters): RED and OPS lossless parsing and filtering

* fix(filters): logscore and EP parsing

* fix(filters): tests

* fix(filters): tests

* feat(definitions): RED parse title variable

* feat(indexers): setup indexer to filter tests

* feat(indexers): tests and improve parsing

* feat(indexers): improve tests
This commit is contained in:
ze0s 2024-01-28 22:03:25 +01:00 • committed by GitHub
parent 9db5a8b116
commit 5328078b32
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 1093 additions and 360 deletions
Download patch file Download diff file Expand all files Collapse all files

82
internal/indexer/service.go
View file

@ -516,6 +516,46 @@ func (s *service) LoadIndexerDefinitions() error {
return nil
}
var ErrIndexerDefinitionDeprecated = errors.New("DEPRECATED: indexer definition version")
func isValidExtension(ext string) bool {
return ext == ".yaml" || ext == ".yml"
}
func OpenAndProcessDefinition(file string) (*domain.IndexerDefinition, error) {
f, err := os.Open(file)
if err != nil {
return nil, errors.Wrap(err, "could not open file: %s", file)
}
defer f.Close()
var d *domain.IndexerDefinitionCustom
dec := yaml.NewDecoder(f)
dec.KnownFields(false)
if err = dec.Decode(&d); err != nil {
return nil, errors.Wrap(err, "could not decode definition file: %s", file)
}
if d == nil {
//s.log.Warn().Msgf("skipping empty file: %s", file)
return nil, errors.New("empty definition file")
}
if d.Implementation == "" {
d.Implementation = "irc"
}
//if d.Implementation == "irc" && d.IRC != nil {
// if d.IRC.Parse == nil {
// s.log.Warn().Msgf("DEPRECATED: indexer definition version: %s", file)
// }
//}
return d.ToIndexerDefinition(), nil
}
// LoadCustomIndexerDefinitions load definitions from custom path
func (s *service) LoadCustomIndexerDefinitions() error {
if s.config.CustomDefinitions == "" {
@ -539,51 +579,23 @@ func (s *service) LoadCustomIndexerDefinitions() error {
customCount := 0
for _, f := range entries {
fileExtension := filepath.Ext(f.Name())
if fileExtension != ".yaml" && fileExtension != ".yml" {
s.log.Warn().Stack().Msgf("skipping unknown extension definition file: %s", f.Name())
ext := filepath.Ext(f.Name())
if !isValidExtension(ext) {
s.log.Warn().Msgf("unsupported extension %s, definition file: %s", ext, f.Name())
continue
}
file := filepath.Join(s.config.CustomDefinitions, f.Name())
s.log.Trace().Msgf("parsing custom: %s", file)
s.log.Trace().Msgf("parsing custom definition: %s", file)
data, err := os.ReadFile(file)
definition, err := OpenAndProcessDefinition(file)
if err != nil {
s.log.Error().Stack().Err(err).Msgf("failed reading file: %s", file)
return errors.Wrap(err, "could not read file: %s", file)
}
var d *domain.IndexerDefinitionCustom
dec := yaml.NewDecoder(bytes.NewReader(data))
// Do _not_ fail on unknown fields while parsing custom indexer
// definitions for better backwards compatibility. See discussion:
// https://github.com/autobrr/autobrr/pull/1257#issuecomment-1813821391
dec.KnownFields(false)
if err = dec.Decode(&d); err != nil {
s.log.Error().Stack().Err(err).Msgf("failed unmarshal file: %s", file)
return errors.Wrap(err, "could not unmarshal file: %s", file)
}
if d == nil {
s.log.Warn().Msgf("skipping empty file: %s", file)
s.log.Error().Err(err).Msgf("could not open definition file: %s", file)
continue
}
if d.Implementation == "" {
d.Implementation = "irc"
}
// to prevent crashing from non-updated definitions lets skip
if d.Implementation == "irc" && d.IRC != nil {
if d.IRC.Parse == nil {
s.log.Warn().Msgf("DEPRECATED: indexer definition version: %s", file)
}
}
s.definitions[d.Identifier] = *d.ToIndexerDefinition()
s.definitions[definition.Identifier] = *definition
customCount++
}