fix(indexers): duplicates in list (#317)

* feat(indexers): refactor setup and updates

* feat(indexers): use definitions as non-pointer

internal/http/indexer.go

@@ -16,7 +16,7 @@ type indexerService interface {
 	Update(ctx context.Context, indexer domain.Indexer) (*domain.Indexer, error)
 	List(ctx context.Context) ([]domain.Indexer, error)
 	GetAll() ([]*domain.IndexerDefinition, error)
-	GetTemplates() ([]*domain.IndexerDefinition, error)
+	GetTemplates() ([]domain.IndexerDefinition, error)
 	Delete(ctx context.Context, id int) error
 }
 

internal/indexer/service.go

@@ -7,6 +7,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/autobrr/autobrr/internal/domain"
@@ -14,6 +15,7 @@ import (
 	"github.com/autobrr/autobrr/internal/scheduler"
 
 	"github.com/gosimple/slug"
+	"github.com/rs/zerolog"
 	"gopkg.in/yaml.v3"
 )
 
@@ -24,7 +26,7 @@ type Service interface {
 	FindByFilterID(ctx context.Context, id int) ([]domain.Indexer, error)
 	List(ctx context.Context) ([]domain.Indexer, error)
 	GetAll() ([]*domain.IndexerDefinition, error)
-	GetTemplates() ([]*domain.IndexerDefinition, error)
+	GetTemplates() ([]domain.IndexerDefinition, error)
 	LoadIndexerDefinitions() error
 	GetIndexersByIRCNetwork(server string) []*domain.IndexerDefinition
 	GetTorznabIndexers() []domain.IndexerDefinition
@@ -32,43 +34,43 @@ type Service interface {
 }
 
 type service struct {
-	log        logger.Logger
+	log        zerolog.Logger
 	config     *domain.Config
 	repo       domain.IndexerRepo
 	apiService APIService
 	scheduler  scheduler.Service
 
 	// contains all raw indexer definitions
-	indexerDefinitions map[string]*domain.IndexerDefinition
+	definitions map[string]domain.IndexerDefinition
-
+	// definition with indexer data
+	mappedDefinitions map[string]*domain.IndexerDefinition
 	// map server:channel:announce to indexer.Identifier
-	mapIndexerIRCToName map[string]string
-
 	lookupIRCServerDefinition map[string]map[string]*domain.IndexerDefinition
-
+	// torznab indexers
 	torznabIndexers map[string]*domain.IndexerDefinition
 }
 
 func NewService(log logger.Logger, config *domain.Config, repo domain.IndexerRepo, apiService APIService, scheduler scheduler.Service) Service {
 	return &service{
-		log:                       log,
+		log:                       log.With().Str("service", "indexer").Logger(),
 		config:                    config,
 		repo:                      repo,
 		apiService:                apiService,
 		scheduler:                 scheduler,
-		indexerDefinitions:        make(map[string]*domain.IndexerDefinition),
-		mapIndexerIRCToName:       make(map[string]string),
 		lookupIRCServerDefinition: make(map[string]map[string]*domain.IndexerDefinition),
 		torznabIndexers:           make(map[string]*domain.IndexerDefinition),
+		definitions:               make(map[string]domain.IndexerDefinition),
+		mappedDefinitions:         make(map[string]*domain.IndexerDefinition),
 	}
 }
 
 func (s *service) Store(ctx context.Context, indexer domain.Indexer) (*domain.Indexer, error) {
 	identifier := indexer.Identifier
-	if indexer.Identifier == "torznab" {
+	//if indexer.Identifier == "torznab" {
+	if indexer.Implementation == "torznab" {
 		// if the name already contains torznab remove it
 		cleanName := strings.ReplaceAll(strings.ToLower(indexer.Name), "torznab", "")
-		identifier = slug.Make(fmt.Sprintf("%v-%v", indexer.Identifier, cleanName))
+		identifier = slug.Make(fmt.Sprintf("%v-%v", indexer.Implementation, cleanName)) // torznab-name
 	}
 
 	indexer.Identifier = identifier
@@ -96,7 +98,7 @@ func (s *service) Update(ctx context.Context, indexer domain.Indexer) (*domain.I
 	}
 
 	// add to indexerInstances
-	err = s.addIndexer(*i)
+	err = s.updateIndexer(*i)
 	if err != nil {
 		s.log.Error().Stack().Err(err).Msgf("failed to add indexer: %v", indexer.Name)
 		return nil, err
@@ -131,13 +133,30 @@ func (s *service) List(ctx context.Context) ([]domain.Indexer, error) {
 }
 
 func (s *service) GetAll() ([]*domain.IndexerDefinition, error) {
+	var res = make([]*domain.IndexerDefinition, 0)
+
+	for _, indexer := range s.mappedDefinitions {
+		if indexer == nil {
+			continue
+		}
+
+		res = append(res, indexer)
+	}
+
+	// sort by name
+	sort.SliceStable(res, func(i, j int) bool {
+		return strings.ToLower(res[i].Name) < strings.ToLower(res[j].Name)
+	})
+
+	return res, nil
+}
+
+func (s *service) mapIndexers() (map[string]*domain.IndexerDefinition, error) {
 	indexers, err := s.repo.List(context.Background())
 	if err != nil {
 		return nil, err
 	}
 
-	var res = make([]*domain.IndexerDefinition, 0)
-
 	for _, indexer := range indexers {
 		indexerDefinition, err := s.mapIndexer(indexer)
 		if err != nil {
@@ -148,60 +167,97 @@ func (s *service) GetAll() ([]*domain.IndexerDefinition, error) {
 			continue
 		}
 
-		res = append(res, indexerDefinition)
+		s.mappedDefinitions[indexer.Identifier] = indexerDefinition
 	}
 
-	return res, nil
+	return s.mappedDefinitions, nil
 }
 
 func (s *service) mapIndexer(indexer domain.Indexer) (*domain.IndexerDefinition, error) {
-
+	definitionName := indexer.Identifier
-	var in *domain.IndexerDefinition
 	if indexer.Implementation == "torznab" {
-		in = s.getDefinitionByName("torznab")
+		definitionName = "torznab"
-		if in == nil {
-			// if no indexerDefinition found, continue
-			return nil, nil
-		}
-	} else {
-		in = s.getDefinitionByName(indexer.Identifier)
-		if in == nil {
-			// if no indexerDefinition found, continue
-			return nil, nil
-		}
 	}
 
-	in.ID = int(indexer.ID)
+	d := s.getDefinitionByName(definitionName)
-	in.Name = indexer.Name
+	if d == nil {
-	in.Identifier = indexer.Identifier
+		// if no indexerDefinition found, continue
-	in.Implementation = indexer.Implementation
+		return nil, nil
-	in.Enabled = indexer.Enabled
+	}
-	in.SettingsMap = make(map[string]string)
 
-	if in.Implementation == "" {
+	d.ID = int(indexer.ID)
-		in.Implementation = "irc"
+	d.Name = indexer.Name
+	d.Identifier = indexer.Identifier
+	d.Implementation = indexer.Implementation
+	d.Enabled = indexer.Enabled
+
+	if d.SettingsMap == nil {
+		d.SettingsMap = make(map[string]string)
+	}
+
+	if d.Implementation == "" {
+		d.Implementation = "irc"
 	}
 
 	// map settings
 	// add value to settings objects
-	for i, setting := range in.Settings {
+	for i, setting := range d.Settings {
 		if v, ok := indexer.Settings[setting.Name]; ok {
 			setting.Value = v
 
-			in.SettingsMap[setting.Name] = v
+			d.SettingsMap[setting.Name] = v
 		}
 
-		in.Settings[i] = setting
+		d.Settings[i] = setting
 	}
 
-	return in, nil
+	return d, nil
 }
 
-func (s *service) GetTemplates() ([]*domain.IndexerDefinition, error) {
+func (s *service) updateMapIndexer(indexer domain.Indexer) (*domain.IndexerDefinition, error) {
+	definitionName := indexer.Identifier
+	if indexer.Implementation == "torznab" {
+		definitionName = "torznab"
+	}
 
-	definitions := s.indexerDefinitions
+	d, ok := s.mappedDefinitions[definitionName]
+	if !ok {
+		return nil, nil
+	}
 
-	ret := make([]*domain.IndexerDefinition, 0)
+	d.ID = int(indexer.ID)
+	d.Name = indexer.Name
+	d.Identifier = indexer.Identifier
+	d.Implementation = indexer.Implementation
+	d.Enabled = indexer.Enabled
+
+	if d.SettingsMap == nil {
+		d.SettingsMap = make(map[string]string)
+	}
+
+	if d.Implementation == "" {
+		d.Implementation = "irc"
+	}
+
+	// map settings
+	// add value to settings objects
+	for i, setting := range d.Settings {
+		if v, ok := indexer.Settings[setting.Name]; ok {
+			setting.Value = v
+
+			d.SettingsMap[setting.Name] = v
+		}
+
+		d.Settings[i] = setting
+	}
+
+	return d, nil
+}
+
+func (s *service) GetTemplates() ([]domain.IndexerDefinition, error) {
+	definitions := s.definitions
+
+	ret := make([]domain.IndexerDefinition, 0)
 	for _, definition := range definitions {
 		ret = append(ret, definition)
 	}
@@ -225,15 +281,13 @@ func (s *service) Start() error {
 	}
 
 	// load the indexers' setup by the user
-	indexerDefinitions, err := s.GetAll()
+	indexerDefinitions, err := s.mapIndexers()
 	if err != nil {
 		return err
 	}
 
 	for _, indexer := range indexerDefinitions {
 		if indexer.IRC != nil {
-			s.mapIRCIndexerLookup(indexer.Identifier, indexer)
-
 			// add to irc server lookup table
 			s.mapIRCServerDefinitionLookup(indexer.IRC.Server, indexer)
 
@@ -257,18 +311,12 @@ func (s *service) Start() error {
 }
 
 func (s *service) removeIndexer(indexer domain.Indexer) error {
-
+	delete(s.definitions, indexer.Identifier)
-	delete(s.indexerDefinitions, indexer.Identifier)
-
-	// TODO delete from mapIndexerIRCToName
 
 	return nil
 }
 
 func (s *service) addIndexer(indexer domain.Indexer) error {
-
-	// TODO only add if not already there?? Overwrite?
-
 	indexerDefinition, err := s.mapIndexer(indexer)
 	if err != nil {
 		return err
@@ -278,14 +326,7 @@ func (s *service) addIndexer(indexer domain.Indexer) error {
 		return errors.New("addindexer: could not find definition")
 	}
 
-	// TODO only add enabled?
-	//if !indexer.Enabled {
-	//	continue
-	//}
-
 	if indexerDefinition.IRC != nil {
-		s.mapIRCIndexerLookup(indexer.Identifier, indexerDefinition)
-
 		// add to irc server lookup table
 		s.mapIRCServerDefinitionLookup(indexerDefinition.IRC.Server, indexerDefinition)
 
@@ -305,25 +346,34 @@ func (s *service) addIndexer(indexer domain.Indexer) error {
 	return nil
 }
 
-func (s *service) mapIRCIndexerLookup(indexerIdentifier string, indexerDefinition *domain.IndexerDefinition) {
+func (s *service) updateIndexer(indexer domain.Indexer) error {
-	// map irc stuff to indexer.name
+	indexerDefinition, err := s.updateMapIndexer(indexer)
-	// map[irc.network.test:channel:announcer1] = indexer1
+	if err != nil {
-	// map[irc.network.test:channel:announcer2] = indexer2
+		return err
+	}
+
+	if indexerDefinition == nil {
+		return errors.New("update indexer: could not find definition")
+	}
+
 	if indexerDefinition.IRC != nil {
-		server := indexerDefinition.IRC.Server
+		// add to irc server lookup table
-		channels := indexerDefinition.IRC.Channels
+		s.mapIRCServerDefinitionLookup(indexerDefinition.IRC.Server, indexerDefinition)
-		announcers := indexerDefinition.IRC.Announcers
 
-		for _, channel := range channels {
+		// check if it has api and add to api service
-			for _, announcer := range announcers {
+		if indexerDefinition.Enabled && indexerDefinition.HasApi() {
-				// format to server:channel:announcer
+			if err := s.apiService.AddClient(indexerDefinition.Identifier, indexerDefinition.SettingsMap); err != nil {
-				val := fmt.Sprintf("%v:%v:%v", server, channel, announcer)
+				s.log.Error().Stack().Err(err).Msgf("indexer.start: could not init api client for: '%v'", indexer.Identifier)
-				val = strings.ToLower(val)
-
-				s.mapIndexerIRCToName[val] = indexerIdentifier
 			}
 		}
 	}
+
+	// handle Torznab
+	if indexerDefinition.Implementation == "torznab" {
+		s.torznabIndexers[indexer.Identifier] = indexerDefinition
+	}
+
+	return nil
 }
 
 // mapIRCServerDefinitionLookup map irc stuff to indexer.name
@@ -343,7 +393,6 @@ func (s *service) mapIRCServerDefinitionLookup(ircServer string, indexerDefiniti
 
 // LoadIndexerDefinitions load definitions from golang embed fs
 func (s *service) LoadIndexerDefinitions() error {
-
 	entries, err := fs.ReadDir(Definitions, "definitions")
 	if err != nil {
 		s.log.Fatal().Stack().Msgf("failed reading directory: %s", err)
@@ -382,10 +431,10 @@ func (s *service) LoadIndexerDefinitions() error {
 			d.Implementation = "irc"
 		}
 
-		s.indexerDefinitions[d.Identifier] = d
+		s.definitions[d.Identifier] = *d
 	}
 
-	s.log.Debug().Msgf("Loaded %d indexer definitions", len(s.indexerDefinitions))
+	s.log.Debug().Msgf("Loaded %d indexer definitions", len(s.definitions))
 
 	return nil
 }
@@ -397,7 +446,6 @@ func (s *service) LoadCustomIndexerDefinitions() error {
 	}
 
 	outputDirRead, err := os.Open(s.config.CustomDefinitions)
-
 	if err != nil {
 		s.log.Warn().Stack().Msgf("failed opening custom definitions directory %q: %s", s.config.CustomDefinitions, err)
 		return nil
@@ -405,7 +453,6 @@ func (s *service) LoadCustomIndexerDefinitions() error {
 
 	defer outputDirRead.Close()
 
-	//entries, err := fs.ReadDir(Definitions, "definitions")
 	entries, err := outputDirRead.ReadDir(0)
 	if err != nil {
 		s.log.Fatal().Stack().Msgf("failed reading directory: %s", err)
@@ -446,7 +493,7 @@ func (s *service) LoadCustomIndexerDefinitions() error {
 			d.Implementation = "irc"
 		}
 
-		s.indexerDefinitions[d.Identifier] = d
+		s.definitions[d.Identifier] = *d
 
 		customCount++
 	}
@@ -472,7 +519,6 @@ func (s *service) GetIndexersByIRCNetwork(server string) []*domain.IndexerDefini
 }
 
 func (s *service) GetTorznabIndexers() []domain.IndexerDefinition {
-
 	indexerDefinitions := make([]domain.IndexerDefinition, 0)
 
 	for _, definition := range s.torznabIndexers {
@@ -485,20 +531,8 @@ func (s *service) GetTorznabIndexers() []domain.IndexerDefinition {
 }
 
 func (s *service) getDefinitionByName(name string) *domain.IndexerDefinition {
-
+	if v, ok := s.definitions[name]; ok {
-	if v, ok := s.indexerDefinitions[name]; ok {
+		return &v
-		return v
-	}
-
-	return nil
-}
-
-func (s *service) getDefinitionForAnnounce(name string) *domain.IndexerDefinition {
-
-	// map[network:channel:announcer] = indexer01
-
-	if v, ok := s.indexerDefinitions[name]; ok {
-		return v
 	}
 
 	return nil