feat(indexers): load custom definitions (#186)

cmd/autobrr/main.go

@@ -80,7 +80,7 @@ func main() {
 		downloadClientService = download_client.NewService(downloadClientRepo)
 		actionService         = action.NewService(actionRepo, downloadClientService, bus)
 		apiService            = indexer.NewAPIService()
-		indexerService        = indexer.NewService(indexerRepo, apiService)
+		indexerService        = indexer.NewService(cfg, indexerRepo, apiService)
 		filterService         = filter.NewService(filterRepo, actionRepo, apiService, indexerService)
 		releaseService        = release.NewService(releaseRepo, actionService)
 		ircService            = irc.NewService(ircRepo, filterService, indexerService, releaseService)

config.toml

@@ -38,3 +38,7 @@ logLevel = "TRACE"
 # Session secret
 #
 sessionSecret = "secret-session-key"
+
+# Custom definitions
+#
+#customDefinitions = "test/definitions"

internal/config/config.go

@@ -16,12 +16,13 @@ var Config domain.Config
 
 func Defaults() domain.Config {
 	return domain.Config{
-		Host:          "localhost",
-		Port:          8989,
-		LogLevel:      "DEBUG",
-		LogPath:       "",
-		BaseURL:       "/",
-		SessionSecret: "secret-session-key",
+		Host:              "localhost",
+		Port:              8989,
+		LogLevel:          "TRACE",
+		LogPath:           "",
+		BaseURL:           "/",
+		SessionSecret:     "secret-session-key",
+		CustomDefinitions: "",
 	}
 }
 

internal/domain/config.go

@@ -1,10 +1,11 @@
 package domain
 
 type Config struct {
-	Host          string `toml:"host"`
-	Port          int    `toml:"port"`
-	LogLevel      string `toml:"logLevel"`
-	LogPath       string `toml:"logPath"`
-	BaseURL       string `toml:"baseUrl"`
-	SessionSecret string `toml:"sessionSecret"`
+	Host              string `toml:"host"`
+	Port              int    `toml:"port"`
+	LogLevel          string `toml:"logLevel"`
+	LogPath           string `toml:"logPath"`
+	BaseURL           string `toml:"baseUrl"`
+	SessionSecret     string `toml:"sessionSecret"`
+	CustomDefinitions string `toml:"customDefinitions"`
 }

internal/indexer/service.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"io/fs"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/rs/zerolog/log"
@@ -26,6 +28,7 @@ type Service interface {
 }
 
 type service struct {
+	config     domain.Config
 	repo       domain.IndexerRepo
 	apiService APIService
 
@@ -38,8 +41,9 @@ type service struct {
 	lookupIRCServerDefinition map[string]map[string]domain.IndexerDefinition
 }
 
-func NewService(repo domain.IndexerRepo, apiService APIService) Service {
+func NewService(config domain.Config, repo domain.IndexerRepo, apiService APIService) Service {
 	return &service{
+		config:                    config,
 		repo:                      repo,
 		apiService:                apiService,
 		indexerDefinitions:        make(map[string]domain.IndexerDefinition),
@@ -193,6 +197,14 @@ func (s *service) Start() error {
 		return err
 	}
 
+	if s.config.CustomDefinitions != "" {
+		// load custom indexer definitions
+		err = s.LoadCustomIndexerDefinitions()
+		if err != nil {
+			return fmt.Errorf("could not load custom indexer definitions: %w", err)
+		}
+	}
+
 	// load the indexers' setup by the user
 	indexerDefinitions, err := s.GetAll()
 	if err != nil {
@@ -334,6 +346,59 @@ func (s *service) LoadIndexerDefinitions() error {
 	return nil
 }
 
+// LoadCustomIndexerDefinitions load definitions from custom path
+func (s *service) LoadCustomIndexerDefinitions() error {
+	if s.config.CustomDefinitions == "" {
+		return nil
+	}
+
+	outputDirRead, _ := os.Open(s.config.CustomDefinitions)
+
+	//entries, err := fs.ReadDir(Definitions, "definitions")
+	entries, err := outputDirRead.ReadDir(0)
+	if err != nil {
+		log.Fatal().Stack().Msgf("failed reading directory: %s", err)
+	}
+
+	if len(entries) == 0 {
+		log.Fatal().Stack().Msgf("failed reading directory: %s", err)
+		return err
+	}
+
+	customCount := 0
+
+	for _, f := range entries {
+		file := filepath.Join(s.config.CustomDefinitions, f.Name())
+
+		if strings.Contains(f.Name(), ".yaml") {
+			log.Trace().Msgf("parsing custom: %v", file)
+
+			var d domain.IndexerDefinition
+
+			//data, err := fs.ReadFile(Definitions, filePath)
+			data, err := os.ReadFile(file)
+			if err != nil {
+				log.Error().Stack().Err(err).Msgf("failed reading file: %v", file)
+				return err
+			}
+
+			err = yaml.Unmarshal(data, &d)
+			if err != nil {
+				log.Error().Stack().Err(err).Msgf("failed unmarshal file: %v", file)
+				return err
+			}
+
+			s.indexerDefinitions[d.Identifier] = d
+
+			customCount++
+		}
+	}
+
+	log.Debug().Msgf("Loaded %d custom indexer definitions", customCount)
+
+	return nil
+}
+
 func (s *service) GetIndexersByIRCNetwork(server string) []domain.IndexerDefinition {
 	server = strings.ToLower(server)
 

test/definitions/mock.yaml

@@ -0,0 +1,62 @@
+---
+#id: mock
+name: MockIndexer
+identifier: mock
+description: MockIndexer is a mock indexer.
+language: en-us
+urls:
+  - http://localhost.test
+privacy: private
+protocol: torrent
+supports:
+  - irc
+source: custom
+settings:
+  - name: rsskey
+    type: secret
+    label: RSS key
+    help: "Go to your profile and copy your RSS key"
+    regex: /([\da-fA-F]{20})
+
+irc:
+  network: LocalHost
+  server: localhost
+  port: 6697
+  tls: true
+  channels:
+    - "#announces"
+  announcers:
+    - _AnnounceBot_
+  settings:
+    - name: nickserv.account
+      type: text
+      required: false
+      label: NickServ Account
+      help: NickServ account. Make sure to group your user and bot. Eg. user_bot
+    - name: nickserv.password
+      type: secret
+      required: false
+      label: NickServ Password
+      help: NickServ password
+
+parse:
+  type: single
+  lines:
+    - test:
+        - "New Torrent Announcement: <PC :: Iso>  Name:'debian live 10 6 0 amd64 standard iso' uploaded by 'Anonymous' -  http://www.localhost.test/torrent/000000"
+        - "New Torrent Announcement: <PC :: Iso>  Name:'debian live 10 6 0 amd64 standard iso' uploaded by 'Anonymous' freeleech -  http://www.localhost.test/torrent/000000"
+      pattern: New Torrent Announcement:\s*<([^>]*)>\s*Name:'(.*)' uploaded by '([^']*)'\s*(freeleech)*\s*-\s*(https?\:\/\/[^\/]+\/)torrent\/(\d+)
+      vars:
+        - category
+        - torrentName
+        - uploader
+        - freeleech
+        - baseUrl
+        - torrentId
+
+  match:
+    torrenturl: "{{ .baseUrl }}rss/download/{{ .torrentId }}/{{ .rsskey }}/{{ .torrentName }}.torrent"
+    encode:
+      - torrentName
+
+