feat(lists): add anilist support (#1949)

* fix(lists): clear selected list

* chore(web): improve onChange set values for select_wide

* feat(web): add anilist lists

* feat(web): Filter is required on ListForm

* fix(web): ListForm reset url when change type

* feat(lists): add anilist support

* feat(lists): filter duplicates for anilist

* feat(anilist): handle special characters

* fix(lists): better title matching
fix(lists): add alternatives to apostrophe replacement

* test(title): add some anime cases

* feat(anilist): replace unicodes with regex

* feat(lists): move additional anilist processing to autobrr instead of brr api

* feat(lists): clean Unicode Block “Latin Extended-A” chars

---------

Co-authored-by: martylukyy <35452459+martylukyy@users.noreply.github.com>

internal/domain/list.go

@@ -37,6 +37,7 @@ const (
 	ListTypePlaintext  ListType = "PLAINTEXT"
 	ListTypeTrakt      ListType = "TRAKT"
 	ListTypeSteam      ListType = "STEAM"
+	ListTypeAniList    ListType = "ANILIST"
 )
 
 type ListRefreshStatus string
@@ -108,7 +109,7 @@ func (l *List) ListTypeArr() bool {
 }
 
 func (l *List) ListTypeList() bool {
-	return l.Type == ListTypeMDBList || l.Type == ListTypeMetacritic || l.Type == ListTypePlaintext || l.Type == ListTypeTrakt || l.Type == ListTypeSteam
+	return l.Type == ListTypeMDBList || l.Type == ListTypeMetacritic || l.Type == ListTypePlaintext || l.Type == ListTypeTrakt || l.Type == ListTypeSteam || l.Type == ListTypeAniList
 }
 
 func (l *List) ShouldProcessItem(monitored bool) bool {

internal/list/process_list_anilist.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2021 - 2025, Ludvig Lundgren and the autobrr contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+package list
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/autobrr/autobrr/internal/domain"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	// including math and curreny symbols: $¤<~♡+=^ etc
+	symbolsRegexp          = regexp.MustCompile(`\p{S}`)
+	latin1SupplementRegexp = regexp.MustCompile(`[\x{0080}-\x{00FF}]`) // Unicode Block “Latin-1 Supplement”
+	latinExtendedARegexp   = regexp.MustCompile(`[\x{0100}-\x{017F}]`)
+)
+
+func (s *service) anilist(ctx context.Context, list *domain.List) error {
+	l := s.log.With().Str("type", "anilist").Str("list", list.Name).Logger()
+
+	if list.URL == "" {
+		return errors.New("no URL provided for AniList")
+	}
+
+	l.Debug().Msgf("fetching titles from %s", list.URL)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "could not make new request for URL: %s", list.URL)
+	}
+
+	list.SetRequestHeaders(req)
+
+	resp, err := s.httpClient.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "failed to fetch titles from URL: %s", list.URL)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("failed to fetch titles from URL: %s", list.URL)
+	}
+
+	var data []struct {
+		Romaji   string   `json:"romaji"`
+		English  string   `json:"english"`
+		Synonyms []string `json:"synonyms"`
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return errors.Wrapf(err, "failed to decode JSON data from URL: %s", list.URL)
+	}
+
+	// remove duplicates
+	titleSet := make(map[string]struct{})
+	for _, item := range data {
+		titlesToProcess := make(map[string]struct{})
+		titlesToProcess[item.Romaji] = struct{}{}
+		titlesToProcess[item.English] = struct{}{}
+		for _, synonym := range item.Synonyms {
+			titlesToProcess[synonym] = struct{}{}
+		}
+
+		for title := range titlesToProcess {
+			// replace unicode symbols, Unicode Block “Latin-1 Supplement” and Unicode Block “Latin Extended-A” chars by "?"
+			clearedTitle := symbolsRegexp.ReplaceAllString(title, "?")
+			clearedTitle = latin1SupplementRegexp.ReplaceAllString(clearedTitle, "?")
+			clearedTitle = latinExtendedARegexp.ReplaceAllString(clearedTitle, "?")
+			for _, processedTitle := range processTitle(clearedTitle, list.MatchRelease) {
+				titleSet[processedTitle] = struct{}{}
+			}
+		}
+	}
+
+	filterTitles := make([]string, 0, len(titleSet))
+	for title := range titleSet {
+		filterTitles = append(filterTitles, title)
+	}
+
+	if len(filterTitles) == 0 {
+		l.Debug().Msgf("no titles found to update for list: %v", list.Name)
+		return nil
+	}
+
+	sort.Strings(filterTitles)
+	joinedTitles := strings.Join(filterTitles, ",")
+
+	l.Trace().Str("titles", joinedTitles).Msgf("found %d titles", len(joinedTitles))
+
+	filterUpdate := domain.FilterUpdate{Shows: &joinedTitles}
+
+	if list.MatchRelease {
+		filterUpdate.Shows = &nullString
+		filterUpdate.MatchReleases = &joinedTitles
+	}
+
+	for _, filter := range list.Filters {
+		l.Debug().Msgf("updating filter: %v", filter.ID)
+
+		filterUpdate.ID = filter.ID
+
+		if err := s.filterSvc.UpdatePartial(ctx, filterUpdate); err != nil {
+			return errors.Wrapf(err, "error updating filter: %v", filter.ID)
+		}
+
+		l.Debug().Msgf("successfully updated filter: %v", filter.ID)
+	}
+
+	return nil
+}

internal/list/service.go

@@ -229,6 +229,9 @@ func (s *service) refreshList(ctx context.Context, listItem *domain.List) error
 	case domain.ListTypePlaintext:
 		err = s.plaintext(ctx, listItem)
 
+	case domain.ListTypeAniList:
+		err = s.anilist(ctx, listItem)
+
 	default:
 		err = errors.Errorf("unsupported list type: %s", listItem.Type)
 	}

internal/list/title.go

@@ -15,7 +15,7 @@ import (
 var (
 	replaceRegexp        = regexp.MustCompile(`[\p{P}\p{Z}\x{00C0}-\x{017E}\x{00AE}]`)
 	questionmarkRegexp   = regexp.MustCompile(`[?]{2,}`)
-	regionCodeRegexp     = regexp.MustCompile(`\(.+\)$`)
+	regionCodeRegexp     = regexp.MustCompile(`\(\S+\)`)
 	parenthesesEndRegexp = regexp.MustCompile(`\)$`)
 )
 
@@ -27,8 +27,8 @@ func processTitle(title string, matchRelease bool) []string {
 	}
 
 	// cleans year like (2020) from arr title
-	//var re = regexp.MustCompile(`(?m)\s(\(\d+\))`)
-	//title = re.ReplaceAllString(title, "")
+	// var re = regexp.MustCompile(`(?m)\s(\(\d+\))`)
+	// title = re.ReplaceAllString(title, "")
 
 	t := NewTitleSlice()
 
@@ -45,7 +45,7 @@ func processTitle(title string, matchRelease bool) []string {
 
 		// title with apostrophes removed and all non-alphanumeric characters replaced by "?"
 		noApostropheTitle := parenthesesEndRegexp.ReplaceAllString(title, "?")
-		noApostropheTitle = strings.ReplaceAll(noApostropheTitle, "'", "")
+		noApostropheTitle = strings.NewReplacer("'", "", "´", "", "`", "", "‘", "", "’", "").Replace(noApostropheTitle)
 		noApostropheTitle = replaceRegexp.ReplaceAllString(noApostropheTitle, "?")
 		noApostropheTitle = questionmarkRegexp.ReplaceAllString(noApostropheTitle, "*")
 
@@ -64,7 +64,7 @@ func processTitle(title string, matchRelease bool) []string {
 		// title with regions in parentheses and apostrophes removed and all non-alphanumeric characters replaced by "?"
 		removedRegionCodeNoApostrophe := regionCodeRegexp.ReplaceAllString(title, "")
 		removedRegionCodeNoApostrophe = strings.TrimRight(removedRegionCodeNoApostrophe, " ")
-		removedRegionCodeNoApostrophe = strings.ReplaceAll(removedRegionCodeNoApostrophe, "'", "")
+		removedRegionCodeNoApostrophe = strings.NewReplacer("'", "", "´", "", "`", "", "‘", "", "’", "").Replace(removedRegionCodeNoApostrophe)
 		removedRegionCodeNoApostrophe = replaceRegexp.ReplaceAllString(removedRegionCodeNoApostrophe, "?")
 		removedRegionCodeNoApostrophe = questionmarkRegexp.ReplaceAllString(removedRegionCodeNoApostrophe, "*")
 

internal/list/title_test.go

@@ -41,7 +41,7 @@ func Test_processTitle(t *testing.T) {
 				title:        "The Matrix -(Test)- Reloaded (2929)",
 				matchRelease: false,
 			},
-			want: []string{"The?Matrix*", "The?Matrix", "The?Matrix*Test*Reloaded*2929?", "The?Matrix*Test*Reloaded*2929"},
+			want: []string{"The?Matrix*Reloaded", "The?Matrix*Test*Reloaded*2929?", "The?Matrix*Test*Reloaded*2929"},
 		},
 		{
 			name: "test_04",
@@ -253,6 +253,30 @@ func Test_processTitle(t *testing.T) {
 			},
 			want: []string{"The?Office", "The?Office*US", "The?Office*US?"},
 		},
+		{
+			name: "test_30",
+			args: args{
+				title:        "this is him (can’t be anyone else)",
+				matchRelease: false,
+			},
+			want: []string{"this?is?him*can?t?be?anyone?else?", "this?is?him*can?t?be?anyone?else", "this?is?him*cant?be?anyone?else?", "this?is?him*cant?be?anyone?else"},
+		},
+		{
+			name: "test_31",
+			args: args{
+				title:        "solo leveling 2ª temporada -ergam-se das sombras-",
+				matchRelease: false,
+			},
+			want: []string{"solo?leveling?2ª?temporada*ergam?se?das?sombras", "solo?leveling?2ª?temporada*ergam?se?das?sombras?"},
+		},
+		{
+			name: "test_32",
+			args: args{
+				title:        "pokémon",
+				matchRelease: false,
+			},
+			want: []string{"pok?mon"},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {