feat(lists): add anilist support (#1949)

* fix(lists): clear selected list

* chore(web): improve onChange set values for select_wide

* feat(web): add anilist lists

* feat(web): Filter is required on ListForm

* fix(web): ListForm reset url when change type

* feat(lists): add anilist support

* feat(lists): filter duplicates for anilist

* feat(anilist): handle special characters

* fix(lists): better title matching
fix(lists): add alternatives to apostrophe replacement

* test(title): add some anime cases

* feat(anilist): replace unicodes with regex

* feat(lists): move additional anilist processing to autobrr instead of brr api

* feat(lists): clean Unicode Block “Latin Extended-A” chars

---------

Co-authored-by: martylukyy <35452459+martylukyy@users.noreply.github.com>

internal/domain/list.go

@@ -37,6 +37,7 @@ const (
 	ListTypePlaintext  ListType = "PLAINTEXT"
 	ListTypeTrakt      ListType = "TRAKT"
 	ListTypeSteam      ListType = "STEAM"
+	ListTypeAniList    ListType = "ANILIST"
 )
 
 type ListRefreshStatus string
@@ -108,7 +109,7 @@ func (l *List) ListTypeArr() bool {
 }
 
 func (l *List) ListTypeList() bool {
-	return l.Type == ListTypeMDBList || l.Type == ListTypeMetacritic || l.Type == ListTypePlaintext || l.Type == ListTypeTrakt || l.Type == ListTypeSteam
+	return l.Type == ListTypeMDBList || l.Type == ListTypeMetacritic || l.Type == ListTypePlaintext || l.Type == ListTypeTrakt || l.Type == ListTypeSteam || l.Type == ListTypeAniList
 }
 
 func (l *List) ShouldProcessItem(monitored bool) bool {

internal/list/process_list_anilist.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2021 - 2025, Ludvig Lundgren and the autobrr contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+package list
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/autobrr/autobrr/internal/domain"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	// including math and curreny symbols: $¤<~♡+=^ etc
+	symbolsRegexp          = regexp.MustCompile(`\p{S}`)
+	latin1SupplementRegexp = regexp.MustCompile(`[\x{0080}-\x{00FF}]`) // Unicode Block “Latin-1 Supplement”
+	latinExtendedARegexp   = regexp.MustCompile(`[\x{0100}-\x{017F}]`)
+)
+
+func (s *service) anilist(ctx context.Context, list *domain.List) error {
+	l := s.log.With().Str("type", "anilist").Str("list", list.Name).Logger()
+
+	if list.URL == "" {
+		return errors.New("no URL provided for AniList")
+	}
+
+	l.Debug().Msgf("fetching titles from %s", list.URL)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "could not make new request for URL: %s", list.URL)
+	}
+
+	list.SetRequestHeaders(req)
+
+	resp, err := s.httpClient.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "failed to fetch titles from URL: %s", list.URL)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("failed to fetch titles from URL: %s", list.URL)
+	}
+
+	var data []struct {
+		Romaji   string   `json:"romaji"`
+		English  string   `json:"english"`
+		Synonyms []string `json:"synonyms"`
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return errors.Wrapf(err, "failed to decode JSON data from URL: %s", list.URL)
+	}
+
+	// remove duplicates
+	titleSet := make(map[string]struct{})
+	for _, item := range data {
+		titlesToProcess := make(map[string]struct{})
+		titlesToProcess[item.Romaji] = struct{}{}
+		titlesToProcess[item.English] = struct{}{}
+		for _, synonym := range item.Synonyms {
+			titlesToProcess[synonym] = struct{}{}
+		}
+
+		for title := range titlesToProcess {
+			// replace unicode symbols, Unicode Block “Latin-1 Supplement” and Unicode Block “Latin Extended-A” chars by "?"
+			clearedTitle := symbolsRegexp.ReplaceAllString(title, "?")
+			clearedTitle = latin1SupplementRegexp.ReplaceAllString(clearedTitle, "?")
+			clearedTitle = latinExtendedARegexp.ReplaceAllString(clearedTitle, "?")
+			for _, processedTitle := range processTitle(clearedTitle, list.MatchRelease) {
+				titleSet[processedTitle] = struct{}{}
+			}
+		}
+	}
+
+	filterTitles := make([]string, 0, len(titleSet))
+	for title := range titleSet {
+		filterTitles = append(filterTitles, title)
+	}
+
+	if len(filterTitles) == 0 {
+		l.Debug().Msgf("no titles found to update for list: %v", list.Name)
+		return nil
+	}
+
+	sort.Strings(filterTitles)
+	joinedTitles := strings.Join(filterTitles, ",")
+
+	l.Trace().Str("titles", joinedTitles).Msgf("found %d titles", len(joinedTitles))
+
+	filterUpdate := domain.FilterUpdate{Shows: &joinedTitles}
+
+	if list.MatchRelease {
+		filterUpdate.Shows = &nullString
+		filterUpdate.MatchReleases = &joinedTitles
+	}
+
+	for _, filter := range list.Filters {
+		l.Debug().Msgf("updating filter: %v", filter.ID)
+
+		filterUpdate.ID = filter.ID
+
+		if err := s.filterSvc.UpdatePartial(ctx, filterUpdate); err != nil {
+			return errors.Wrapf(err, "error updating filter: %v", filter.ID)
+		}
+
+		l.Debug().Msgf("successfully updated filter: %v", filter.ID)
+	}
+
+	return nil
+}

internal/list/service.go

@@ -229,6 +229,9 @@ func (s *service) refreshList(ctx context.Context, listItem *domain.List) error
 	case domain.ListTypePlaintext:
 		err = s.plaintext(ctx, listItem)
 
+	case domain.ListTypeAniList:
+		err = s.anilist(ctx, listItem)
+
 	default:
 		err = errors.Errorf("unsupported list type: %s", listItem.Type)
 	}

internal/list/title.go

@@ -15,7 +15,7 @@ import (
 var (
 	replaceRegexp        = regexp.MustCompile(`[\p{P}\p{Z}\x{00C0}-\x{017E}\x{00AE}]`)
 	questionmarkRegexp   = regexp.MustCompile(`[?]{2,}`)
-	regionCodeRegexp     = regexp.MustCompile(`\(.+\)$`)
+	regionCodeRegexp     = regexp.MustCompile(`\(\S+\)`)
 	parenthesesEndRegexp = regexp.MustCompile(`\)$`)
 )
 
@@ -27,8 +27,8 @@ func processTitle(title string, matchRelease bool) []string {
 	}
 
 	// cleans year like (2020) from arr title
-	//var re = regexp.MustCompile(`(?m)\s(\(\d+\))`)
+	// var re = regexp.MustCompile(`(?m)\s(\(\d+\))`)
-	//title = re.ReplaceAllString(title, "")
+	// title = re.ReplaceAllString(title, "")
 
 	t := NewTitleSlice()
 
@@ -45,7 +45,7 @@ func processTitle(title string, matchRelease bool) []string {
 
 		// title with apostrophes removed and all non-alphanumeric characters replaced by "?"
 		noApostropheTitle := parenthesesEndRegexp.ReplaceAllString(title, "?")
-		noApostropheTitle = strings.ReplaceAll(noApostropheTitle, "'", "")
+		noApostropheTitle = strings.NewReplacer("'", "", "´", "", "`", "", "‘", "", "’", "").Replace(noApostropheTitle)
 		noApostropheTitle = replaceRegexp.ReplaceAllString(noApostropheTitle, "?")
 		noApostropheTitle = questionmarkRegexp.ReplaceAllString(noApostropheTitle, "*")
 
@@ -64,7 +64,7 @@ func processTitle(title string, matchRelease bool) []string {
 		// title with regions in parentheses and apostrophes removed and all non-alphanumeric characters replaced by "?"
 		removedRegionCodeNoApostrophe := regionCodeRegexp.ReplaceAllString(title, "")
 		removedRegionCodeNoApostrophe = strings.TrimRight(removedRegionCodeNoApostrophe, " ")
-		removedRegionCodeNoApostrophe = strings.ReplaceAll(removedRegionCodeNoApostrophe, "'", "")
+		removedRegionCodeNoApostrophe = strings.NewReplacer("'", "", "´", "", "`", "", "‘", "", "’", "").Replace(removedRegionCodeNoApostrophe)
 		removedRegionCodeNoApostrophe = replaceRegexp.ReplaceAllString(removedRegionCodeNoApostrophe, "?")
 		removedRegionCodeNoApostrophe = questionmarkRegexp.ReplaceAllString(removedRegionCodeNoApostrophe, "*")
 

internal/list/title_test.go

@@ -41,7 +41,7 @@ func Test_processTitle(t *testing.T) {
 				title:        "The Matrix -(Test)- Reloaded (2929)",
 				matchRelease: false,
 			},
-			want: []string{"The?Matrix*", "The?Matrix", "The?Matrix*Test*Reloaded*2929?", "The?Matrix*Test*Reloaded*2929"},
+			want: []string{"The?Matrix*Reloaded", "The?Matrix*Test*Reloaded*2929?", "The?Matrix*Test*Reloaded*2929"},
 		},
 		{
 			name: "test_04",
@@ -253,6 +253,30 @@ func Test_processTitle(t *testing.T) {
 			},
 			want: []string{"The?Office", "The?Office*US", "The?Office*US?"},
 		},
+		{
+			name: "test_30",
+			args: args{
+				title:        "this is him (can’t be anyone else)",
+				matchRelease: false,
+			},
+			want: []string{"this?is?him*can?t?be?anyone?else?", "this?is?him*can?t?be?anyone?else", "this?is?him*cant?be?anyone?else?", "this?is?him*cant?be?anyone?else"},
+		},
+		{
+			name: "test_31",
+			args: args{
+				title:        "solo leveling 2ª temporada -ergam-se das sombras-",
+				matchRelease: false,
+			},
+			want: []string{"solo?leveling?2ª?temporada*ergam?se?das?sombras", "solo?leveling?2ª?temporada*ergam?se?das?sombras?"},
+		},
+		{
+			name: "test_32",
+			args: args{
+				title:        "pokémon",
+				matchRelease: false,
+			},
+			want: []string{"pok?mon"},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

web/src/components/inputs/select_wide.tsx

@@ -78,12 +78,8 @@ export function SelectFieldCreatable<T>({ name, label, help, placeholder, toolti
               // value={field?.value ? field.value : options.find(o => o.value == field?.value)}
               value={field?.value ? { value: field.value, label: field.value  } : field.value}
               onChange={(newValue: unknown) => {
-                if (newValue) {
+                const option = newValue as { value: string };
-                  setFieldValue(field.name, (newValue as { value: string }).value);
+                setFieldValue(field.name, option?.value ?? "");
-                }
-                else {
-                  setFieldValue(field.name, "")
-                }
               }}
               options={[...[...options, { value: field.value, label: field.value  }].reduce((map, obj) => map.set(obj.value, obj), new Map()).values()]}
             />
@@ -143,12 +139,8 @@ export function SelectField<T>({ name, label, help, placeholder, options }: Sele
               // value={field?.value ? field.value : options.find(o => o.value == field?.value)}
               value={field?.value ? { value: field.value, label: field.value  } : field.value}
               onChange={(newValue: unknown) => {
-                if (newValue) {
+                const option = newValue as { value: string };
-                  setFieldValue(field.name, (newValue as { value: string }).value);
+                setFieldValue(field.name, option?.value ?? "");
-                }
-                else {
-                  setFieldValue(field.name, "")
-                }
               }}
               options={[...[...options, { value: field.value, label: field.value  }].reduce((map, obj) => map.set(obj.value, obj), new Map()).values()]}
             />
@@ -213,12 +205,8 @@ export function SelectFieldBasic<T>({ name, label, help, placeholder, required,
               defaultValue={defaultValue}
               value={field?.value && options.find(o => o.value == field?.value)}
               onChange={(newValue: unknown) => {
-                if (newValue) {
+                const option = newValue as { value: string };
-                  setFieldValue(field.name, (newValue as { value: string }).value);
+                setFieldValue(field.name, option?.value ?? "");
-                }
-                else {
-                  setFieldValue(field.name, "")
-                }
               }}
               options={options}
             />
@@ -247,7 +235,7 @@ interface ListFilterMultiSelectOption {
   name: string;
 }
 
-export function ListFilterMultiSelectField({ name, label, help, tooltip, options }: MultiSelectFieldProps) {
+export function ListFilterMultiSelectField({ name, label, help, tooltip, options, required }: MultiSelectFieldProps) {
   return (
     <div className="flex items-center space-y-1 p-4 sm:space-y-0 sm:grid sm:grid-cols-3 sm:gap-4">
       <div>
@@ -259,11 +247,12 @@ export function ListFilterMultiSelectField({ name, label, help, tooltip, options
             {tooltip ? (
               <DocsTooltip label={label}>{tooltip}</DocsTooltip>
             ) : label}
+            <common.RequiredField required={required} />
           </div>
         </label>
       </div>
       <div className="sm:col-span-2">
-        <Field name={name} type="select">
+        <Field name={name} type="select" required={required}>
           {({
               field,
               form: { setFieldValue }

web/src/domain/constants.ts

@@ -450,6 +450,10 @@ export const ListTypeOptions: OptionBasicTyped<ListType>[] = [
     label: "Metacritic",
     value: "METACRITIC"
   },
+  {
+    label: "AniList",
+    value: "ANILIST"
+  },
 ];
 
 export const ListTypeNameMap: Record<ListType, string> = {
@@ -463,6 +467,7 @@ export const ListTypeNameMap: Record<ListType, string> = {
   "METACRITIC": "Metacritic",
   "STEAM": "Steam",
   "PLAINTEXT": "Plaintext",
+  "ANILIST": "AniList",
 };
 
 export const NotificationTypeOptions: OptionBasicTyped<NotificationType>[] = [
@@ -708,3 +713,18 @@ export const ListsMDBListOptions: OptionBasic[] = [
     value: "https://mdblist.com/lists/garycrawfordgc/latest-tv-shows/json"
   },
 ];
+
+export const ListsAniListOptions: OptionBasic[] = [
+  {
+    label: "Current anime season",
+    value: "https://api.autobrr.com/lists/anilist/seasonal"
+  },
+  {
+    label: "Trending animes",
+    value: "https://api.autobrr.com/lists/anilist/trending"
+  },
+  {
+    label: "Next anime season",
+    value: "https://api.autobrr.com/lists/anilist/upcoming"
+  },
+];

web/src/forms/settings/ListForms.tsx

@@ -20,7 +20,9 @@ import {
   DialogPanel,
   DialogTitle,
   Listbox,
-  ListboxButton, ListboxOption, ListboxOptions,
+  ListboxButton,
+  ListboxOption,
+  ListboxOptions,
   Transition,
   TransitionChild
 } from "@headlessui/react";
@@ -41,8 +43,9 @@ import {
   ListsMDBListOptions,
   ListsMetacriticOptions,
   ListsTraktOptions,
-  ListTypeOptions, OptionBasicTyped,
+  ListsAniListOptions,
-  SelectOption
+  ListTypeOptions,
+  OptionBasicTyped
 } from "@domain/constants";
 import { DEBUG } from "@components/debug";
 import {
@@ -53,6 +56,7 @@ import {
 import { classNames, sleep } from "@utils";
 import {
   ListFilterMultiSelectField,
+  SelectFieldBasic,
   SelectFieldCreatable
 } from "@components/inputs/select_wide";
 import { DocsTooltip } from "@components/tooltips/DocsTooltip";
@@ -215,15 +219,9 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) {
                                       }
                                     })}
                                     value={field?.value && field.value.value}
-                                    onChange={(option: unknown) => {
+                                    onChange={(newValue: unknown) => {
-                                      // resetForm();
+                                      const option = newValue as { value: string };
-
+                                      setFieldValue(field.name, option?.value ?? "");
-                                      const opt = option as SelectOption;
-                                      // setFieldValue("name", option?.label ?? "")
-                                      setFieldValue(
-                                        field.name,
-                                        opt.value ?? ""
-                                      );
                                     }}
                                     options={ListTypeOptions}
                                   />
@@ -235,7 +233,7 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) {
                           <SwitchGroupWide name="enabled" label="Enabled"/>
                         </div>
 
-                        <ListTypeForm listType={values.type} clients={clients ?? []}/>
+                        <ListTypeForm listType={values.type as ListType} clients={clients ?? []}/>
 
                         <div className="flex flex-col space-y-4 py-6 sm:py-0 sm:space-y-0">
                           <div className="border-t border-gray-200 dark:border-gray-700 py-4">
@@ -248,7 +246,12 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) {
                               </p>
                             </div>
 
-                            <ListFilterMultiSelectField name="filters" label="Filters" options={filterQuery.data?.map(f => ({ value: f.id, label: f.name })) ?? []} />
+                            <ListFilterMultiSelectField
+                              name="filters"
+                              label="Filters"
+                              required={true}
+                              options={filterQuery.data?.map(f => ({ value: f.id, label: f.name })) ?? []}
+                            />
 
                           </div>
                         </div>
@@ -422,10 +425,12 @@ export function ListUpdateForm({ isOpen, toggle, data }: UpdateFormProps<List>)
                                 </p>
                               </div>
 
-                              <ListFilterMultiSelectField name="filters" label="Filters" options={filterQuery.data?.map(f => ({
+                              <ListFilterMultiSelectField
-                                value: f.id,
+                                name="filters"
-                                label: f.name
+                                label="Filters"
-                              })) ?? []}/>
+                                required={true}
+                                options={filterQuery.data?.map(f => ({ value: f.id, label: f.name })) ?? []}
+                              />
 
                             </div>
                           </div>
@@ -522,27 +527,27 @@ const SubmitButton = (props: SubmitButtonProps) => {
 
 interface ListTypeFormProps {
   listID?: number;
-  listType: string;
+  listType: ListType;
   clients: DownloadClient[];
 }
 
 const ListTypeForm = (props: ListTypeFormProps) => {
   const { setFieldValue } = useFormikContext();
   const [prevActionType, setPrevActionType] = useState<string | null>(null);
-
   const { listType } = props;
 
   useEffect(() => {
-    // if (prevActionType !== null && prevActionType !== list.type && ListTypeOptions.map(l => l.value).includes(list.type)) {
+    if (prevActionType !== null && prevActionType !== listType && ListTypeOptions.map(l => l.value).includes(listType)) {
-    if (prevActionType !== null && prevActionType !== listType && ListTypeOptions.map(l => l.value).includes(listType as ListType)) {
       // Reset the client_id field value
-      setFieldValue(`client_id`, 0);
+      setFieldValue('client_id', 0);
+      // Reset the  url
+      setFieldValue('url', '');
     }
 
     setPrevActionType(listType);
   }, [listType, prevActionType, setFieldValue]);
 
-  switch (props.listType) {
+  switch (listType) {
     case "RADARR":
       return <ListTypeArr {...props} />;
     case "SONARR":
@@ -563,6 +568,8 @@ const ListTypeForm = (props: ListTypeFormProps) => {
       return <ListTypeMDBList />;
     case "PLAINTEXT":
       return <ListTypePlainText />;
+    case "ANILIST":
+        return <ListTypeAniList />;
     default:
       return null;
   }
@@ -677,6 +684,34 @@ function ListTypeTrakt() {
   )
 }
 
+function ListTypeAniList() {
+  return (
+    <div className="border-t border-gray-200 dark:border-gray-700 py-4">
+      <div className="px-4 space-y-1">
+        <DialogTitle className="text-lg font-medium text-gray-900 dark:text-white">
+          Source list
+        </DialogTitle>
+        <p className="text-sm text-gray-500 dark:text-gray-400">
+          Use an AniList list from one of the default autobrr hosted lists.
+        </p>
+      </div>
+
+      <SelectFieldBasic
+        name="url"
+        label="List URL"
+        options={ListsAniListOptions.map(u => ({ value: u.value, label: u.label, key: u.label }))}
+      />
+
+      <div className="space-y-1">
+        <fieldset>
+          <legend className="sr-only">Settings</legend>
+          <SwitchGroupWide name="match_release" label="Match Release" description="Use Match Releases field. Uses Movies/Shows field by default." />
+        </fieldset>
+      </div>
+    </div>
+  )
+}
+
 function ListTypePlainText() {
   return (
     <div className="border-t border-gray-200 dark:border-gray-700 py-4">
@@ -689,8 +724,8 @@ function ListTypePlainText() {
         </p>
       </div>
 
-      <TextFieldWide 
+      <TextFieldWide
-        name="url" 
+        name="url"
         label="List URL"
         help="URL to a plain text file with one item per line"
         placeholder="https://example.com/list.txt"

web/src/screens/filters/List.tsx

@@ -8,7 +8,8 @@ import { Link } from '@tanstack/react-router'
 import {
   Listbox,
   ListboxButton,
-  ListboxOption, ListboxOptions,
+  ListboxOption,
+  ListboxOptions,
   Menu,
   MenuButton,
   MenuItem,

web/src/types/List.d.ts

@@ -41,4 +41,15 @@ interface ListCreate {
   include_alternate_titles: boolean;
 }
 
-type ListType = "SONARR" | "RADARR" | "LIDARR" | "READARR" | "WHISPARR" | "MDBLIST" | "TRAKT" | "METACRITIC" | "STEAM" | "PLAINTEXT";
+type ListType =
+  | "SONARR"
+  | "RADARR"
+  | "LIDARR"
+  | "READARR"
+  | "WHISPARR"
+  | "MDBLIST"
+  | "TRAKT"
+  | "METACRITIC"
+  | "STEAM"
+  | "PLAINTEXT"
+  | "ANILIST";