feat(lists): add anilist support (#1949)

* fix(lists): clear selected list

* chore(web): improve onChange set values for select_wide

* feat(web): add anilist lists

* feat(web): Filter is required on ListForm

* fix(web): ListForm reset url when change type

* feat(lists): add anilist support

* feat(lists): filter duplicates for anilist

* feat(anilist): handle special characters

* fix(lists): better title matching
fix(lists): add alternatives to apostrophe replacement

* test(title): add some anime cases

* feat(anilist): replace unicodes with regex

* feat(lists): move additional anilist processing to autobrr instead of brr api

* feat(lists): clean Unicode Block “Latin Extended-A” chars

---------

Co-authored-by: martylukyy <35452459+martylukyy@users.noreply.github.com>
This commit is contained in:
Fabricio Silva 2025-01-31 18:17:23 +00:00 committed by GitHub
parent 5e2769639f
commit b724429b97
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 257 additions and 55 deletions

View file

@ -37,6 +37,7 @@ const (
ListTypePlaintext ListType = "PLAINTEXT"
ListTypeTrakt ListType = "TRAKT"
ListTypeSteam ListType = "STEAM"
ListTypeAniList ListType = "ANILIST"
)
type ListRefreshStatus string
@ -108,7 +109,7 @@ func (l *List) ListTypeArr() bool {
}
func (l *List) ListTypeList() bool {
return l.Type == ListTypeMDBList || l.Type == ListTypeMetacritic || l.Type == ListTypePlaintext || l.Type == ListTypeTrakt || l.Type == ListTypeSteam
return l.Type == ListTypeMDBList || l.Type == ListTypeMetacritic || l.Type == ListTypePlaintext || l.Type == ListTypeTrakt || l.Type == ListTypeSteam || l.Type == ListTypeAniList
}
func (l *List) ShouldProcessItem(monitored bool) bool {

View file

@ -0,0 +1,118 @@
// Copyright (c) 2021 - 2025, Ludvig Lundgren and the autobrr contributors.
// SPDX-License-Identifier: GPL-2.0-or-later
package list
import (
"context"
"encoding/json"
"net/http"
"regexp"
"sort"
"strings"
"github.com/autobrr/autobrr/internal/domain"
"github.com/pkg/errors"
)
var (
// including math and curreny symbols: $¤<~♡+=^ etc
symbolsRegexp = regexp.MustCompile(`\p{S}`)
latin1SupplementRegexp = regexp.MustCompile(`[\x{0080}-\x{00FF}]`) // Unicode Block “Latin-1 Supplement”
latinExtendedARegexp = regexp.MustCompile(`[\x{0100}-\x{017F}]`)
)
func (s *service) anilist(ctx context.Context, list *domain.List) error {
l := s.log.With().Str("type", "anilist").Str("list", list.Name).Logger()
if list.URL == "" {
return errors.New("no URL provided for AniList")
}
l.Debug().Msgf("fetching titles from %s", list.URL)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
if err != nil {
return errors.Wrapf(err, "could not make new request for URL: %s", list.URL)
}
list.SetRequestHeaders(req)
resp, err := s.httpClient.Do(req)
if err != nil {
return errors.Wrapf(err, "failed to fetch titles from URL: %s", list.URL)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.Errorf("failed to fetch titles from URL: %s", list.URL)
}
var data []struct {
Romaji string `json:"romaji"`
English string `json:"english"`
Synonyms []string `json:"synonyms"`
}
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return errors.Wrapf(err, "failed to decode JSON data from URL: %s", list.URL)
}
// remove duplicates
titleSet := make(map[string]struct{})
for _, item := range data {
titlesToProcess := make(map[string]struct{})
titlesToProcess[item.Romaji] = struct{}{}
titlesToProcess[item.English] = struct{}{}
for _, synonym := range item.Synonyms {
titlesToProcess[synonym] = struct{}{}
}
for title := range titlesToProcess {
// replace unicode symbols, Unicode Block “Latin-1 Supplement” and Unicode Block “Latin Extended-A” chars by "?"
clearedTitle := symbolsRegexp.ReplaceAllString(title, "?")
clearedTitle = latin1SupplementRegexp.ReplaceAllString(clearedTitle, "?")
clearedTitle = latinExtendedARegexp.ReplaceAllString(clearedTitle, "?")
for _, processedTitle := range processTitle(clearedTitle, list.MatchRelease) {
titleSet[processedTitle] = struct{}{}
}
}
}
filterTitles := make([]string, 0, len(titleSet))
for title := range titleSet {
filterTitles = append(filterTitles, title)
}
if len(filterTitles) == 0 {
l.Debug().Msgf("no titles found to update for list: %v", list.Name)
return nil
}
sort.Strings(filterTitles)
joinedTitles := strings.Join(filterTitles, ",")
l.Trace().Str("titles", joinedTitles).Msgf("found %d titles", len(joinedTitles))
filterUpdate := domain.FilterUpdate{Shows: &joinedTitles}
if list.MatchRelease {
filterUpdate.Shows = &nullString
filterUpdate.MatchReleases = &joinedTitles
}
for _, filter := range list.Filters {
l.Debug().Msgf("updating filter: %v", filter.ID)
filterUpdate.ID = filter.ID
if err := s.filterSvc.UpdatePartial(ctx, filterUpdate); err != nil {
return errors.Wrapf(err, "error updating filter: %v", filter.ID)
}
l.Debug().Msgf("successfully updated filter: %v", filter.ID)
}
return nil
}

View file

@ -229,6 +229,9 @@ func (s *service) refreshList(ctx context.Context, listItem *domain.List) error
case domain.ListTypePlaintext:
err = s.plaintext(ctx, listItem)
case domain.ListTypeAniList:
err = s.anilist(ctx, listItem)
default:
err = errors.Errorf("unsupported list type: %s", listItem.Type)
}

View file

@ -15,7 +15,7 @@ import (
var (
replaceRegexp = regexp.MustCompile(`[\p{P}\p{Z}\x{00C0}-\x{017E}\x{00AE}]`)
questionmarkRegexp = regexp.MustCompile(`[?]{2,}`)
regionCodeRegexp = regexp.MustCompile(`\(.+\)$`)
regionCodeRegexp = regexp.MustCompile(`\(\S+\)`)
parenthesesEndRegexp = regexp.MustCompile(`\)$`)
)
@ -27,8 +27,8 @@ func processTitle(title string, matchRelease bool) []string {
}
// cleans year like (2020) from arr title
//var re = regexp.MustCompile(`(?m)\s(\(\d+\))`)
//title = re.ReplaceAllString(title, "")
// var re = regexp.MustCompile(`(?m)\s(\(\d+\))`)
// title = re.ReplaceAllString(title, "")
t := NewTitleSlice()
@ -45,7 +45,7 @@ func processTitle(title string, matchRelease bool) []string {
// title with apostrophes removed and all non-alphanumeric characters replaced by "?"
noApostropheTitle := parenthesesEndRegexp.ReplaceAllString(title, "?")
noApostropheTitle = strings.ReplaceAll(noApostropheTitle, "'", "")
noApostropheTitle = strings.NewReplacer("'", "", "´", "", "`", "", "", "", "", "").Replace(noApostropheTitle)
noApostropheTitle = replaceRegexp.ReplaceAllString(noApostropheTitle, "?")
noApostropheTitle = questionmarkRegexp.ReplaceAllString(noApostropheTitle, "*")
@ -64,7 +64,7 @@ func processTitle(title string, matchRelease bool) []string {
// title with regions in parentheses and apostrophes removed and all non-alphanumeric characters replaced by "?"
removedRegionCodeNoApostrophe := regionCodeRegexp.ReplaceAllString(title, "")
removedRegionCodeNoApostrophe = strings.TrimRight(removedRegionCodeNoApostrophe, " ")
removedRegionCodeNoApostrophe = strings.ReplaceAll(removedRegionCodeNoApostrophe, "'", "")
removedRegionCodeNoApostrophe = strings.NewReplacer("'", "", "´", "", "`", "", "", "", "", "").Replace(removedRegionCodeNoApostrophe)
removedRegionCodeNoApostrophe = replaceRegexp.ReplaceAllString(removedRegionCodeNoApostrophe, "?")
removedRegionCodeNoApostrophe = questionmarkRegexp.ReplaceAllString(removedRegionCodeNoApostrophe, "*")

View file

@ -41,7 +41,7 @@ func Test_processTitle(t *testing.T) {
title: "The Matrix -(Test)- Reloaded (2929)",
matchRelease: false,
},
want: []string{"The?Matrix*", "The?Matrix", "The?Matrix*Test*Reloaded*2929?", "The?Matrix*Test*Reloaded*2929"},
want: []string{"The?Matrix*Reloaded", "The?Matrix*Test*Reloaded*2929?", "The?Matrix*Test*Reloaded*2929"},
},
{
name: "test_04",
@ -253,6 +253,30 @@ func Test_processTitle(t *testing.T) {
},
want: []string{"The?Office", "The?Office*US", "The?Office*US?"},
},
{
name: "test_30",
args: args{
title: "this is him (cant be anyone else)",
matchRelease: false,
},
want: []string{"this?is?him*can?t?be?anyone?else?", "this?is?him*can?t?be?anyone?else", "this?is?him*cant?be?anyone?else?", "this?is?him*cant?be?anyone?else"},
},
{
name: "test_31",
args: args{
title: "solo leveling 2ª temporada -ergam-se das sombras-",
matchRelease: false,
},
want: []string{"solo?leveling?2ª?temporada*ergam?se?das?sombras", "solo?leveling?2ª?temporada*ergam?se?das?sombras?"},
},
{
name: "test_32",
args: args{
title: "pokémon",
matchRelease: false,
},
want: []string{"pok?mon"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

View file

@ -78,12 +78,8 @@ export function SelectFieldCreatable<T>({ name, label, help, placeholder, toolti
// value={field?.value ? field.value : options.find(o => o.value == field?.value)}
value={field?.value ? { value: field.value, label: field.value } : field.value}
onChange={(newValue: unknown) => {
if (newValue) {
setFieldValue(field.name, (newValue as { value: string }).value);
}
else {
setFieldValue(field.name, "")
}
const option = newValue as { value: string };
setFieldValue(field.name, option?.value ?? "");
}}
options={[...[...options, { value: field.value, label: field.value }].reduce((map, obj) => map.set(obj.value, obj), new Map()).values()]}
/>
@ -143,12 +139,8 @@ export function SelectField<T>({ name, label, help, placeholder, options }: Sele
// value={field?.value ? field.value : options.find(o => o.value == field?.value)}
value={field?.value ? { value: field.value, label: field.value } : field.value}
onChange={(newValue: unknown) => {
if (newValue) {
setFieldValue(field.name, (newValue as { value: string }).value);
}
else {
setFieldValue(field.name, "")
}
const option = newValue as { value: string };
setFieldValue(field.name, option?.value ?? "");
}}
options={[...[...options, { value: field.value, label: field.value }].reduce((map, obj) => map.set(obj.value, obj), new Map()).values()]}
/>
@ -213,12 +205,8 @@ export function SelectFieldBasic<T>({ name, label, help, placeholder, required,
defaultValue={defaultValue}
value={field?.value && options.find(o => o.value == field?.value)}
onChange={(newValue: unknown) => {
if (newValue) {
setFieldValue(field.name, (newValue as { value: string }).value);
}
else {
setFieldValue(field.name, "")
}
const option = newValue as { value: string };
setFieldValue(field.name, option?.value ?? "");
}}
options={options}
/>
@ -247,7 +235,7 @@ interface ListFilterMultiSelectOption {
name: string;
}
export function ListFilterMultiSelectField({ name, label, help, tooltip, options }: MultiSelectFieldProps) {
export function ListFilterMultiSelectField({ name, label, help, tooltip, options, required }: MultiSelectFieldProps) {
return (
<div className="flex items-center space-y-1 p-4 sm:space-y-0 sm:grid sm:grid-cols-3 sm:gap-4">
<div>
@ -259,11 +247,12 @@ export function ListFilterMultiSelectField({ name, label, help, tooltip, options
{tooltip ? (
<DocsTooltip label={label}>{tooltip}</DocsTooltip>
) : label}
<common.RequiredField required={required} />
</div>
</label>
</div>
<div className="sm:col-span-2">
<Field name={name} type="select">
<Field name={name} type="select" required={required}>
{({
field,
form: { setFieldValue }

View file

@ -450,6 +450,10 @@ export const ListTypeOptions: OptionBasicTyped<ListType>[] = [
label: "Metacritic",
value: "METACRITIC"
},
{
label: "AniList",
value: "ANILIST"
},
];
export const ListTypeNameMap: Record<ListType, string> = {
@ -463,6 +467,7 @@ export const ListTypeNameMap: Record<ListType, string> = {
"METACRITIC": "Metacritic",
"STEAM": "Steam",
"PLAINTEXT": "Plaintext",
"ANILIST": "AniList",
};
export const NotificationTypeOptions: OptionBasicTyped<NotificationType>[] = [
@ -708,3 +713,18 @@ export const ListsMDBListOptions: OptionBasic[] = [
value: "https://mdblist.com/lists/garycrawfordgc/latest-tv-shows/json"
},
];
export const ListsAniListOptions: OptionBasic[] = [
{
label: "Current anime season",
value: "https://api.autobrr.com/lists/anilist/seasonal"
},
{
label: "Trending animes",
value: "https://api.autobrr.com/lists/anilist/trending"
},
{
label: "Next anime season",
value: "https://api.autobrr.com/lists/anilist/upcoming"
},
];

View file

@ -20,7 +20,9 @@ import {
DialogPanel,
DialogTitle,
Listbox,
ListboxButton, ListboxOption, ListboxOptions,
ListboxButton,
ListboxOption,
ListboxOptions,
Transition,
TransitionChild
} from "@headlessui/react";
@ -41,8 +43,9 @@ import {
ListsMDBListOptions,
ListsMetacriticOptions,
ListsTraktOptions,
ListTypeOptions, OptionBasicTyped,
SelectOption
ListsAniListOptions,
ListTypeOptions,
OptionBasicTyped
} from "@domain/constants";
import { DEBUG } from "@components/debug";
import {
@ -53,6 +56,7 @@ import {
import { classNames, sleep } from "@utils";
import {
ListFilterMultiSelectField,
SelectFieldBasic,
SelectFieldCreatable
} from "@components/inputs/select_wide";
import { DocsTooltip } from "@components/tooltips/DocsTooltip";
@ -215,15 +219,9 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) {
}
})}
value={field?.value && field.value.value}
onChange={(option: unknown) => {
// resetForm();
const opt = option as SelectOption;
// setFieldValue("name", option?.label ?? "")
setFieldValue(
field.name,
opt.value ?? ""
);
onChange={(newValue: unknown) => {
const option = newValue as { value: string };
setFieldValue(field.name, option?.value ?? "");
}}
options={ListTypeOptions}
/>
@ -235,7 +233,7 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) {
<SwitchGroupWide name="enabled" label="Enabled"/>
</div>
<ListTypeForm listType={values.type} clients={clients ?? []}/>
<ListTypeForm listType={values.type as ListType} clients={clients ?? []}/>
<div className="flex flex-col space-y-4 py-6 sm:py-0 sm:space-y-0">
<div className="border-t border-gray-200 dark:border-gray-700 py-4">
@ -248,7 +246,12 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) {
</p>
</div>
<ListFilterMultiSelectField name="filters" label="Filters" options={filterQuery.data?.map(f => ({ value: f.id, label: f.name })) ?? []} />
<ListFilterMultiSelectField
name="filters"
label="Filters"
required={true}
options={filterQuery.data?.map(f => ({ value: f.id, label: f.name })) ?? []}
/>
</div>
</div>
@ -422,10 +425,12 @@ export function ListUpdateForm({ isOpen, toggle, data }: UpdateFormProps<List>)
</p>
</div>
<ListFilterMultiSelectField name="filters" label="Filters" options={filterQuery.data?.map(f => ({
value: f.id,
label: f.name
})) ?? []}/>
<ListFilterMultiSelectField
name="filters"
label="Filters"
required={true}
options={filterQuery.data?.map(f => ({ value: f.id, label: f.name })) ?? []}
/>
</div>
</div>
@ -522,27 +527,27 @@ const SubmitButton = (props: SubmitButtonProps) => {
interface ListTypeFormProps {
listID?: number;
listType: string;
listType: ListType;
clients: DownloadClient[];
}
const ListTypeForm = (props: ListTypeFormProps) => {
const { setFieldValue } = useFormikContext();
const [prevActionType, setPrevActionType] = useState<string | null>(null);
const { listType } = props;
useEffect(() => {
// if (prevActionType !== null && prevActionType !== list.type && ListTypeOptions.map(l => l.value).includes(list.type)) {
if (prevActionType !== null && prevActionType !== listType && ListTypeOptions.map(l => l.value).includes(listType as ListType)) {
if (prevActionType !== null && prevActionType !== listType && ListTypeOptions.map(l => l.value).includes(listType)) {
// Reset the client_id field value
setFieldValue(`client_id`, 0);
setFieldValue('client_id', 0);
// Reset the url
setFieldValue('url', '');
}
setPrevActionType(listType);
}, [listType, prevActionType, setFieldValue]);
switch (props.listType) {
switch (listType) {
case "RADARR":
return <ListTypeArr {...props} />;
case "SONARR":
@ -563,6 +568,8 @@ const ListTypeForm = (props: ListTypeFormProps) => {
return <ListTypeMDBList />;
case "PLAINTEXT":
return <ListTypePlainText />;
case "ANILIST":
return <ListTypeAniList />;
default:
return null;
}
@ -677,6 +684,34 @@ function ListTypeTrakt() {
)
}
function ListTypeAniList() {
return (
<div className="border-t border-gray-200 dark:border-gray-700 py-4">
<div className="px-4 space-y-1">
<DialogTitle className="text-lg font-medium text-gray-900 dark:text-white">
Source list
</DialogTitle>
<p className="text-sm text-gray-500 dark:text-gray-400">
Use an AniList list from one of the default autobrr hosted lists.
</p>
</div>
<SelectFieldBasic
name="url"
label="List URL"
options={ListsAniListOptions.map(u => ({ value: u.value, label: u.label, key: u.label }))}
/>
<div className="space-y-1">
<fieldset>
<legend className="sr-only">Settings</legend>
<SwitchGroupWide name="match_release" label="Match Release" description="Use Match Releases field. Uses Movies/Shows field by default." />
</fieldset>
</div>
</div>
)
}
function ListTypePlainText() {
return (
<div className="border-t border-gray-200 dark:border-gray-700 py-4">
@ -689,8 +724,8 @@ function ListTypePlainText() {
</p>
</div>
<TextFieldWide
name="url"
<TextFieldWide
name="url"
label="List URL"
help="URL to a plain text file with one item per line"
placeholder="https://example.com/list.txt"

View file

@ -8,7 +8,8 @@ import { Link } from '@tanstack/react-router'
import {
Listbox,
ListboxButton,
ListboxOption, ListboxOptions,
ListboxOption,
ListboxOptions,
Menu,
MenuButton,
MenuItem,

View file

@ -41,4 +41,15 @@ interface ListCreate {
include_alternate_titles: boolean;
}
type ListType = "SONARR" | "RADARR" | "LIDARR" | "READARR" | "WHISPARR" | "MDBLIST" | "TRAKT" | "METACRITIC" | "STEAM" | "PLAINTEXT";
type ListType =
| "SONARR"
| "RADARR"
| "LIDARR"
| "READARR"
| "WHISPARR"
| "MDBLIST"
| "TRAKT"
| "METACRITIC"
| "STEAM"
| "PLAINTEXT"
| "ANILIST";