feat(filters): skip duplicates (#1711)

* feat(filters): skip duplicates

* fix: add interface instead of any

* fix(filters): tonullint

* feat(filters): skip dupes check month day

* chore: cleanup

* feat(db): set autoincrement id

* feat(filters): add repack and proper to dupe profile

* feat(filters): add default dupe profiles

* feat(duplicates): check audio and website

* feat(duplicates): update tests

* feat(duplicates): add toggles on addform

* feat(duplicates): fix sqlite upgrade path and initialize duplicate profiles

* feat(duplicates): simplify sqlite upgrade

avoiding temp table and unwieldy select.  Besides, FK constraints
are turned off anyway in #229.

* feat(duplicates): change CheckIsDuplicateRelease treatment of PROPER and REPACK

"Proper" and "Repack" are not parallel to the other conditions like "Title",
so they do not belong as dedup conditions.  "PROPER" means there was an issue in
the previous release, and so a PROPER is never a duplicate, even if it replaces
another PROPER.  Similarly, "REPACK" means there was an issue in the previous
release by that group, and so it is a duplicate only if we previously took a
release from a DIFFERENT group.

I have not removed Proper and Repack from the UI or the schema yet.

* feat(duplicates): update postgres schema to match sqlite

* feat(duplicates): fix web build errors

* feat(duplicates): fix postgres errors

* feat(filters): do leftjoin for duplicate profile

* fix(filters): partial update dupe profile

* go fmt `internal/domain/filter.go`

* feat(duplicates): restore straightforward logic for proper/repack

* feat(duplicates): remove mostly duplicate TV duplicate profiles

Having one profile seems the cleanest.  If somebody wants multiple
resolutions then they can add Resolution to the duplicate profile.
Tested this profile with both weekly episodic releases and daily
show releases.

* feat(release): add db indexes and sub_title

* feat(release): add IsDuplicate tests

* feat(release): update action handler

* feat(release): add more tests for skip duplicates

* feat(duplicates): check audio

* feat(duplicates): add more tests

* feat(duplicates): match edition cut and more

* fix(duplicates): tests

* fix(duplicates): missing imports

* fix(duplicates): tests

* feat(duplicates): handle sub_title edition and language in ui

* fix(duplicates): tests

* feat(duplicates): check name against normalized hash

* fix(duplicates): tests

* chore: update .gitignore to ignore .pnpm-store

* fix: tests

* fix(filters): tests

* fix: bad conflict merge

* fix: update release type in test

* fix: use vendored hot-toast

* fix: release_test.go

* fix: rss_test.go

* feat(duplicates): improve title hashing for unique check

* feat(duplicates): further improve title hashing for unique check with lang

* feat(duplicates): fix tests

* feat(duplicates): add macros IsDuplicate and DuplicateProfile ID and name

* feat(duplicates): add normalized hash match option

* fix: headlessui-state prop warning

* fix(duplicates): add missing year in daily ep normalize

* fix(duplicates): check rejections len

---------

Co-authored-by: ze0s <ze0s@riseup.net>

web/src/forms/_shared.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2021 - 2024, Ludvig Lundgren and the autobrr contributors.
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+export interface AddFormProps {
+  isOpen: boolean;
+  toggle: () => void;
+}
+
+export interface UpdateFormProps<T> {
+  isOpen: boolean;
+  toggle: () => void;
+  data: T;
+}

web/src/forms/filters/FilterAddForm.tsx

@@ -16,14 +16,9 @@ import { FilterKeys } from "@api/query_keys";
 import { DEBUG } from "@components/debug";
 import { toast } from "@components/hot-toast";
 import Toast from "@components/notifications/Toast";
+import { AddFormProps } from "@forms/_shared";
 
-
-interface filterAddFormProps {
-    isOpen: boolean;
-    toggle: () => void;
-}
-
-export function FilterAddForm({ isOpen, toggle }: filterAddFormProps) {
+export function FilterAddForm({ isOpen, toggle }: AddFormProps) {
   const inputRef = useRef(null)
   const queryClient = useQueryClient();
   const navigate = useNavigate();

web/src/forms/settings/APIKeyAddForm.tsx

@@ -15,13 +15,9 @@ import { ApiKeys } from "@api/query_keys";
 import { DEBUG } from "@components/debug";
 import { toast } from "@components/hot-toast";
 import Toast from "@components/notifications/Toast";
+import { AddFormProps } from "@forms/_shared";
 
-interface apiKeyAddFormProps {
-  isOpen: boolean;
-  toggle: () => void;
-}
-
-export function APIKeyAddForm({ isOpen, toggle }: apiKeyAddFormProps) {
+export function APIKeyAddForm({ isOpen, toggle }: AddFormProps) {
   const queryClient = useQueryClient();
 
   const mutation = useMutation({

web/src/forms/settings/DownloadClientForms.tsx

@@ -27,6 +27,7 @@ import {
 } from "@components/inputs";
 import { DocsLink, ExternalLink } from "@components/ExternalLink";
 import { SelectFieldBasic } from "@components/inputs/select_wide";
+import { AddFormProps, UpdateFormProps } from "@forms/_shared";
 
 interface InitialValuesSettings {
   basic?: {
@@ -691,12 +692,7 @@ function DownloadClientFormButtons({
   );
 }
 
-interface formProps {
-  isOpen: boolean;
-  toggle: () => void;
-}
-
-export function DownloadClientAddForm({ isOpen, toggle }: formProps) {
+export function DownloadClientAddForm({ isOpen, toggle }: AddFormProps) {
   const [isTesting, setIsTesting] = useState(false);
   const [isSuccessfulTest, setIsSuccessfulTest] = useState(false);
   const [isErrorTest, setIsErrorTest] = useState(false);
@@ -856,13 +852,7 @@ export function DownloadClientAddForm({ isOpen, toggle }: formProps) {
   );
 }
 
-interface updateFormProps {
-  isOpen: boolean;
-  toggle: () => void;
-  client: DownloadClient;
-}
-
-export function DownloadClientUpdateForm({ client, isOpen, toggle }: updateFormProps) {
+export function DownloadClientUpdateForm({ isOpen, toggle, data: client}: UpdateFormProps<DownloadClient>) {
   const [isTesting, setIsTesting] = useState(false);
   const [isSuccessfulTest, setIsSuccessfulTest] = useState(false);
   const [isErrorTest, setIsErrorTest] = useState(false);

web/src/forms/settings/FeedForms.tsx

@@ -18,13 +18,7 @@ import { componentMapType } from "./DownloadClientForms";
 import { sleep } from "@utils";
 import { ImplementationBadges } from "@screens/settings/Indexer";
 import { FeedDownloadTypeOptions } from "@domain/constants";
-
-
-interface UpdateProps {
-  isOpen: boolean;
-  toggle: () => void;
-  feed: Feed;
-}
+import { UpdateFormProps } from "@forms/_shared";
 
 interface InitialValues {
   id: number;
@@ -41,7 +35,8 @@ interface InitialValues {
   settings: FeedSettings;
 }
 
-export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) {
+export function FeedUpdateForm({ isOpen, toggle, data}: UpdateFormProps<Feed>) {
+  const feed = data;
   const [isTesting, setIsTesting] = useState(false);
   const [isTestSuccessful, setIsSuccessfulTest] = useState(false);
   const [isTestError, setIsErrorTest] = useState(false);

web/src/forms/settings/IndexerForms.tsx

@@ -25,6 +25,7 @@ import { FeedDownloadTypeOptions } from "@domain/constants";
 import { DocsLink } from "@components/ExternalLink";
 import * as common from "@components/inputs/common";
 import { SelectField } from "@forms/settings/IrcForms";
+import { AddFormProps, UpdateFormProps } from "@forms/_shared";
 
 // const isRequired = (message: string) => (value?: string | undefined) => (!!value ? undefined : message);
 
@@ -255,12 +256,7 @@ type SelectValue = {
   value: string;
 };
 
-export interface AddProps {
-  isOpen: boolean;
-  toggle: () => void;
-}
-
-export function IndexerAddForm({ isOpen, toggle }: AddProps) {
+export function IndexerAddForm({ isOpen, toggle }: AddFormProps) {
   const [indexer, setIndexer] = useState<IndexerDefinition>({} as IndexerDefinition);
 
   const queryClient = useQueryClient();
@@ -729,13 +725,7 @@ interface IndexerUpdateInitialValues {
   }
 }
 
-interface UpdateProps {
-  isOpen: boolean;
-  toggle: () => void;
-  indexer: IndexerDefinition;
-}
-
-export function IndexerUpdateForm({ isOpen, toggle, indexer }: UpdateProps) {
+export function IndexerUpdateForm({ isOpen, toggle, data: indexer }: UpdateFormProps<IndexerDefinition>) {
   const queryClient = useQueryClient();
 
   const proxies = useQuery(ProxiesQueryOptions());

web/src/forms/settings/IrcForms.tsx

@@ -22,6 +22,7 @@ import Toast from "@components/notifications/Toast";
 import * as common from "@components/inputs/common";
 import { classNames } from "@utils";
 import { ProxiesQueryOptions } from "@api/queries";
+import { AddFormProps, UpdateFormProps } from "@forms/_shared";
 
 interface ChannelsFieldArrayProps {
   channels: IrcChannel[];
@@ -122,11 +123,6 @@ interface IrcNetworkAddFormValues {
     channels: IrcChannel[];
 }
 
-interface AddFormProps {
-  isOpen: boolean;
-  toggle: () => void;
-}
-
 export function IrcNetworkAddForm({ isOpen, toggle }: AddFormProps) {
   const queryClient = useQueryClient();
 
@@ -275,17 +271,11 @@ interface IrcNetworkUpdateFormValues {
     proxy_id: number;
 }
 
-interface IrcNetworkUpdateFormProps {
-    isOpen: boolean;
-    toggle: () => void;
-    network: IrcNetwork;
-}
-
 export function IrcNetworkUpdateForm({
   isOpen,
   toggle,
-  network
-}: IrcNetworkUpdateFormProps) {
+  data: network
+}: UpdateFormProps<IrcNetwork>) {
   const queryClient = useQueryClient();
 
   const proxies = useQuery(ProxiesQueryOptions());

web/src/forms/settings/NotificationForms.tsx

@@ -23,6 +23,7 @@ import * as common from "@components/inputs/common";
 import { NumberFieldWide, PasswordFieldWide, SwitchGroupWide, TextFieldWide } from "@components/inputs";
 
 import { componentMapType } from "./DownloadClientForms";
+import { AddFormProps, UpdateFormProps } from "@forms/_shared";
 
 function FormFieldsDiscord() {
   return (
@@ -311,12 +312,7 @@ interface NotificationAddFormValues {
     enabled: boolean;
 }
 
-interface AddProps {
-    isOpen: boolean;
-    toggle: () => void;
-}
-
-export function NotificationAddForm({ isOpen, toggle }: AddProps) {
+export function NotificationAddForm({ isOpen, toggle }: AddFormProps) {
   const queryClient = useQueryClient();
 
   const createMutation = useMutation({
@@ -565,12 +561,6 @@ const EventCheckBoxes = () => (
   </fieldset>
 );
 
-interface UpdateProps {
-    isOpen: boolean;
-    toggle: () => void;
-    notification: ServiceNotification;
-}
-
 interface InitialValues {
   id: number;
   enabled: boolean;
@@ -587,7 +577,7 @@ interface InitialValues {
   username?: string
 }
 
-export function NotificationUpdateForm({ isOpen, toggle, notification }: UpdateProps) {
+export function NotificationUpdateForm({ isOpen, toggle, data: notification }: UpdateFormProps<ServiceNotification>) {
   const queryClient = useQueryClient();
 
   const mutation = useMutation({

web/src/forms/settings/ProxyForms.tsx

@@ -9,7 +9,7 @@ import { Dialog, DialogPanel, DialogTitle, Transition, TransitionChild } from "@
 import { XMarkIcon } from "@heroicons/react/24/solid";
 import { useMutation, useQueryClient } from "@tanstack/react-query";
 
-import { AddProps } from "@forms/settings/IndexerForms";
+import { AddFormProps } from "@forms/_shared";
 import { DEBUG } from "@components/debug.tsx";
 import { PasswordFieldWide, SwitchGroupWide, TextFieldWide } from "@components/inputs";
 import { SelectFieldBasic } from "@components/inputs/select_wide";
@@ -20,7 +20,7 @@ import { toast } from "@components/hot-toast";
 import Toast from "@components/notifications/Toast";
 import { SlideOver } from "@components/panels";
 
-export function ProxyAddForm({ isOpen, toggle }: AddProps) {
+export function ProxyAddForm({ isOpen, toggle }: AddFormProps) {
   const queryClient = useQueryClient();
 
   const createMutation = useMutation({

web/src/forms/settings/ReleaseForms.tsx

@@ -0,0 +1,198 @@
+/*
+ * Copyright (c) 2021 - 2024, Ludvig Lundgren and the autobrr contributors.
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { APIClient } from "@api/APIClient.ts";
+import { ReleaseProfileDuplicateKeys } from "@api/query_keys.ts";
+import { toast } from "@components/hot-toast";
+import Toast from "@components/notifications/Toast.tsx";
+import { SwitchGroupWide, TextFieldWide } from "@components/inputs";
+import { SlideOver } from "@components/panels";
+import { AddFormProps, UpdateFormProps } from "@forms/_shared";
+
+export function ReleaseProfileDuplicateAddForm({ isOpen, toggle }: AddFormProps) {
+  const queryClient = useQueryClient();
+
+  const addMutation = useMutation({
+    mutationFn: (profile: ReleaseProfileDuplicate) => APIClient.release.profiles.duplicates.store(profile),
+    onSuccess: () => {
+      queryClient.invalidateQueries({ queryKey: ReleaseProfileDuplicateKeys.lists() });
+      toast.custom((t) => <Toast type="success" body="Profile was added" t={t} />);
+
+      toggle();
+    },
+    onError: () => {
+      toast.custom((t) => <Toast type="error" body="Profile could not be added" t={t} />);
+    }
+  });
+
+  const onSubmit = (data: unknown) => addMutation.mutate(data as ReleaseProfileDuplicate);
+
+  const initialValues: ReleaseProfileDuplicate = {
+    id: 0,
+    name: "",
+    protocol: false,
+    release_name: false,
+    hash: false,
+    title: false,
+    sub_title: false,
+    year: false,
+    month: false,
+    day: false,
+    source: false,
+    resolution: false,
+    codec: false,
+    container: false,
+    dynamic_range: false,
+    audio: false,
+    group: false,
+    season: false,
+    episode: false,
+    website: false,
+    proper: false,
+    repack: false,
+    edition: false,
+    language: false,
+  };
+
+  return (
+    <SlideOver
+      type="CREATE"
+      title="Duplicate Profile"
+      isOpen={isOpen}
+      toggle={toggle}
+      onSubmit={onSubmit}
+      initialValues={initialValues}
+    >
+      {() => (
+        <div className="py-2 space-y-6 sm:py-0 sm:space-y-0 divide-y divide-gray-200 dark:divide-gray-700">
+          <TextFieldWide required name="name" label="Name"/>
+
+          <SwitchGroupWide name="release_name" label="Release name" description="Full release name" />
+          <SwitchGroupWide name="hash" label="Hash" description="Normalized hash of the release name. Use with Releae name for exact match" />
+          <SwitchGroupWide name="title" label="Title" description="Parsed title" />
+          <SwitchGroupWide name="sub_title" label="Sub Title" description="Parsed Sub Title like Episode Name" />
+          <SwitchGroupWide name="year" label="Year" />
+          <SwitchGroupWide name="month" label="Month" description="For daily releases" />
+          <SwitchGroupWide name="day" label="Day" description="For daily releases" />
+          <SwitchGroupWide name="source" label="Source" />
+          <SwitchGroupWide name="resolution" label="Resolution" />
+          <SwitchGroupWide name="codec" label="Codec" />
+          <SwitchGroupWide name="container" label="Container" />
+          <SwitchGroupWide name="dynamic_range" label="Dynamic Range" />
+          <SwitchGroupWide name="audio" label="Audio" />
+          <SwitchGroupWide name="group" label="Group" description="Release group" />
+          <SwitchGroupWide name="season" label="Season" />
+          <SwitchGroupWide name="episode" label="Episode" />
+          <SwitchGroupWide name="website" label="Website/Service" description="Services such as AMZN/HULU/NF" />
+          <SwitchGroupWide name="proper" label="Proper" />
+          <SwitchGroupWide name="repack" label="Repack" />
+          <SwitchGroupWide name="edition" label="Edition" />
+          <SwitchGroupWide name="language" label="Language" />
+        </div>
+      )}
+    </SlideOver>
+  );
+}
+
+export function ReleaseProfileDuplicateUpdateForm({ isOpen, toggle, data: profile }: UpdateFormProps<ReleaseProfileDuplicate>) {
+  const queryClient = useQueryClient();
+
+  const storeMutation = useMutation({
+    mutationFn: (profile: ReleaseProfileDuplicate) => APIClient.release.profiles.duplicates.store(profile),
+    onSuccess: () => {
+      queryClient.invalidateQueries({ queryKey: ReleaseProfileDuplicateKeys.lists() });
+      toast.custom((t) => <Toast type="success" body="Profile was added" t={t} />);
+
+      toggle();
+    },
+    onError: () => {
+      toast.custom((t) => <Toast type="error" body="Profile could not be added" t={t} />);
+    }
+  });
+
+  const onSubmit = (data: unknown) => storeMutation.mutate(data as ReleaseProfileDuplicate);
+
+  const deleteMutation = useMutation({
+    mutationFn: (profileId: number) => APIClient.release.profiles.duplicates.delete(profileId),
+    onSuccess: () => {
+      queryClient.invalidateQueries({ queryKey: ReleaseProfileDuplicateKeys.lists() });
+      queryClient.invalidateQueries({ queryKey: ReleaseProfileDuplicateKeys.detail(profile.id) });
+
+      toast.custom((t) => <Toast type="success" body={`Profile ${profile.name} was deleted!`} t={t} />);
+
+      toggle();
+    },
+  });
+
+  const onDelete = () => deleteMutation.mutate(profile.id);
+
+  const initialValues: ReleaseProfileDuplicate = {
+    id: profile.id,
+    name: profile.name,
+    protocol: profile.protocol,
+    release_name: profile.release_name,
+    hash: profile.hash,
+    title: profile.title,
+    sub_title: profile.sub_title,
+    year: profile.year,
+    month: profile.month,
+    day: profile.day,
+    source: profile.source,
+    resolution: profile.resolution,
+    codec: profile.codec,
+    container: profile.container,
+    dynamic_range: profile.dynamic_range,
+    audio: profile.audio,
+    group: profile.group,
+    season: profile.season,
+    episode: profile.episode,
+    website: profile.website,
+    proper: profile.proper,
+    repack: profile.repack,
+    edition: profile.edition,
+    language: profile.language,
+  };
+
+  return (
+    <SlideOver
+      type="UPDATE"
+      title="Duplicate Profile"
+      isOpen={isOpen}
+      toggle={toggle}
+      deleteAction={onDelete}
+      onSubmit={onSubmit}
+      initialValues={initialValues}
+    >
+      {() => (
+        <div className="py-2 space-y-6 sm:py-0 sm:space-y-0 divide-y divide-gray-200 dark:divide-gray-700">
+          <TextFieldWide required name="name" label="Name"/>
+
+          <SwitchGroupWide name="release_name" label="Release name" description="Full release name" />
+          <SwitchGroupWide name="hash" label="Hash" description="Normalized hash of the release name. Use with Releae name for exact match" />
+          <SwitchGroupWide name="title" label="Title" description="Parsed title" />
+          <SwitchGroupWide name="sub_title" label="Sub Title" description="Parsed Sub Title like Episode Name" />
+          <SwitchGroupWide name="year" label="Year" />
+          <SwitchGroupWide name="month" label="Month" description="For daily releases" />
+          <SwitchGroupWide name="day" label="Day" description="For daily releases" />
+          <SwitchGroupWide name="source" label="Source" />
+          <SwitchGroupWide name="resolution" label="Resolution" />
+          <SwitchGroupWide name="codec" label="Codec" />
+          <SwitchGroupWide name="container" label="Container" />
+          <SwitchGroupWide name="dynamic_range" label="Dynamic Range (HDR,DV etc)" />
+          <SwitchGroupWide name="audio" label="Audio" />
+          <SwitchGroupWide name="group" label="Group" description="Release group" />
+          <SwitchGroupWide name="season" label="Season" />
+          <SwitchGroupWide name="episode" label="Episode" />
+          <SwitchGroupWide name="website" label="Website/Service" description="Services such as AMZN/HULU/NF" />
+          <SwitchGroupWide name="repack" label="Repack" />
+          <SwitchGroupWide name="proper" label="Proper" />
+          <SwitchGroupWide name="edition" label="Edition and Cut" />
+          <SwitchGroupWide name="language" label="Language and Region" />
+        </div>
+      )}
+    </SlideOver>
+  );
+}