feat(feeds): add generic RSS support (#410)

* feat(feeds): add generic rss support

* feat(feeds/web): add generic rss support

* implement rss downloading

* gosum + mod

* re-add size from Custom field.

* implement uploader + category

* sync

* remove double assignment (+torznab)

* didn't save the rss file >.>

* cleanup

* fixfeeds): create rss indexer

* fix(feeds): stop feed

* feat(feeds): support nexusphp rss enclosure link

* feat(feeds): check size for custom size

* fix(feeds): race condition and only stop enabled feeds

* fix(feeds): unify indexer implementation badge

Co-authored-by: Kyle Sanderson <kyle.leet@gmail.com>

web/src/forms/settings/FeedForms.tsx

@@ -5,10 +5,10 @@ import { toast } from "react-hot-toast";
 import Toast from "../../components/notifications/Toast";
 import { SlideOver } from "../../components/panels";
 import { NumberFieldWide, PasswordFieldWide, SwitchGroupWide, TextFieldWide } from "../../components/inputs";
-import { ImplementationMap } from "../../screens/settings/Feed";
 import { componentMapType } from "./DownloadClientForms";
 import { sleep } from "../../utils";
 import { useState } from "react";
+import { ImplementationBadges } from "../../screens/settings/Indexer";
 
 interface UpdateProps {
   isOpen: boolean;
@@ -126,11 +126,11 @@ export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) {
                 </label>
               </div>
               <div className="flex justify-end sm:col-span-2">
-                {ImplementationMap[feed.type]}
+                {ImplementationBadges[feed.type.toLowerCase()]}
               </div>
             </div>
 
-            <div className="py-6 px-6 space-y-6 sm:py-0 sm:space-y-0 sm:divide-y sm:divide-gray-200">
+            <div className="py-6 space-y-6 sm:py-0 sm:space-y-0 sm:divide-y sm:divide-gray-200">
               <SwitchGroupWide name="enabled" label="Enabled"/>
             </div>
           </div>
@@ -158,6 +158,21 @@ function FormFieldsTorznab() {
   );
 }
 
+function FormFieldsRSS() {
+  return (
+    <div className="border-t border-gray-200 dark:border-gray-700 py-5">
+      <TextFieldWide
+        name="url"
+        label="URL"
+        help="RSS url"
+      />
+
+      <NumberFieldWide name="interval" label="Refresh interval" help="Minutes. Recommended 15-30. To low and risk ban."/>
+    </div>
+  );
+}
+
 const componentMap: componentMapType = {
-  TORZNAB: <FormFieldsTorznab/>
+  TORZNAB: <FormFieldsTorznab/>,
+  RSS: <FormFieldsRSS />
 };

web/src/forms/settings/IndexerForms.tsx

@@ -119,6 +119,37 @@ const FeedSettingFields = (ind: IndexerDefinition, indexer: string) => {
   }
 };
 
+const RSSFeedSettingFields = (ind: IndexerDefinition, indexer: string) => {
+  if (indexer !== "") {
+    return (
+      <Fragment>
+        {ind && ind.rss && ind.rss.settings && (
+          <div className="">
+            <div className="px-4 space-y-1">
+              <Dialog.Title className="text-lg font-medium text-gray-900 dark:text-white">RSS</Dialog.Title>
+              <p className="text-sm text-gray-500 dark:text-gray-200">
+                RSS feed
+              </p>
+            </div>
+
+            <TextFieldWide name="name" label="Name" defaultValue={""} />
+
+            {ind.rss.settings.map((f: IndexerSetting, idx: number) => {
+              switch (f.type) {
+              case "text":
+                return <TextFieldWide name={`feed.${f.name}`} label={f.label} required={f.required} key={idx} help={f.help} />;
+              case "secret":
+                return <PasswordFieldWide name={`feed.${f.name}`} label={f.label} required={f.required} key={idx} help={f.help} defaultValue={f.default} />;
+              }
+              return null;
+            })}
+          </div>
+        )}
+      </Fragment>
+    );
+  }
+};
+
 const SettingFields = (ind: IndexerDefinition, indexer: string) => {
   if (indexer !== "") {
     return (
@@ -144,10 +175,13 @@ const SettingFields = (ind: IndexerDefinition, indexer: string) => {
   }
 };
 
-function slugIdentifier(name: string) {
+function slugIdentifier(name: string, prefix?: string) {
   const l = name.toLowerCase();
-  const r = l.replaceAll("torznab", "");
-  return slugify(`torznab-${r}`);
+  if (prefix && prefix !== "") {
+    const r = l.replaceAll(prefix, "");
+    return slugify(`${prefix}-${r}`);
+  }
+  return slugify(l);
 }
 
 // interface initialValues {
@@ -210,7 +244,7 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) {
 
     if (formData.implementation === "torznab") {
       // create slug for indexer identifier as "torznab-indexer_name"
-      const name = slugIdentifier(formData.name);
+      const name = slugIdentifier(formData.name, "torznab");
 
       const createFeed: FeedCreate = {
         name: formData.name,
@@ -234,6 +268,31 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) {
       return;
     }
 
+    if (formData.implementation === "rss") {
+      // create slug for indexer identifier as "torznab-indexer_name"
+      const name = slugIdentifier(formData.name, "rss");
+
+      const createFeed: FeedCreate = {
+        name: formData.name,
+        enabled: false,
+        type: "RSS",
+        url: formData.feed.url,
+        interval: 30,
+        indexer: name,
+        indexer_id: 0
+      };
+
+      mutation.mutate(formData as Indexer, {
+        onSuccess: (indexer) => {
+          // @eslint-ignore
+          createFeed.indexer_id = indexer.id;
+
+          feedMutation.mutate(createFeed);
+        }
+      });
+      return;
+    }
+
     if (formData.implementation === "irc") {
 
       const channels: IrcChannel[] = [];
@@ -399,6 +458,7 @@ export function IndexerAddForm({ isOpen, toggle }: AddProps) {
 
                         {IrcSettingFields(indexer, values.identifier)}
                         {FeedSettingFields(indexer, values.identifier)}
+                        {RSSFeedSettingFields(indexer, values.identifier)}
                       </div>
 
                       <div

web/src/screens/settings/Feed.tsx

@@ -17,7 +17,7 @@ import {
 } from "@heroicons/react/outline";
 import { FeedUpdateForm } from "../../forms/settings/FeedForms";
 import { EmptySimple } from "../../components/emptystates";
-import { componentMapType } from "../../forms/settings/DownloadClientForms";
+import { ImplementationBadges } from "./Indexer";
 
 function FeedSettings() {
   const { data } = useQuery(
@@ -46,10 +46,13 @@ function FeedSettings() {
                   className="col-span-2 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Enabled
                 </div>
                 <div
-                  className="col-span-6 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Name
+                  className="col-span-4 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Name
                 </div>
                 <div
-                  className="col-span-2 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Type
+                  className="col-span-3 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Indexer
+                </div>
+                <div
+                  className="col-span-3 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Type
                 </div>
                 {/*<div className="col-span-4 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Events</div>*/}
               </li>
@@ -65,18 +68,6 @@ function FeedSettings() {
   );
 }
 
-const ImplementationTorznab = () => (
-  <span
-    className="inline-flex items-center px-2.5 py-0.5 rounded-md text-sm font-medium bg-orange-200 dark:bg-orange-400 text-orange-800 dark:text-orange-800"
-  >
-        Torznab
-  </span>
-);
-
-export const ImplementationMap: componentMapType = {
-  "TORZNAB": <ImplementationTorznab/>
-};
-
 interface ListItemProps {
     feed: Feed;
 }
@@ -129,13 +120,16 @@ function ListItem({ feed }: ListItemProps) {
             />
           </Switch>
         </div>
-        <div className="col-span-6 flex items-center sm:px-6 text-sm font-medium text-gray-900 dark:text-white">
+        <div className="col-span-4 flex items-center sm:px-6 text-sm font-medium text-gray-900 dark:text-white">
           {feed.name}
         </div>
-        <div className="col-span-2 flex items-center sm:px-6">
-          {ImplementationMap[feed.type]}
+        <div className="col-span-3 flex items-center sm:px-6 text-sm font-medium text-gray-900 dark:text-gray-500">
+          {feed.indexer}
         </div>
-        <div className="col-span-1 flex items-center sm:px-6">
+        <div className="col-span-2 flex items-center sm:px-6">
+          {ImplementationBadges[feed.type.toLowerCase()]}
+        </div>
+        <div className="col-span-1 flex justify-center items-center sm:px-6">
           <FeedItemDropdown
             feed={feed}
             onToggle={toggleActive}
@@ -223,7 +217,7 @@ const FeedItemDropdown = ({
                     )}
                     aria-hidden="true"
                   />
-                                    Edit
+                  Edit
                 </button>
               )}
             </Menu.Item>
@@ -243,7 +237,7 @@ const FeedItemDropdown = ({
                     )}
                     aria-hidden="true"
                   />
-                                    Toggle
+                  Toggle
                 </button>
               )}
             </Menu.Item>
@@ -265,7 +259,7 @@ const FeedItemDropdown = ({
                     )}
                     aria-hidden="true"
                   />
-                                    Delete
+                  Delete
                 </button>
               )}
             </Menu.Item>

web/src/screens/settings/Indexer.tsx

@@ -7,7 +7,7 @@ import { EmptySimple } from "../../components/emptystates";
 import { APIClient } from "../../api/APIClient";
 import { componentMapType } from "../../forms/settings/DownloadClientForms";
 
-const ImplementationIRC = () => (
+const ImplementationBadgeIRC = () => (
   <span
     className="mr-2 inline-flex items-center px-2.5 py-0.5 rounded-md text-sm font-medium bg-green-200 dark:bg-green-400 text-green-800 dark:text-green-800"
   >
@@ -15,17 +15,26 @@ const ImplementationIRC = () => (
   </span>
 );
 
-const ImplementationTorznab = () => (
+const ImplementationBadgeTorznab = () => (
   <span
     className="inline-flex items-center px-2.5 py-0.5 rounded-md text-sm font-medium bg-orange-200 dark:bg-orange-400 text-orange-800 dark:text-orange-800"
   >
-        Torznab
+    Torznab
   </span>
 );
 
-const implementationMap: componentMapType = {
-  "irc": <ImplementationIRC/>,
-  "torznab": <ImplementationTorznab />
+const ImplementationBadgeRSS = () => (
+  <span
+    className="inline-flex items-center px-2.5 py-0.5 rounded-md text-sm font-medium bg-amber-200 dark:bg-amber-400 text-amber-800 dark:text-amber-800"
+  >
+    RSS
+  </span>
+);
+
+export const ImplementationBadges: componentMapType = {
+  "irc": <ImplementationBadgeIRC/>,
+  "torznab": <ImplementationBadgeTorznab />,
+  "rss": <ImplementationBadgeRSS />
 };
 
 interface ListItemProps {
@@ -59,10 +68,10 @@ const ListItem = ({ indexer }: ListItemProps) => {
         </Switch>
       </td>
       <td className="px-6 py-4 w-full whitespace-nowrap text-sm font-medium text-gray-900 dark:text-white">{indexer.name}</td>
-      <td className="px-6 py-4 w-full whitespace-nowrap text-sm font-medium text-gray-900 dark:text-white">{implementationMap[indexer.implementation]}</td>
+      <td className="px-6 py-4 w-full whitespace-nowrap text-sm font-medium text-gray-900 dark:text-white">{ImplementationBadges[indexer.implementation]}</td>
       <td className="px-6 py-4 whitespace-nowrap text-right text-sm font-medium">
         <span className="text-indigo-600 dark:text-gray-300 hover:text-indigo-900 dark:hover:text-blue-500 cursor-pointer" onClick={toggleUpdate}>
-                    Edit
+          Edit
         </span>
       </td>
     </tr>
@@ -100,7 +109,7 @@ function IndexerSettings() {
               onClick={toggleAddIndexer}
               className="relative inline-flex items-center px-4 py-2 border border-transparent shadow-sm text-sm font-medium rounded-md text-white bg-indigo-600 dark:bg-blue-600 hover:bg-indigo-700 dark:hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 dark:focus:ring-blue-500"
             >
-                            Add new
+              Add new
             </button>
           </div>
         </div>
@@ -117,19 +126,19 @@ function IndexerSettings() {
                           scope="col"
                           className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider"
                         >
-                                                    Enabled
+                          Enabled
                         </th>
                         <th
                           scope="col"
                           className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider"
                         >
-                                                    Name
+                          Name
                         </th>
                         <th
                           scope="col"
                           className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider"
                         >
-                                                    Implementation
+                          Implementation
                         </th>
                         <th scope="col" className="relative px-6 py-3">
                           <span className="sr-only">Edit</span>

web/src/types/Feed.d.ts

@@ -2,7 +2,7 @@ interface Feed {
   id: number;
   indexer: string;
   name: string;
-  type: string;
+  type: FeedType;
   enabled: boolean;
   url: string;
   interval: number;
@@ -11,13 +11,15 @@ interface Feed {
   updated_at: Date;
 }
 
+type FeedType = "TORZNAB" | "RSS";
+
 interface FeedCreate {
   indexer: string;
   name: string;
-  type: string;
+  type: FeedType;
   enabled: boolean;
   url: string;
   interval: number;
-  api_key: string;
+  api_key?: string;
   indexer_id: number;
 }

web/src/types/Indexer.d.ts

@@ -22,6 +22,7 @@ interface IndexerDefinition {
   settings: IndexerSetting[];
   irc: IndexerIRC;
   torznab: IndexerTorznab;
+  rss: IndexerFeed;
   parse: IndexerParse;
 }
 
@@ -53,6 +54,11 @@ interface IndexerTorznab {
   settings: IndexerSetting[];
 }
 
+interface IndexerFeed {
+  minInterval: number;
+  settings: IndexerSetting[];
+}
+
 interface IndexerParse {
   type: string;
   lines: IndexerParseLines[];