feat(feeds): improve RSS (#502)

* feat(feeds): improve rss

* save last_run time
* remove interval check
* refactor feed job keys
* add rss test
* add max_age check

* feat(feeds): rss basic freeleech parsing

* feat(feeds): rss cookie support

* feat(feeds): db get max_age

* feat(feeds): update log messages

* feat(feeds): pass cookie to release for download

* feat(feeds): improve size parsing

* feat(feeds): improve datetime check

web/src/forms/settings/FeedForms.tsx

@@ -9,6 +9,7 @@ import { componentMapType } from "./DownloadClientForms";
 import { sleep } from "../../utils";
 import { useState } from "react";
 import { ImplementationBadges } from "../../screens/settings/Indexer";
+import { useFormikContext } from "formik";
 
 interface UpdateProps {
   isOpen: boolean;
@@ -24,8 +25,10 @@ interface InitialValues {
   name: string;
   url: string;
   api_key: string;
+  cookie: string;
   interval: number;
   timeout: number;
+  max_age: number;
 }
 
 export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) {
@@ -104,8 +107,10 @@ export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) {
     name: feed.name,
     url: feed.url,
     api_key: feed.api_key,
+    cookie: feed.cookie || "",
     interval: feed.interval,
-    timeout: feed.timeout
+    timeout: feed.timeout,
+    max_age: feed.max_age
   };
 
   return (
@@ -153,7 +158,26 @@ export function FeedUpdateForm({ isOpen, toggle, feed }: UpdateProps) {
   );
 }
 
+function WarningLabel() {
+  return (
+    <div className="px-4 py-1">
+      <span className="w-full block px-2 py-2 bg-red-300 dark:bg-red-400 text-red-900 dark:text-red-900 text-sm rounded">
+        <span className="font-semibold">
+          Warning: Indexers might ban you for too low interval!
+        </span>
+        <span className="ml-1">
+          Read the indexer rules.
+        </span>
+      </span>
+    </div>
+  );
+}
+
 function FormFieldsTorznab() {
+  const {
+    values: { interval }
+  } = useFormikContext<InitialValues>();
+
   return (
     <div className="border-t border-gray-200 dark:border-gray-700 py-5">
       <TextFieldWide
@@ -164,14 +188,20 @@ function FormFieldsTorznab() {
 
       <PasswordFieldWide name="api_key" label="API key" />
 
+      {interval < 15 && <WarningLabel />}
       <NumberFieldWide name="interval" label="Refresh interval" help="Minutes. Recommended 15-30. Too low and risk ban."/>
 
       <NumberFieldWide name="timeout" label="Refresh timeout" help="Seconds to wait before cancelling refresh."/>
+      <NumberFieldWide name="max_age" label="Max age" help="Seconds. Will not grab older than this value."/>
     </div>
   );
 }
 
 function FormFieldsRSS() {
+  const {
+    values: { interval }
+  } = useFormikContext<InitialValues>();
+
   return (
     <div className="border-t border-gray-200 dark:border-gray-700 py-5">
       <TextFieldWide
@@ -180,8 +210,12 @@ function FormFieldsRSS() {
         help="RSS url"
       />
 
+      {interval < 15 && <WarningLabel />}
       <NumberFieldWide name="interval" label="Refresh interval" help="Minutes. Recommended 15-30. Too low and risk ban."/>
       <NumberFieldWide name="timeout" label="Refresh timeout" help="Seconds to wait before cancelling refresh."/>
+      <NumberFieldWide name="max_age" label="Max age" help="Seconds. Will not grab older than this value."/>
+
+      <PasswordFieldWide name="cookie" label="Cookie" help="Not commonly used" />
     </div>
   );
 }

web/src/screens/settings/Feed.tsx

@@ -3,7 +3,7 @@ import { useMutation, useQuery, useQueryClient } from "react-query";
 import { APIClient } from "../../api/APIClient";
 import { Menu, Switch, Transition } from "@headlessui/react";
 
-import { classNames } from "../../utils";
+import { classNames, IsEmptyDate, simplifyDate } from "../../utils";
 import { Fragment, useRef, useState } from "react";
 import { toast } from "react-hot-toast";
 import Toast from "../../components/notifications/Toast";
@@ -44,10 +44,10 @@ function FeedSettings() {
                   className="col-span-4 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Name
                 </div>
                 <div
-                  className="col-span-3 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Indexer
+                  className="col-span-2 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Type
                 </div>
                 <div
-                  className="col-span-3 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Type
+                  className="col-span-3 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Last run
                 </div>
                 {/*<div className="col-span-4 px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">Events</div>*/}
               </li>
@@ -115,15 +115,20 @@ function ListItem({ feed }: ListItemProps) {
             />
           </Switch>
         </div>
-        <div className="col-span-4 flex items-center sm:px-6 text-sm font-medium text-gray-900 dark:text-white">
-          {feed.name}
-        </div>
-        <div className="col-span-3 flex items-center sm:px-6 text-sm font-medium text-gray-900 dark:text-gray-500">
-          {feed.indexer}
+        <div className="col-span-4 flex flex-col sm:px-6 text-sm font-medium text-gray-900 dark:text-white">
+          <span>{feed.name}</span>
+          <span className="text-gray-900 dark:text-gray-500 text-xs">
+            {feed.indexer}
+          </span>
         </div>
         <div className="col-span-2 flex items-center sm:px-6">
           {ImplementationBadges[feed.type.toLowerCase()]}
         </div>
+        <div className="col-span-3 flex items-center sm:px-6 text-sm font-medium text-gray-900 dark:text-gray-500">
+          <span title={simplifyDate(feed.last_run)}>
+            {IsEmptyDate(feed.last_run)}
+          </span>
+        </div>
         <div className="col-span-1 flex justify-center items-center sm:px-6">
           <FeedItemDropdown
             feed={feed}

web/src/types/Feed.d.ts

@@ -7,7 +7,11 @@ interface Feed {
   url: string;
   interval: number;
   timeout: number;
+  max_age: number;
   api_key: string;
+  cookie: string;
+  last_run: string;
+  last_run_data: string;
   created_at: Date;
   updated_at: Date;
 }