feat(lists): add option to skip cleaning of Plaintext data (#2036)

* Added: Plaintext untouched

* Revert "Added: Plaintext untouched"

This reverts commit e6ceaec5f4776cfc8335ae2c02e1caa4a2bbb0bc.

* Added: skipCleanSanitize

* TS definition for List object doesn't yet know about the new skip_clean_sanitize property

* Update: ListForms.tsx with the bypass option

* Update: Database internals for skip_clean_sanitize

* Fix: Snake case

internal/database/list.go

@@ -44,6 +44,7 @@ func (r *ListRepo) List(ctx context.Context) ([]*domain.List, error) {
 		"tags_excluded",
 		"include_unmonitored",
 		"include_alternate_titles",
+		"skip_clean_sanitize",
 		"last_refresh_time",
 		"last_refresh_status",
 		"last_refresh_data",
@@ -73,7 +74,7 @@ func (r *ListRepo) List(ctx context.Context) ([]*domain.List, error) {
 		var lastRefreshTime sql.Null[time.Time]
 		var clientID sql.Null[int]
 
-		err = rows.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &lastRefreshTime, &lastRefreshStatus, &lastRefreshData, &list.CreatedAt, &list.UpdatedAt)
+		err = rows.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &list.SkipCleanSanitize, &lastRefreshTime, &lastRefreshStatus, &lastRefreshData, &list.CreatedAt, &list.UpdatedAt)
 		if err != nil {
 			return nil, err
 		}
@@ -107,6 +108,7 @@ func (r *ListRepo) FindByID(ctx context.Context, listID int64) (*domain.List, er
 		"tags_excluded",
 		"include_unmonitored",
 		"include_alternate_titles",
+		"skip_clean_sanitize",
 		"last_refresh_time",
 		"last_refresh_status",
 		"last_refresh_data",
@@ -135,7 +137,7 @@ func (r *ListRepo) FindByID(ctx context.Context, listID int64) (*domain.List, er
 	var url, apiKey sql.Null[string]
 	var clientID sql.Null[int]
 
-	err = row.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &list.LastRefreshTime, &list.LastRefreshStatus, &list.LastRefreshData, &list.CreatedAt, &list.UpdatedAt)
+	err = row.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &list.SkipCleanSanitize, &list.LastRefreshTime, &list.LastRefreshStatus, &list.LastRefreshData, &list.CreatedAt, &list.UpdatedAt)
 	if err != nil {
 		return nil, err
 	}
@@ -169,6 +171,7 @@ func (r *ListRepo) Store(ctx context.Context, list *domain.List) error {
 			"tags_excluded",
 			"include_unmonitored",
 			"include_alternate_titles",
+			"skip_clean_sanitize",
 		).
 		Values(
 			list.Name,
@@ -183,6 +186,7 @@ func (r *ListRepo) Store(ctx context.Context, list *domain.List) error {
 			pq.Array(list.TagsExclude),
 			list.IncludeUnmonitored,
 			list.IncludeAlternateTitles,
+			list.SkipCleanSanitize,
 		).Suffix("RETURNING id").RunWith(tx)
 
 	//query, args, err := qb.ToSql()
@@ -226,6 +230,7 @@ func (r *ListRepo) Update(ctx context.Context, list *domain.List) error {
 		Set("tags_excluded", pq.Array(list.TagsExclude)).
 		Set("include_unmonitored", list.IncludeUnmonitored).
 		Set("include_alternate_titles", list.IncludeAlternateTitles).
+		Set("skip_clean_sanitize", list.SkipCleanSanitize).
 		Set("updated_at", sq.Expr("CURRENT_TIMESTAMP")).
 		Where(sq.Eq{"id": list.ID})
 

internal/database/postgres_migrate.go

@@ -299,7 +299,7 @@ CREATE TABLE "release"
     protocol          TEXT,
     implementation    TEXT,
     timestamp         TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
-    announce_type     TEXT      DEFAULT 'NEW', 
+    announce_type     TEXT      DEFAULT 'NEW',
     info_url          TEXT,
     download_url      TEXT,
     group_id          TEXT,
@@ -549,6 +549,7 @@ CREATE TABLE list
     tags_excluded            TEXT [] DEFAULT '{}' NOT NULL,
     include_unmonitored      BOOLEAN,
     include_alternate_titles BOOLEAN,
+    skip_clean_sanitize      BOOLEAN DEFAULT FALSE,
     last_refresh_time        TIMESTAMP,
     last_refresh_status      TEXT,
     last_refresh_data        TEXT,
@@ -1052,7 +1053,7 @@ ALTER TABLE filter
 `,
 	`UPDATE  irc_network
     SET server = 'irc.animefriends.moe',
-        name = CASE  
+        name = CASE
 			WHEN name = 'AnimeBytes-IRC' THEN 'AnimeBytes'
         	ELSE name
         END
@@ -1130,11 +1131,11 @@ CREATE INDEX filter_priority_index
 	`UPDATE irc_network
     SET server = 'irc.scenehd.org'
     WHERE server = 'irc.scenehd.eu';
-	
+
 UPDATE irc_network
     SET server = 'irc.p2p-network.net', name = 'P2P-Network', nick = nick || '_0'
     WHERE server = 'irc.librairc.net';
-	
+
 UPDATE irc_network
     SET server = 'irc.atw-inter.net', name = 'ATW-Inter'
     WHERE server = 'irc.ircnet.com';
@@ -1351,5 +1352,9 @@ CREATE INDEX release_hybrid_index
 	`UPDATE filter
 	SET announce_types = '{"NEW"}'
 	WHERE announce_types = '{}';
+`,
+	`
+	ALTER TABLE list
+		ADD COLUMN skip_clean_sanitize BOOLEAN DEFAULT FALSE;
 `,
 }

internal/database/sqlite_migrate.go

@@ -302,7 +302,7 @@ CREATE TABLE "release"
     protocol          TEXT,
     implementation    TEXT,
     timestamp         TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-    announce_type     TEXT      DEFAULT 'NEW', 
+    announce_type     TEXT      DEFAULT 'NEW',
     info_url          TEXT,
     download_url      TEXT,
     group_id          TEXT,
@@ -1698,7 +1698,7 @@ ALTER TABLE filter
 `,
 	`UPDATE  irc_network
     SET server = 'irc.animefriends.moe',
-        name = CASE  
+        name = CASE
 			WHEN name = 'AnimeBytes-IRC' THEN 'AnimeBytes'
         	ELSE name
         END
@@ -1777,11 +1777,11 @@ CREATE INDEX filter_priority_index
 	`UPDATE irc_network
 	SET server = 'irc.scenehd.org'
 	WHERE server = 'irc.scenehd.eu';
-	
+
 UPDATE irc_network
 	SET server = 'irc.p2p-network.net', name = 'P2P-Network', nick = nick || '_0'
 	WHERE server = 'irc.librairc.net';
-	
+
 UPDATE irc_network
 	SET server = 'irc.atw-inter.net', name = 'ATW-Inter'
 	WHERE server = 'irc.ircnet.com';
@@ -1815,6 +1815,7 @@ UPDATE irc_network
     tags_excluded            TEXT [] DEFAULT '{}' NOT NULL,
     include_unmonitored      BOOLEAN,
     include_alternate_titles BOOLEAN,
+    skip_clean_sanitize      BOOLEAN DEFAULT FALSE,
     last_refresh_time        TIMESTAMP,
     last_refresh_status      TEXT,
     last_refresh_data        TEXT,
@@ -1985,7 +1986,7 @@ CREATE INDEX release_cut_index
 CREATE INDEX release_hybrid_index
     ON "release" (hybrid);
 `,
-	`UPDATE irc_channel 
+	`UPDATE irc_channel
     SET name = '#ptp-announce'
     WHERE name = '#ptp-announce-dev' AND NOT EXISTS (SELECT 1 FROM irc_channel WHERE name = '#ptp-announce');
 `,
@@ -1996,5 +1997,9 @@ CREATE INDEX release_hybrid_index
 	`UPDATE filter
 	SET announce_types = '{"NEW"}'
 	WHERE announce_types = '{}';
+`,
+	`
+	ALTER TABLE list
+		ADD COLUMN skip_clean_sanitize BOOLEAN DEFAULT FALSE;
 `,
 }

internal/domain/list.go

@@ -67,6 +67,7 @@ type List struct {
 	LastRefreshStatus      ListRefreshStatus `json:"last_refresh_status"`
 	CreatedAt              time.Time         `json:"created_at"`
 	UpdatedAt              time.Time         `json:"updated_at"`
+	SkipCleanSanitize      bool              `json:"skip_clean_sanitize"`
 }
 
 func (l *List) Validate() error {

internal/list/process_list_plaintext.go

@@ -98,7 +98,11 @@ func (s *service) plaintext(ctx context.Context, list *domain.List) error {
 		if title == "" {
 			continue
 		}
-		titles = append(titles, processTitle(title, list.MatchRelease)...)
+		if list.SkipCleanSanitize {
+			titles = append(titles, title) // Add title as-is
+		} else {
+			titles = append(titles, processTitle(title, list.MatchRelease)...) // Existing logic
+		}
 	}
 
 	if len(titles) == 0 {

web/src/forms/settings/ListForms.tsx

@@ -142,6 +142,7 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) {
                     tags_excluded: [],
                     include_unmonitored: false,
                     include_alternate_titles: false,
+                    skip_clean_sanitize: false,
                   }}
                   onSubmit={onSubmit}
                   validate={validate}
@@ -373,6 +374,7 @@ export function ListUpdateForm({ isOpen, toggle, data }: UpdateFormProps<List>)
                     tags_excluded: data.tags_excluded,
                     include_unmonitored: data.include_unmonitored,
                     include_alternate_titles: data.include_alternate_titles,
+                    skip_clean_sanitize: data.skip_clean_sanitize,
                   }}
                   onSubmit={onSubmit}
                   // validate={validate}
@@ -597,6 +599,13 @@ const FilterOptionCheckBoxes = (props: ListTypeFormProps) => {
           <SwitchGroupWide name="include_unmonitored" label="Include Unmonitored" description="By default only monitored titles are filtered." />
         </fieldset>
       );
+    case "PLAINTEXT":
+      return (
+        <fieldset>
+          <legend className="sr-only">Settings</legend>
+          <SwitchGroupWide name="skip_clean_sanitize" label="Bypass the cleanup and sanitization and use the list as-is" description="By default, titles are automatically sanitized and checked for unusual characters." />
+        </fieldset>
+      );
   }
 }
 
@@ -748,6 +757,12 @@ function ListTypePlainText() {
           <SwitchGroupWide name="match_release" label="Match Release" description="Use Match Releases field. Uses Movies/Shows field by default." />
         </fieldset>
       </div>
+      <div className="space-y-1">
+        <fieldset>
+          <legend className="sr-only">Settings</legend>
+          <SwitchGroupWide name="skip_clean_sanitize" label="Bypass the cleanup and sanitization and use the list as-is" description="By default, titles are automatically sanitized and checked for unusual characters." />
+        </fieldset>
+      </div>
     </div>
   )
 }

web/src/types/List.d.ts

@@ -18,6 +18,7 @@ interface List {
   tags_excluded: string[];
   include_unmonitored: boolean;
   include_alternate_titles: boolean;
+  skip_clean_sanitize: boolean;
 }
 
 interface ListFilter {
@@ -39,6 +40,7 @@ interface ListCreate {
   tags_exclude: string[];
   include_unmonitored: boolean;
   include_alternate_titles: boolean;
+  skip_clean_sanitize: boolean;
 }
 
 type ListType =