From 4ce224199134e643108dd315e5d801f492734f6c Mon Sep 17 00:00:00 2001 From: Lucian Maly Date: Sun, 1 Jun 2025 23:27:48 +1000 Subject: [PATCH] feat(lists): add option to skip cleaning of Plaintext data (#2036) * Added: Plaintext untouched * Revert "Added: Plaintext untouched" This reverts commit e6ceaec5f4776cfc8335ae2c02e1caa4a2bbb0bc. * Added: skipCleanSanitize * TS definition for List object doesn't yet know about the new skip_clean_sanitize property * Update: ListForms.tsx with the bypass option * Update: Database internals for skip_clean_sanitize * Fix: Snake case --- internal/database/list.go | 9 +++++++-- internal/database/postgres_migrate.go | 13 +++++++++---- internal/database/sqlite_migrate.go | 15 ++++++++++----- internal/domain/list.go | 1 + internal/list/process_list_plaintext.go | 6 +++++- web/src/forms/settings/ListForms.tsx | 15 +++++++++++++++ web/src/types/List.d.ts | 2 ++ 7 files changed, 49 insertions(+), 12 deletions(-) diff --git a/internal/database/list.go b/internal/database/list.go index 57f09e4..51ed458 100644 --- a/internal/database/list.go +++ b/internal/database/list.go @@ -44,6 +44,7 @@ func (r *ListRepo) List(ctx context.Context) ([]*domain.List, error) { "tags_excluded", "include_unmonitored", "include_alternate_titles", + "skip_clean_sanitize", "last_refresh_time", "last_refresh_status", "last_refresh_data", @@ -73,7 +74,7 @@ func (r *ListRepo) List(ctx context.Context) ([]*domain.List, error) { var lastRefreshTime sql.Null[time.Time] var clientID sql.Null[int] - err = rows.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &lastRefreshTime, &lastRefreshStatus, &lastRefreshData, &list.CreatedAt, &list.UpdatedAt) + err = rows.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &list.SkipCleanSanitize, &lastRefreshTime, &lastRefreshStatus, &lastRefreshData, &list.CreatedAt, &list.UpdatedAt) if err != nil { return nil, err } @@ -107,6 +108,7 @@ func (r *ListRepo) FindByID(ctx context.Context, listID int64) (*domain.List, er "tags_excluded", "include_unmonitored", "include_alternate_titles", + "skip_clean_sanitize", "last_refresh_time", "last_refresh_status", "last_refresh_data", @@ -135,7 +137,7 @@ func (r *ListRepo) FindByID(ctx context.Context, listID int64) (*domain.List, er var url, apiKey sql.Null[string] var clientID sql.Null[int] - err = row.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &list.LastRefreshTime, &list.LastRefreshStatus, &list.LastRefreshData, &list.CreatedAt, &list.UpdatedAt) + err = row.Scan(&list.ID, &list.Name, &list.Enabled, &list.Type, &clientID, &url, pq.Array(&list.Headers), &list.APIKey, &list.MatchRelease, pq.Array(&list.TagsInclude), pq.Array(&list.TagsExclude), &list.IncludeUnmonitored, &list.IncludeAlternateTitles, &list.SkipCleanSanitize, &list.LastRefreshTime, &list.LastRefreshStatus, &list.LastRefreshData, &list.CreatedAt, &list.UpdatedAt) if err != nil { return nil, err } @@ -169,6 +171,7 @@ func (r *ListRepo) Store(ctx context.Context, list *domain.List) error { "tags_excluded", "include_unmonitored", "include_alternate_titles", + "skip_clean_sanitize", ). Values( list.Name, @@ -183,6 +186,7 @@ func (r *ListRepo) Store(ctx context.Context, list *domain.List) error { pq.Array(list.TagsExclude), list.IncludeUnmonitored, list.IncludeAlternateTitles, + list.SkipCleanSanitize, ).Suffix("RETURNING id").RunWith(tx) //query, args, err := qb.ToSql() @@ -226,6 +230,7 @@ func (r *ListRepo) Update(ctx context.Context, list *domain.List) error { Set("tags_excluded", pq.Array(list.TagsExclude)). Set("include_unmonitored", list.IncludeUnmonitored). Set("include_alternate_titles", list.IncludeAlternateTitles). + Set("skip_clean_sanitize", list.SkipCleanSanitize). Set("updated_at", sq.Expr("CURRENT_TIMESTAMP")). Where(sq.Eq{"id": list.ID}) diff --git a/internal/database/postgres_migrate.go b/internal/database/postgres_migrate.go index 10e76c3..060a304 100644 --- a/internal/database/postgres_migrate.go +++ b/internal/database/postgres_migrate.go @@ -299,7 +299,7 @@ CREATE TABLE "release" protocol TEXT, implementation TEXT, timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - announce_type TEXT DEFAULT 'NEW', + announce_type TEXT DEFAULT 'NEW', info_url TEXT, download_url TEXT, group_id TEXT, @@ -549,6 +549,7 @@ CREATE TABLE list tags_excluded TEXT [] DEFAULT '{}' NOT NULL, include_unmonitored BOOLEAN, include_alternate_titles BOOLEAN, + skip_clean_sanitize BOOLEAN DEFAULT FALSE, last_refresh_time TIMESTAMP, last_refresh_status TEXT, last_refresh_data TEXT, @@ -1052,7 +1053,7 @@ ALTER TABLE filter `, `UPDATE irc_network SET server = 'irc.animefriends.moe', - name = CASE + name = CASE WHEN name = 'AnimeBytes-IRC' THEN 'AnimeBytes' ELSE name END @@ -1130,11 +1131,11 @@ CREATE INDEX filter_priority_index `UPDATE irc_network SET server = 'irc.scenehd.org' WHERE server = 'irc.scenehd.eu'; - + UPDATE irc_network SET server = 'irc.p2p-network.net', name = 'P2P-Network', nick = nick || '_0' WHERE server = 'irc.librairc.net'; - + UPDATE irc_network SET server = 'irc.atw-inter.net', name = 'ATW-Inter' WHERE server = 'irc.ircnet.com'; @@ -1351,5 +1352,9 @@ CREATE INDEX release_hybrid_index `UPDATE filter SET announce_types = '{"NEW"}' WHERE announce_types = '{}'; +`, + ` + ALTER TABLE list + ADD COLUMN skip_clean_sanitize BOOLEAN DEFAULT FALSE; `, } diff --git a/internal/database/sqlite_migrate.go b/internal/database/sqlite_migrate.go index 8a019f9..b22af72 100644 --- a/internal/database/sqlite_migrate.go +++ b/internal/database/sqlite_migrate.go @@ -302,7 +302,7 @@ CREATE TABLE "release" protocol TEXT, implementation TEXT, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - announce_type TEXT DEFAULT 'NEW', + announce_type TEXT DEFAULT 'NEW', info_url TEXT, download_url TEXT, group_id TEXT, @@ -1698,7 +1698,7 @@ ALTER TABLE filter `, `UPDATE irc_network SET server = 'irc.animefriends.moe', - name = CASE + name = CASE WHEN name = 'AnimeBytes-IRC' THEN 'AnimeBytes' ELSE name END @@ -1777,11 +1777,11 @@ CREATE INDEX filter_priority_index `UPDATE irc_network SET server = 'irc.scenehd.org' WHERE server = 'irc.scenehd.eu'; - + UPDATE irc_network SET server = 'irc.p2p-network.net', name = 'P2P-Network', nick = nick || '_0' WHERE server = 'irc.librairc.net'; - + UPDATE irc_network SET server = 'irc.atw-inter.net', name = 'ATW-Inter' WHERE server = 'irc.ircnet.com'; @@ -1815,6 +1815,7 @@ UPDATE irc_network tags_excluded TEXT [] DEFAULT '{}' NOT NULL, include_unmonitored BOOLEAN, include_alternate_titles BOOLEAN, + skip_clean_sanitize BOOLEAN DEFAULT FALSE, last_refresh_time TIMESTAMP, last_refresh_status TEXT, last_refresh_data TEXT, @@ -1985,7 +1986,7 @@ CREATE INDEX release_cut_index CREATE INDEX release_hybrid_index ON "release" (hybrid); `, - `UPDATE irc_channel + `UPDATE irc_channel SET name = '#ptp-announce' WHERE name = '#ptp-announce-dev' AND NOT EXISTS (SELECT 1 FROM irc_channel WHERE name = '#ptp-announce'); `, @@ -1996,5 +1997,9 @@ CREATE INDEX release_hybrid_index `UPDATE filter SET announce_types = '{"NEW"}' WHERE announce_types = '{}'; +`, + ` + ALTER TABLE list + ADD COLUMN skip_clean_sanitize BOOLEAN DEFAULT FALSE; `, } diff --git a/internal/domain/list.go b/internal/domain/list.go index d00aa6e..bf426df 100644 --- a/internal/domain/list.go +++ b/internal/domain/list.go @@ -67,6 +67,7 @@ type List struct { LastRefreshStatus ListRefreshStatus `json:"last_refresh_status"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` + SkipCleanSanitize bool `json:"skip_clean_sanitize"` } func (l *List) Validate() error { diff --git a/internal/list/process_list_plaintext.go b/internal/list/process_list_plaintext.go index 70b9446..bbe6917 100644 --- a/internal/list/process_list_plaintext.go +++ b/internal/list/process_list_plaintext.go @@ -98,7 +98,11 @@ func (s *service) plaintext(ctx context.Context, list *domain.List) error { if title == "" { continue } - titles = append(titles, processTitle(title, list.MatchRelease)...) + if list.SkipCleanSanitize { + titles = append(titles, title) // Add title as-is + } else { + titles = append(titles, processTitle(title, list.MatchRelease)...) // Existing logic + } } if len(titles) == 0 { diff --git a/web/src/forms/settings/ListForms.tsx b/web/src/forms/settings/ListForms.tsx index eca3cf0..d9347fa 100644 --- a/web/src/forms/settings/ListForms.tsx +++ b/web/src/forms/settings/ListForms.tsx @@ -142,6 +142,7 @@ export function ListAddForm({ isOpen, toggle }: AddFormProps) { tags_excluded: [], include_unmonitored: false, include_alternate_titles: false, + skip_clean_sanitize: false, }} onSubmit={onSubmit} validate={validate} @@ -373,6 +374,7 @@ export function ListUpdateForm({ isOpen, toggle, data }: UpdateFormProps) tags_excluded: data.tags_excluded, include_unmonitored: data.include_unmonitored, include_alternate_titles: data.include_alternate_titles, + skip_clean_sanitize: data.skip_clean_sanitize, }} onSubmit={onSubmit} // validate={validate} @@ -597,6 +599,13 @@ const FilterOptionCheckBoxes = (props: ListTypeFormProps) => { ); + case "PLAINTEXT": + return ( +
+ Settings + +
+ ); } } @@ -748,6 +757,12 @@ function ListTypePlainText() { +
+
+ Settings + +
+
) } diff --git a/web/src/types/List.d.ts b/web/src/types/List.d.ts index 1f490d1..bbf34ce 100644 --- a/web/src/types/List.d.ts +++ b/web/src/types/List.d.ts @@ -18,6 +18,7 @@ interface List { tags_excluded: string[]; include_unmonitored: boolean; include_alternate_titles: boolean; + skip_clean_sanitize: boolean; } interface ListFilter { @@ -39,6 +40,7 @@ interface ListCreate { tags_exclude: string[]; include_unmonitored: boolean; include_alternate_titles: boolean; + skip_clean_sanitize: boolean; } type ListType =