feat(feeds): optimize existing cache items check (#2078)

* feat(feeds): optimize existing items cache check

* feat(feeds): remove ttl from repo method ExistingItems

* feat(feeds): add db integration test for ExistingItems

* feat(feeds): improve release and filter processing

* feat(feeds): fix failing test
This commit is contained in:
ze0s 2025-06-07 12:46:08 +02:00 committed by GitHub
parent 92ddb919a5
commit 46f6fbe5cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 266 additions and 45 deletions

View file

@ -157,6 +157,48 @@ func (r *FeedCacheRepo) Exists(feedId int, key string) (bool, error) {
return exists, nil
}
// ExistingItems checks multiple keys in the cache for a given feed ID
// and returns a map of existing keys to their values
func (r *FeedCacheRepo) ExistingItems(ctx context.Context, feedId int, keys []string) (map[string]bool, error) {
if len(keys) == 0 {
return make(map[string]bool), nil
}
// Build a query that returns all keys that exist in the cache
queryBuilder := r.db.squirrel.
Select("key").
From("feed_cache").
Where(sq.Eq{"feed_id": feedId}).
Where(sq.Eq{"key": keys})
query, args, err := queryBuilder.ToSql()
if err != nil {
return nil, errors.Wrap(err, "error building query")
}
rows, err := r.db.handler.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, "error executing query")
}
defer rows.Close()
result := make(map[string]bool)
for rows.Next() {
var key string
if err := rows.Scan(&key); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}
result[key] = true
}
if err := rows.Err(); err != nil {
return nil, errors.Wrap(err, "row error")
}
return result, nil
}
func (r *FeedCacheRepo) Put(feedId int, key string, val []byte, ttl time.Time) error {
queryBuilder := r.db.squirrel.
Insert("feed_cache").