idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 08:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(feeds): optimize existing cache items check (#2078)

Browse source 
* feat(feeds): optimize existing items cache check

* feat(feeds): remove ttl from repo method ExistingItems

* feat(feeds): add db integration test for ExistingItems

* feat(feeds): improve release and filter processing

* feat(feeds): fix failing test
This commit is contained in:
ze0s 2025-06-07 12:46:08 +02:00 • committed by GitHub
parent 92ddb919a5
commit 46f6fbe5cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 266 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

42
internal/database/feed_cache.go
View file

@ -157,6 +157,48 @@ func (r *FeedCacheRepo) Exists(feedId int, key string) (bool, error) {
return exists, nil
}
// ExistingItems checks multiple keys in the cache for a given feed ID
// and returns a map of existing keys to their values
func (r *FeedCacheRepo) ExistingItems(ctx context.Context, feedId int, keys []string) (map[string]bool, error) {
if len(keys) == 0 {
return make(map[string]bool), nil
}
// Build a query that returns all keys that exist in the cache
queryBuilder := r.db.squirrel.
Select("key").
From("feed_cache").
Where(sq.Eq{"feed_id": feedId}).
Where(sq.Eq{"key": keys})
query, args, err := queryBuilder.ToSql()
if err != nil {
return nil, errors.Wrap(err, "error building query")
}
rows, err := r.db.handler.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, "error executing query")
}
defer rows.Close()
result := make(map[string]bool)
for rows.Next() {
var key string
if err := rows.Scan(&key); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}
result[key] = true
}
if err := rows.Err(); err != nil {
return nil, errors.Wrap(err, "row error")
}
return result, nil
}
func (r *FeedCacheRepo) Put(feedId int, key string, val []byte, ttl time.Time) error {
queryBuilder := r.db.squirrel.
Insert("feed_cache").

101
internal/database/feed_cache_test.go
View file

@ -155,6 +155,107 @@ func TestFeedCacheRepo_Exists(t *testing.T) {
}
}
func TestFeedCacheRepo_ExistingItems(t *testing.T) {
for dbType, db := range testDBs {
log := setupLoggerForTest()
repo := NewFeedCacheRepo(log, db)
feedRepo := NewFeedRepo(log, db)
indexerRepo := NewIndexerRepo(log, db)
mockData := getMockFeed()
indexerMockData := getMockIndexer()
t.Run(fmt.Sprintf("ExistingItems_SingleItem_Multi_Keys [%s]", dbType), func(t *testing.T) {
// Setup
indexer, err := indexerRepo.Store(t.Context(), indexerMockData)
assert.NoError(t, err)
mockData.IndexerID = int(indexer.ID)
err = feedRepo.Store(t.Context(), mockData)
assert.NoError(t, err)
err = repo.Put(mockData.ID, "test_key", []byte("test_value"), time.Now().Add(time.Hour))
assert.NoError(t, err)
keys := []string{"test_key", "test_key_2"}
// Execute
items, err := repo.ExistingItems(t.Context(), mockData.ID, keys)
assert.NoError(t, err)
assert.Len(t, items, 1)
//assert.True(t, exists)
// Cleanup
_ = feedRepo.Delete(t.Context(), mockData.ID)
_ = indexerRepo.Delete(t.Context(), int(indexer.ID))
_ = repo.Delete(t.Context(), mockData.ID, "test_key")
})
t.Run(fmt.Sprintf("ExistingItems_MultipleItems [%s]", dbType), func(t *testing.T) {
// Setup
indexer, err := indexerRepo.Store(t.Context(), indexerMockData)
assert.NoError(t, err)
mockData.IndexerID = int(indexer.ID)
err = feedRepo.Store(t.Context(), mockData)
assert.NoError(t, err)
err = repo.Put(mockData.ID, "test_key", []byte("test_value"), time.Now().Add(time.Hour))
assert.NoError(t, err)
err = repo.Put(mockData.ID, "test_key_2", []byte("test_value_2"), time.Now().Add(time.Hour))
assert.NoError(t, err)
keys := []string{"test_key", "test_key_2"}
// Execute
items, err := repo.ExistingItems(t.Context(), mockData.ID, keys)
assert.NoError(t, err)
assert.Len(t, items, 2)
// Cleanup
_ = feedRepo.Delete(t.Context(), mockData.ID)
_ = indexerRepo.Delete(t.Context(), int(indexer.ID))
_ = repo.Delete(t.Context(), mockData.ID, "test_key")
})
t.Run(fmt.Sprintf("ExistingItems_MultipleItems_Single_Key [%s]", dbType), func(t *testing.T) {
// Setup
indexer, err := indexerRepo.Store(t.Context(), indexerMockData)
assert.NoError(t, err)
mockData.IndexerID = int(indexer.ID)
err = feedRepo.Store(t.Context(), mockData)
assert.NoError(t, err)
err = repo.Put(mockData.ID, "test_key", []byte("test_value"), time.Now().Add(time.Hour))
assert.NoError(t, err)
err = repo.Put(mockData.ID, "test_key_2", []byte("test_value_2"), time.Now().Add(time.Hour))
assert.NoError(t, err)
keys := []string{"test_key"}
// Execute
items, err := repo.ExistingItems(t.Context(), mockData.ID, keys)
assert.NoError(t, err)
assert.Len(t, items, 1)
// Cleanup
_ = feedRepo.Delete(t.Context(), mockData.ID)
_ = indexerRepo.Delete(t.Context(), int(indexer.ID))
_ = repo.Delete(t.Context(), mockData.ID, "test_key")
})
t.Run(fmt.Sprintf("ExistsItems_Nonexistent_Key [%s]", dbType), func(t *testing.T) {
// Execute
exists, err := repo.Exists(-1, "nonexistent_key")
assert.NoError(t, err)
assert.False(t, exists)
})
}
}
func TestFeedCacheRepo_Put(t *testing.T) {
for dbType, db := range testDBs {
log := setupLoggerForTest()