idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 08:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(feeds): add scheduled cleanup (#1073)

Browse source 
* refactor(feeds): make feed scheduling more robust

* feat(feeds): add daily cleanup job

* removes feed cache older than 30 days

* fix(feeds): fmt wrong type
This commit is contained in:
ze0s 2023-09-02 22:44:28 +02:00 • committed by GitHub
parent cfc2436d50
commit 6fd8626507
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 301 additions and 215 deletions
Download patch file Download diff file Expand all files Collapse all files

72
internal/database/feed_cache.go
View file

@ -28,14 +28,14 @@ func NewFeedCacheRepo(log logger.Logger, db *DB) domain.FeedCacheRepo {
}
}
func (r *FeedCacheRepo) Get(bucket string, key string) ([]byte, error) {
func (r *FeedCacheRepo) Get(feedId int, key string) ([]byte, error) {
queryBuilder := r.db.squirrel.
Select(
"value",
"ttl",
).
From("feed_cache").
Where(sq.Eq{"bucket": bucket}).
Where(sq.Eq{"feed_id": feedId}).
Where(sq.Eq{"key": key}).
Where(sq.Gt{"ttl": time.Now()})
@ -63,16 +63,16 @@ func (r *FeedCacheRepo) Get(bucket string, key string) ([]byte, error) {
return value, nil
}
func (r *FeedCacheRepo) GetByBucket(ctx context.Context, bucket string) ([]domain.FeedCacheItem, error) {
func (r *FeedCacheRepo) GetByFeed(ctx context.Context, feedId int) ([]domain.FeedCacheItem, error) {
queryBuilder := r.db.squirrel.
Select(
"bucket",
"feed_id",
"key",
"value",
"ttl",
).
From("feed_cache").
Where(sq.Eq{"bucket": bucket})
Where(sq.Eq{"feed_id": feedId})
query, args, err := queryBuilder.ToSql()
if err != nil {
@ -91,7 +91,7 @@ func (r *FeedCacheRepo) GetByBucket(ctx context.Context, bucket string) ([]domai
for rows.Next() {
var d domain.FeedCacheItem
if err := rows.Scan(&d.Bucket, &d.Key, &d.Value, &d.TTL); err != nil {
if err := rows.Scan(&d.FeedId, &d.Key, &d.Value, &d.TTL); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}
@ -105,12 +105,11 @@ func (r *FeedCacheRepo) GetByBucket(ctx context.Context, bucket string) ([]domai
return data, nil
}
func (r *FeedCacheRepo) GetCountByBucket(ctx context.Context, bucket string) (int, error) {
func (r *FeedCacheRepo) GetCountByFeed(ctx context.Context, feedId int) (int, error) {
queryBuilder := r.db.squirrel.
Select("COUNT(*)").
From("feed_cache").
Where(sq.Eq{"bucket": bucket})
Where(sq.Eq{"feed_id": feedId})
query, args, err := queryBuilder.ToSql()
if err != nil {
@ -131,12 +130,12 @@ func (r *FeedCacheRepo) GetCountByBucket(ctx context.Context, bucket string) (in
return count, nil
}
func (r *FeedCacheRepo) Exists(bucket string, key string) (bool, error) {
func (r *FeedCacheRepo) Exists(feedId int, key string) (bool, error) {
queryBuilder := r.db.squirrel.
Select("1").
Prefix("SELECT EXISTS (").
From("feed_cache").
Where(sq.Eq{"bucket": bucket}).
Where(sq.Eq{"feed_id": feedId}).
Where(sq.Eq{"key": key}).
Suffix(")")
@ -149,7 +148,7 @@ func (r *FeedCacheRepo) Exists(bucket string, key string) (bool, error) {
err = r.db.handler.QueryRow(query, args...).Scan(&exists)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return false, domain.ErrRecordNotFound
return false, nil
}
return false, errors.Wrap(err, "error query")
@ -158,11 +157,11 @@ func (r *FeedCacheRepo) Exists(bucket string, key string) (bool, error) {
return exists, nil
}
func (r *FeedCacheRepo) Put(bucket string, key string, val []byte, ttl time.Time) error {
func (r *FeedCacheRepo) Put(feedId int, key string, val []byte, ttl time.Time) error {
queryBuilder := r.db.squirrel.
Insert("feed_cache").
Columns("bucket", "key", "value", "ttl").
Values(bucket, key, val, ttl)
Columns("feed_id", "key", "value", "ttl").
Values(feedId, key, val, ttl)
query, args, err := queryBuilder.ToSql()
if err != nil {
@ -176,10 +175,10 @@ func (r *FeedCacheRepo) Put(bucket string, key string, val []byte, ttl time.Time
return nil
}
func (r *FeedCacheRepo) Delete(ctx context.Context, bucket string, key string) error {
func (r *FeedCacheRepo) Delete(ctx context.Context, feedId int, key string) error {
queryBuilder := r.db.squirrel.
Delete("feed_cache").
Where(sq.Eq{"bucket": bucket}).
Where(sq.Eq{"feed_id": feedId}).
Where(sq.Eq{"key": key})
query, args, err := queryBuilder.ToSql()
@ -195,10 +194,8 @@ func (r *FeedCacheRepo) Delete(ctx context.Context, bucket string, key string) e
return nil
}
func (r *FeedCacheRepo) DeleteBucket(ctx context.Context, bucket string) error {
queryBuilder := r.db.squirrel.
Delete("feed_cache").
Where(sq.Eq{"bucket": bucket})
func (r *FeedCacheRepo) DeleteByFeed(ctx context.Context, feedId int) error {
queryBuilder := r.db.squirrel.Delete("feed_cache").Where(sq.Eq{"feed_id": feedId})
query, args, err := queryBuilder.ToSql()
if err != nil {
@ -215,9 +212,36 @@ func (r *FeedCacheRepo) DeleteBucket(ctx context.Context, bucket string) error {
return errors.Wrap(err, "error exec result")
}
if rows == 0 {
r.log.Warn().Msgf("no rows affected for delete of bucket: %s", bucket)
}
r.log.Debug().Msgf("deleted %d rows from feed cache: %d", rows, feedId)
return nil
}
func (r *FeedCacheRepo) DeleteStale(ctx context.Context) error {
queryBuilder := r.db.squirrel.Delete("feed_cache")
if r.db.Driver == "sqlite" {
queryBuilder = queryBuilder.Where(sq.Expr("ttl < datetime('now', 'localtime', '-30 days')"))
} else {
queryBuilder = queryBuilder.Where(sq.Lt{"ttl": time.Now().AddDate(0, 0, -30)})
}
query, args, err := queryBuilder.ToSql()
if err != nil {
return errors.Wrap(err, "error building query")
}
result, err := r.db.handler.ExecContext(ctx, query, args...)
if err != nil {
return errors.Wrap(err, "error executing query")
}
rows, err := result.RowsAffected()
if err != nil {
return errors.Wrap(err, "error exec result")
}
r.log.Debug().Msgf("deleted %d rows from stale feed cache", rows)
return nil
}