feat(feeds): improve RSS (#502)

* feat(feeds): improve rss

* save last_run time
* remove interval check
* refactor feed job keys
* add rss test
* add max_age check

* feat(feeds): rss basic freeleech parsing

* feat(feeds): rss cookie support

* feat(feeds): db get max_age

* feat(feeds): update log messages

* feat(feeds): pass cookie to release for download

* feat(feeds): improve size parsing

* feat(feeds): improve datetime check
This commit is contained in:
ze0s 2022-10-18 18:51:10 +02:00 committed by GitHub
parent ac988f28f4
commit e2bb14afa4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 741 additions and 209 deletions

View file

@ -55,6 +55,74 @@ func (r *FeedCacheRepo) Get(bucket string, key string) ([]byte, error) {
return value, nil
}
func (r *FeedCacheRepo) GetByBucket(ctx context.Context, bucket string) ([]domain.FeedCacheItem, error) {
queryBuilder := r.db.squirrel.
Select(
"bucket",
"key",
"value",
"ttl",
).
From("feed_cache").
Where("bucket = ?", bucket)
query, args, err := queryBuilder.ToSql()
if err != nil {
return nil, errors.Wrap(err, "error building query")
}
rows, err := r.db.handler.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, "error executing query")
}
defer rows.Close()
var data []domain.FeedCacheItem
for rows.Next() {
var d domain.FeedCacheItem
if err := rows.Scan(&d.Bucket, &d.Key, &d.Value, &d.TTL); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}
data = append(data, d)
}
if err := rows.Err(); err != nil {
return nil, errors.Wrap(err, "row error")
}
return data, nil
}
func (r *FeedCacheRepo) GetCountByBucket(ctx context.Context, bucket string) (int, error) {
queryBuilder := r.db.squirrel.
Select("COUNT(*)").
From("feed_cache").
Where("bucket = ?", bucket)
query, args, err := queryBuilder.ToSql()
if err != nil {
return 0, errors.Wrap(err, "error building query")
}
row := r.db.handler.QueryRowContext(ctx, query, args...)
if err != nil {
return 0, errors.Wrap(err, "error executing query")
}
var count = 0
if err := row.Scan(&count); err != nil {
return 0, errors.Wrap(err, "error scanning row")
}
return count, nil
}
func (r *FeedCacheRepo) Exists(bucket string, key string) (bool, error) {
queryBuilder := r.db.squirrel.
Select("1").