idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 00:39:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(feeds): improve caching (#1191)

Browse source 
* feat(feeds): improve caching

* fix(feeds): put cache if not empty

* fix(feeds): reassign loop var

* fix(feeds): enable busy_timeout again

* fix(feeds): enable busy_timeout again
This commit is contained in:
ze0s 2023-10-21 17:03:52 +02:00 • committed by GitHub
parent 8c7c147328
commit 9793764905
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 88 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

29
internal/feed/newznab.go
View file

@ -144,7 +144,14 @@ func (j *NewznabJob) getFeed(ctx context.Context) ([]newznab.FeedItem, error) {
return feed.Channel.Items[i].PubDate.After(feed.Channel.Items[j].PubDate.Time)
})
toCache := make([]domain.FeedCacheItem, 0)
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
for _, i := range feed.Channel.Items {
i := i
if i.GUID == "" {
j.Log.Error().Msgf("missing GUID from feed: %s", j.Feed.Name)
continue
@ -163,18 +170,26 @@ func (j *NewznabJob) getFeed(ctx context.Context) ([]newznab.FeedItem, error) {
j.Log.Debug().Msgf("found new release: %s", i.Title)
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
if err := j.CacheRepo.Put(j.Feed.ID, i.GUID, []byte(i.Title), ttl); err != nil {
j.Log.Error().Stack().Err(err).Str("guid", i.GUID).Msg("cache.Put: error storing item in cache")
continue
}
toCache = append(toCache, domain.FeedCacheItem{
FeedId: strconv.Itoa(j.Feed.ID),
Key: i.GUID,
Value: []byte(i.Title),
TTL: ttl,
})
// only append if we successfully added to cache
items = append(items, *i)
}
if len(toCache) > 0 {
go func(items []domain.FeedCacheItem) {
ctx := context.Background()
if err := j.CacheRepo.PutMany(ctx, items); err != nil {
j.Log.Error().Err(err).Msg("cache.PutMany: error storing items in cache")
}
}(toCache)
}
// send to filters
return items, nil
}

22
internal/feed/rss.go
View file

@ -8,6 +8,7 @@ import (
"encoding/xml"
"net/url"
"regexp"
"strconv"
"time"
"github.com/autobrr/autobrr/internal/domain"
@ -240,6 +241,8 @@ func (j *RSSJob) getFeed(ctx context.Context) (items []*gofeed.Item, err error)
//sort.Sort(feed)
toCache := make([]domain.FeedCacheItem, 0)
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
@ -266,15 +269,26 @@ func (j *RSSJob) getFeed(ctx context.Context) (items []*gofeed.Item, err error)
j.Log.Debug().Msgf("found new release: %s", i.Title)
if err := j.CacheRepo.Put(j.Feed.ID, key, []byte(item.Title), ttl); err != nil {
j.Log.Error().Err(err).Str("entry", key).Msg("cache.Put: error storing item in cache")
continue
}
toCache = append(toCache, domain.FeedCacheItem{
FeedId: strconv.Itoa(j.Feed.ID),
Key: i.GUID,
Value: []byte(i.Title),
TTL: ttl,
})
// only append if we successfully added to cache
items = append(items, item)
}
if len(toCache) > 0 {
go func(items []domain.FeedCacheItem) {
ctx := context.Background()
if err := j.CacheRepo.PutMany(ctx, items); err != nil {
j.Log.Error().Err(err).Msg("cache.PutMany: error storing items in cache")
}
}(toCache)
}
// send to filters
return
}

31
internal/feed/torznab.go
View file

@ -208,9 +208,16 @@ func (j *TorznabJob) getFeed(ctx context.Context) ([]torznab.FeedItem, error) {
return feed.Channel.Items[i].PubDate.After(feed.Channel.Items[j].PubDate.Time)
})
toCache := make([]domain.FeedCacheItem, 0)
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
for _, i := range feed.Channel.Items {
i := i
if i.GUID == "" {
j.Log.Error().Err(err).Msgf("missing GUID from feed: %s", j.Feed.Name)
j.Log.Error().Msgf("missing GUID from feed: %s", j.Feed.Name)
continue
}
@ -226,18 +233,26 @@ func (j *TorznabJob) getFeed(ctx context.Context) ([]torznab.FeedItem, error) {
j.Log.Debug().Msgf("found new release: %s", i.Title)
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
if err := j.CacheRepo.Put(j.Feed.ID, i.GUID, []byte(i.Title), ttl); err != nil {
j.Log.Error().Stack().Err(err).Str("guid", i.GUID).Msg("cache.Put: error storing item in cache")
continue
}
toCache = append(toCache, domain.FeedCacheItem{
FeedId: strconv.Itoa(j.Feed.ID),
Key: i.GUID,
Value: []byte(i.Title),
TTL: ttl,
})
// only append if we successfully added to cache
items = append(items, *i)
}
if len(toCache) > 0 {
go func(items []domain.FeedCacheItem) {
ctx := context.Background()
if err := j.CacheRepo.PutMany(ctx, items); err != nil {
j.Log.Error().Err(err).Msg("cache.PutMany: error storing items in cache")
}
}(toCache)
}
// send to filters
return items, nil
}