idanoo/autobrr
1
0
Fork 0
mirror of https://github.com/idanoo/autobrr synced 2025-07-23 08:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(feeds): optimize existing cache items check (#2078)

Browse source 
* feat(feeds): optimize existing items cache check

* feat(feeds): remove ttl from repo method ExistingItems

* feat(feeds): add db integration test for ExistingItems

* feat(feeds): improve release and filter processing

* feat(feeds): fix failing test
This commit is contained in:
ze0s 2025-06-07 12:46:08 +02:00 • committed by GitHub
parent 92ddb919a5
commit 46f6fbe5cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 266 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

33
internal/feed/rss.go
View file

@ -105,7 +105,7 @@ func (j *RSSJob) process(ctx context.Context) error {
}
// process all new releases
go j.ReleaseSvc.ProcessMultiple(releases)
go j.ReleaseSvc.ProcessMultipleFromIndexer(releases, j.Feed.Indexer)
return nil
}
@ -280,11 +280,8 @@ func (j *RSSJob) getFeed(ctx context.Context) (items []*gofeed.Item, err error)
}
//sort.Sort(feed)
toCache := make([]domain.FeedCacheItem, 0)
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
guidItemMap := make(map[string]*gofeed.Item)
var guids []string
for _, item := range feed.Items {
key := item.GUID
@ -295,12 +292,22 @@ func (j *RSSJob) getFeed(ctx context.Context) (items []*gofeed.Item, err error)
}
}
exists, err := j.CacheRepo.Exists(j.Feed.ID, key)
if err != nil {
j.Log.Error().Err(err).Msg("could not check if item exists")
continue
}
if exists {
guidItemMap[key] = item
guids = append(guids, key)
}
existingGuids, err := j.CacheRepo.ExistingItems(ctx, j.Feed.ID, guids)
if err != nil {
j.Log.Error().Err(err).Msgf("error getting existing items from cache")
return
}
// set ttl to 1 month
ttl := time.Now().AddDate(0, 1, 0)
toCache := make([]domain.FeedCacheItem, 0)
for guid, item := range guidItemMap {
if existingGuids[guid] {
j.Log.Trace().Msgf("cache item exists, skipping release: %s", item.Title)
continue
}
@ -309,7 +316,7 @@ func (j *RSSJob) getFeed(ctx context.Context) (items []*gofeed.Item, err error)
toCache = append(toCache, domain.FeedCacheItem{
FeedId: strconv.Itoa(j.Feed.ID),
Key: key,
Key: guid,
Value: []byte(item.Title),
TTL: ttl,
})