Ombi-app/Ombi
1
0
Fork 0
mirror of https://github.com/Ombi-app/Ombi.git synced 2025-12-11 04:37:57 -06:00
Code Issues Packages Projects Releases 456 Wiki Activity

improve batching

Browse Source
This commit is contained in:
tidusjar 2025-10-14 20:34:49 +01:00
parent e43c1c428f
commit 03f38401e7
2 changed files with 29 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/Ombi.Schedule/Jobs/Emby/EmbyEpisodeSync.cs
View File

@ -62,7 +62,8 @@ namespace Ombi.Schedule.Jobs.Emby
private readonly IEmbyContentRepository _repo;
private readonly INotificationHubService _notification;
private const int AmountToTake = 300;
private const int AmountToTake = 500;
private const int DatabaseBatchSize = 1000;
private IEmbyApi Api { get; set; }
@ -122,12 +123,17 @@ namespace Ombi.Schedule.Jobs.Emby
{
allEpisodes = await Api.GetAllEpisodes(server.ApiKey, parentIdFilter, 0, AmountToTake, server.AdministratorId, server.FullUri);
}
var total = allEpisodes.TotalRecordCount;
var processed = 0;
var epToAdd = new HashSet<EmbyEpisode>();
var episodesInCurrentBatch = new HashSet<string>(); // Track episodes in current batch to avoid duplicates
_logger.LogInformation($"Processing {total} episodes in chunks of {AmountToTake}");
while (processed < total)
{
// Process episodes in current chunk
foreach (var ep in allEpisodes.Items)
{
processed++;
@ -195,18 +201,30 @@ namespace Ombi.Schedule.Jobs.Emby
}
}
await _repo.AddRange(epToAdd);
epToAdd.Clear();
episodesInCurrentBatch.Clear();
if (!recentlyAdded)
// Only commit to database when we reach the batch size or finish processing
if (epToAdd.Count >= DatabaseBatchSize || processed >= total)
{
if (epToAdd.Any())
{
await _repo.AddRange(epToAdd);
_logger.LogInformation($"Committed {epToAdd.Count} episodes to database. Progress: {processed}/{total}");
}
epToAdd.Clear();
episodesInCurrentBatch.Clear();
}
// Get next chunk of episodes for processing
if (!recentlyAdded && processed < total)
{
allEpisodes = await Api.GetAllEpisodes(server.ApiKey, parentIdFilter, processed, AmountToTake, server.AdministratorId, server.FullUri);
}
}
// Final commit for any remaining episodes
if (epToAdd.Any())
{
await _repo.AddRange(epToAdd);
_logger.LogInformation($"Final commit: {epToAdd.Count} episodes");
}
}

10
src/Ombi.Store/Repository/EmbyContentRepository.cs
View File

@ -114,18 +114,20 @@ namespace Ombi.Store.Repository
// Performance optimization methods
public async Task<HashSet<string>> GetAllSeriesEmbyIds()
{
return await Db.EmbyContent
var ids = await Db.EmbyContent
.Where(x => !string.IsNullOrEmpty(x.EmbyId))
.Select(x => x.EmbyId)
.ToHashSetAsync();
.ToListAsync();
return new HashSet<string>(ids);
}
public async Task<HashSet<string>> GetAllEpisodeEmbyIds()
{
return await Db.EmbyEpisode
var ids = await Db.EmbyEpisode
.Where(x => !string.IsNullOrEmpty(x.EmbyId))
.Select(x => x.EmbyId)
.ToHashSetAsync();
.ToListAsync();
return new HashSet<string>(ids);
}
}
}