Well, actually, now that I look closer (code shared below... ), maybe there is more to it. I see, in the last part of the code, some logic that seems to stop the deletion once the size trigger is met...
What do your retention logs say? I guess, that might give us some more info. I wonder if it's just stopping, like you say, after 20GB is met.
The code is pretty old, and maybe it's a bug that's gone unnoticed because, I suppose, over time, this would eventually reduce it down to 20GB or so....
private async Task RunRetentionRuleAsync(Tables.FeedRetentionRules_Extended rule)
{
long feedSize = 0;
if (rule.SizeTrigger_KBytes != null && !rule.SizeExclusive_Indicator)
{
this.LogDebug($"Rule has an inclusive size trigger of {rule.SizeTrigger_KBytes} KB.");
this.LogInformation("Calculating feed size...");
this.StatusMessage = "Calculating feed size...";
feedSize = this.GetFeedSize();
this.LogDebug($"Feed size is {feedSize / 1024} KB.");
if ((feedSize / 1024) <= rule.SizeTrigger_KBytes.Value)
{
this.LogInformation("Feed is not taking up enough space to run rule. Skipping...");
return;
}
}
bool cachedOnly = rule.DeleteCached_Indicator;
if (cachedOnly)
this.LogDebug("Only delete cached packages.");
bool prereleaseOnly = rule.DeletePrereleaseVersions_Indicator;
if (prereleaseOnly)
this.LogDebug("Only delete prerelease packages.");
Regex keepRegex = null;
if (!string.IsNullOrWhiteSpace(rule.KeepPackageIds_Csv))
{
this.LogDebug("Never delete packages that match " + rule.KeepPackageIds_Csv);
keepRegex = BuildRegex(rule.KeepPackageIds_Csv);
}
Regex deleteRegex = null;
if (!string.IsNullOrWhiteSpace(rule.DeletePackageIds_Csv))
{
this.LogDebug("Only delete packages that match " + rule.DeletePackageIds_Csv);
deleteRegex = BuildRegex(rule.DeletePackageIds_Csv);
}
Regex keepVersionRegex = null;
if (!string.IsNullOrWhiteSpace(rule.KeepVersions_Csv))
{
this.LogDebug("Never delete packages that match " + rule.KeepVersions_Csv);
keepVersionRegex = BuildRegex(rule.KeepVersions_Csv);
}
Regex deleteVersionRegex = null;
if (!string.IsNullOrWhiteSpace(rule.DeleteVersions_Csv))
{
this.LogDebug("Only delete packages that match " + rule.DeleteVersions_Csv);
deleteVersionRegex = BuildRegex(rule.DeleteVersions_Csv);
}
bool lastUsedCheck = false;
var keepSinceDate = default(DateTime);
if (rule.KeepUsedWithin_Days != null)
{
keepSinceDate = DateTime.UtcNow.AddDays(-rule.KeepUsedWithin_Days.Value);
lastUsedCheck = true;
this.LogDebug($"Only delete packages that have not been requested in the last {rule.KeepUsedWithin_Days} days (since {keepSinceDate.ToLocalTime()})");
}
bool downloadCountCheck = false;
int minDownloadCount = 0;
if (rule.TriggerDownload_Count != null)
{
minDownloadCount = rule.TriggerDownload_Count.Value;
downloadCountCheck = true;
this.LogDebug($"Only delete packages that have been downloaded fewer than {minDownloadCount} times.");
}
if (rule.KeepVersions_Count != null)
this.LogDebug($"Never delete the most recent {rule.KeepVersions_Count} versions of packages.");
var matchingPackages = new Dictionary<string, List<TinyPackageVersion>>();
var versionPool = new InstancePool<string>();
this.LogInformation($"Finding packages that match retention rule {rule.Sequence_Number}...");
this.StatusMessage = $"Finding packages that match retention rule {rule.Sequence_Number}...";
foreach (var package in this.EnumeratePackages(cachedOnly, prereleaseOnly))
{
// skip noncached
if (cachedOnly && !package.Cached)
continue;
// skip stable
if (prereleaseOnly && !package.Prerelease)
continue;
// skip ids that match keep filter
if (keepRegex != null && keepRegex.IsMatch(package.Id))
continue;
// skip ids that do not match delete filter
if (deleteRegex != null && !deleteRegex.IsMatch(package.Id))
continue;
// skip ids that match keep filter
if (keepVersionRegex != null && keepVersionRegex.IsMatch(package.Version))
continue;
// skip ids that do not match delete filter
if (deleteVersionRegex != null && !deleteVersionRegex.IsMatch(package.Version))
continue;
// skip recently used packages
if (lastUsedCheck && package.LastUsed >= keepSinceDate)
continue;
// skip packages that have been downloaded enough times
if (downloadCountCheck && package.Downloads >= minDownloadCount)
continue;
List<TinyPackageVersion> versions;
if (!matchingPackages.TryGetValue(package.Id, out versions))
{
versions = new List<TinyPackageVersion>(10);
matchingPackages.Add(package.Id, versions);
}
versions.Add(new TinyPackageVersion(versionPool.Intern(package.Version), package.Size, package.Cached, package.Prerelease, package.Downloads, package.Extra));
}
int keepRecentVersionCount = rule.KeepVersions_Count ?? 0;
Comparison<TinyPackageVersion> versionComparison = (p1, p2) => this.CompareVersions(p1.Version, p2.Version);
foreach (var versions in matchingPackages.Values)
{
if (keepRecentVersionCount > 0 && versions.Count <= keepRecentVersionCount)
{
// make sure none of the versions are considered for deletion
versions.Clear();
}
else
{
// sort from lowest to highest
versions.Sort(versionComparison);
if (keepRecentVersionCount > 0 && versions.Count >= keepRecentVersionCount)
{
// remove recent versions
versions.RemoveRange(versions.Count - keepRecentVersionCount, keepRecentVersionCount);
}
}
}
if (rule.SizeTrigger_KBytes != null && rule.SizeExclusive_Indicator)
{
// finally have enough info to calculate matching size
this.LogDebug($"Rule has an exclusive size trigger of {rule.SizeTrigger_KBytes} KB.");
this.LogInformation("Calculating size of matching packages...");
this.StatusMessage = "Calculating size of matching packages...";
feedSize = matchingPackages.Values
.SelectMany(v => v)
.Sum(v => v.Size);
this.LogDebug($"Size of matching packages is {feedSize / 1024} KB.");
if ((feedSize / 1024) <= rule.SizeTrigger_KBytes.Value)
{
this.LogInformation("Matching packages are not taking up enough space to run rule. Skipping...");
return;
}
}
this.LogInformation("Getting count of matching packages...");
this.StatusMessage = "Getting count of matching packages...";
int matchCount = matchingPackages.Values.Sum(v => v.Count);
this.LogDebug($"{matchCount} packages qualify for deletion under this rule.");
var sortedMatches = from p in matchingPackages
from v in p.Value.Select((v2, i) => new { Id = p.Key, Version = v2, VersionIndex = i })
orderby v.Version.Cached descending, v.Version.Prerelease descending, v.VersionIndex
select v;
this.LogInformation("Deleting matching packages...");
this.StatusMessage = "Deleting matching packages...";
if (this.retentionDryRun)
this.LogDebug("Dry run mode is set; nothing will actually be deleted.");
long kbToDelete = rule.SizeTrigger_KBytes ?? -1;
long bytesDeleted = 0;
int deletedCount = 0;
foreach (var match in sortedMatches)
{
bytesDeleted += match.Version.Size;
deletedCount++;
this.LogDebug($"Deleting {match.Id} {match.Version.Version}...");
if (this.retentionDryRun)
{
this.DryRunDeleted.Add((match.Id, match.Version));
}
else
{
try
{
await this.DeletePackageAsync(match.Id, match.Version);
}
catch (Exception ex)
{
this.LogWarning($"Could not delete {match.Id} {match.Version.Version}: {ex}");
}
}
if (kbToDelete >= 0 && (bytesDeleted / 1024) >= kbToDelete)
{
this.LogDebug("Trigger size reached; stopping.");
break;
}
}
this.LogInformation($"Deleted {deletedCount} packages ({bytesDeleted / 1024} KB total).");
}
