Only look at NuGet when cleaning darc feeds (#4448)

dkurepa · web-flow · commit 984a3da8411b · 2025-02-13T18:45:51.000+01:00
#4266 Discussed with Matt, we don't need to look at AzDo feeds anymore, everything that's released should be in NuGet
diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs
@@ -10,8 +10,6 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
-using PackagesInReleaseFeeds = System.Collections.Generic.Dictionary<string, System.Collections.Generic.Dictionary<string, System.Collections.Generic.HashSet<string>>>;
-
 namespace ProductConstructionService.FeedCleaner;
 
 public class FeedCleaner
@@ -24,15 +22,16 @@ public class FeedCleaner
     public FeedCleaner(
         IAzureDevOpsClient azureDevOpsClient,
         BuildAssetRegistryContext context,
+        IHttpClientFactory httpClientFactory,
         ILogger<FeedCleaner> logger)
     {
         _azureDevOpsClient = azureDevOpsClient;
         _context = context;
         _logger = logger;
-        _httpClient = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true });
+        _httpClient = httpClientFactory.CreateClient();
     }
 
-    public async Task CleanFeedAsync(AzureDevOpsFeed feed, PackagesInReleaseFeeds packagesInReleaseFeeds)
+    public async Task CleanFeedAsync(AzureDevOpsFeed feed)
     {
         try
         {
@@ -44,10 +43,10 @@ public async Task CleanFeedAsync(AzureDevOpsFeed feed, PackagesInReleaseFeeds pa
 
             foreach (var package in packages)
             {
-                HashSet<string> updatedVersions = await UpdateReleasedVersionsForPackageAsync(feed, package, packagesInReleaseFeeds);
+                HashSet<Asset> updatedAssets = await UpdateReleasedVersionsForPackageAsync(feed, package);
 
-                await DeletePackageVersionsFromFeedAsync(feed, package.Name, updatedVersions);
-                updatedCount += updatedVersions.Count;
+                await DeletePackageVersionsFromFeedAsync(feed, updatedAssets);
+                updatedCount += updatedAssets.Count;
             }
 
             _logger.LogInformation("Feed {feed} cleaning finished with {count}/{totalCount} updated packages", feed.Name, updatedCount, packages.Count);
@@ -74,12 +73,11 @@ public async Task CleanFeedAsync(AzureDevOpsFeed feed, PackagesInReleaseFeeds pa
     /// <param name="package">Package to search for</param>
     /// <param name="dotnetFeedsPackageMapping">Mapping of packages and their versions in the release feeds</param>
     /// <returns>Collection of versions that were updated for the package</returns>
-    private async Task<HashSet<string>> UpdateReleasedVersionsForPackageAsync(
+    private async Task<HashSet<Asset>> UpdateReleasedVersionsForPackageAsync(
         AzureDevOpsFeed feed,
-        AzureDevOpsPackage package,
-        PackagesInReleaseFeeds dotnetFeedsPackageMapping)
+        AzureDevOpsPackage package)
     {
-        var releasedVersions = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        HashSet<Asset> releasedAssets = new();
 
         foreach (var version in package.Versions)
         {
@@ -101,119 +99,97 @@ private async Task<HashSet<string>> UpdateReleasedVersionsForPackageAsync(
                 continue;
             }
 
-            if (matchingAsset.Locations.Any(l => l.Location == FeedConstants.NuGetOrgLocation ||
-                                                 dotnetFeedsPackageMapping.Any(f => l.Location == f.Key)))
+            if (matchingAsset.Locations.Any(l => l.Location == FeedConstants.NuGetOrgLocation))
             {
                 _logger.LogInformation("Package {package}.{version} is already present in a public location.",
                     package.Name,
                     version.Version);
-                releasedVersions.Add(version.Version);
+                releasedAssets.Add(matchingAsset);
                 continue;
             }
 
-            List<string> feedsWherePackageIsAvailable = GetReleaseFeedsWherePackageIsAvailable(
-                package.Name,
-                version.Version,
-                dotnetFeedsPackageMapping);
-
             try
             {
-                if (await IsPackageAvailableInNugetOrgAsync(package.Name, version.Version))
+                if (!await IsPackageAvailableInNugetOrgAsync(package.Name, version.Version))
                 {
-                    feedsWherePackageIsAvailable.Add(FeedConstants.NuGetOrgLocation);
+                    _logger.LogInformation("Package {package}.{version} not found in any of the release feeds", package.Name, version);
+                    continue;
                 }
             }
             catch (HttpRequestException e)
             {
                 _logger.LogWarning(e, "Failed to determine if package {package}.{version} is present in NuGet.org",
                     package.Name,
                     version.Version);
-            }
-
-            if (feedsWherePackageIsAvailable.Count <= 0)
-            {
-                _logger.LogInformation("Package {package}.{version} not found in any of the release feeds", package.Name, version);
                 continue;
             }
 
-            releasedVersions.Add(version.Version);
-            foreach (string feedToAdd in feedsWherePackageIsAvailable)
-            {
-                _logger.LogInformation("Found package {package}.{version} in {feed}, adding location to asset",
-                    package.Name,
-                    version.Version,
-                    feedToAdd);
+            releasedAssets.Add(matchingAsset);
 
-                matchingAsset.Locations.Add(new AssetLocation()
-                {
-                    Location = feedToAdd,
-                    Type = LocationType.NugetFeed
-                });
+            _logger.LogInformation("Found package {package}.{version} in {feed}, adding location to asset",
+                package.Name,
+                version.Version,
+                FeedConstants.NuGetOrgLocation);
 
-                await _context.SaveChangesAsync();
-            }
+            matchingAsset.Locations.Add(new AssetLocation()
+            {
+                Location = FeedConstants.NuGetOrgLocation,
+                Type = LocationType.NugetFeed
+            });
+
+            await _context.SaveChangesAsync();
         }
 
-        return releasedVersions;
+        return releasedAssets;
     }
 
     /// <summary>
     /// Deletes a version of a package from an Azure DevOps feed
     /// </summary>
     /// <param name="feed">Feed to delete the package from</param>
     /// <param name="packageName">package to delete</param>
-    /// <param name="versionsToDelete">Collection of versions to delete</param>
+    /// <param name="assetsToDelete">Collection of versions to delete</param>
     private async Task DeletePackageVersionsFromFeedAsync(
         AzureDevOpsFeed feed,
-        string packageName,
-        HashSet<string> versionsToDelete)
+        HashSet<Asset> assetsToDelete)
     {
-        foreach (string version in versionsToDelete)
+        foreach (Asset asset in assetsToDelete)
         {
             try
             {
                 _logger.LogInformation("Deleting package {package}.{version} from feed {feed}",
-                    packageName, version, feed.Name);
+                    asset.Name, asset.Version, feed.Name);
 
                 await _azureDevOpsClient.DeleteNuGetPackageVersionFromFeedAsync(
                     feed.Account,
                     feed.Project?.Name,
                     feed.Name,
-                    packageName,
-                    version);
+                    asset.Name,
+                    asset.Version);
+
+                var assetLocation = asset.Locations.FirstOrDefault(al => al.Location.Contains(feed.Name, StringComparison.OrdinalIgnoreCase));
+                if (assetLocation != null)
+                {
+                    asset.Locations.Remove(assetLocation);
+                }
             }
             catch (HttpRequestException e)
             {
                 _logger.LogError(e, "There was an error attempting to delete package {package}.{version} from the {feed} feed. Skipping...",
-                    packageName,
-                    version,
+                    asset.Name,
+                    asset.Version,
                     feed.Name);
             }
         }
-    }
 
-    /// <summary>
-    /// Gets a list of feeds where a given package is available
-    /// </summary>
-    /// <param name="name">Package to search for</param>
-    /// <param name="version">Version to search for</param>
-    /// <param name="packageMappings">Feeds to search</param>
-    /// <returns>List of feeds in the package mappings where the provided package and version are available</returns>
-    private static List<string> GetReleaseFeedsWherePackageIsAvailable(
-        string name,
-        string version,
-        PackagesInReleaseFeeds packageMappings)
-    {
-        List<string> feeds = [];
-        foreach ((string feedName, Dictionary<string, HashSet<string>> packages) in packageMappings)
+        try
         {
-            if (packages.TryGetValue(name, out HashSet<string>? versions) && versions.Contains(version))
-            {
-                feeds.Add(feedName);
-            }
+            await _context.SaveChangesAsync();
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "Failed to remove location {feed} from Assets in BAR", feed.Name);
         }
-
-        return feeds;
     }
 
     /// <summary>
diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerConfiguration.cs
@@ -34,6 +34,13 @@ public static void ConfigureFeedCleaner(this IHostApplicationBuilder builder, IT
         builder.Services.AddTransient<IAzureDevOpsClient, AzureDevOpsClient>();
         builder.Services.AddTransient<ILogger>(sp => sp.GetRequiredService<ILogger<FeedCleanerJob>>());
         builder.Services.AddTransient<IProcessManager>(sp => ActivatorUtilities.CreateInstance<ProcessManager>(sp, "git"));
+        builder.Services.AddHttpClient().ConfigureHttpClientDefaults(builder =>
+        {
+            builder.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
+            {
+                CheckCertificateRevocationList = true,
+            });
+        });
 
         builder.Services.AddTransient<FeedCleanerJob>();
         builder.Services.AddTransient<FeedCleaner>();
diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerJob.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerJob.cs
@@ -10,8 +10,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
-using PackagesInReleaseFeeds = System.Collections.Generic.Dictionary<string, System.Collections.Generic.Dictionary<string, System.Collections.Generic.HashSet<string>>>;
-
 namespace ProductConstructionService.FeedCleaner;
 
 public class FeedCleanerJob
@@ -43,15 +41,6 @@ public async Task CleanManagedFeedsAsync()
             return;
         }
 
-        _logger.LogInformation("Loading packages in release feeds...");
-
-        PackagesInReleaseFeeds packagesInReleaseFeeds = await GetPackagesForReleaseFeedsAsync();
-
-        _logger.LogInformation("Loaded {versionCount} versions of {packageCount} packages from {feedCount} feeds",
-            packagesInReleaseFeeds.Sum(feed => feed.Value.Sum(package => package.Value.Count)),
-            packagesInReleaseFeeds.Sum(feed => feed.Value.Keys.Count),
-            packagesInReleaseFeeds.Keys.Count);
-
         foreach (var azdoAccount in Options.AzdoAccounts)
         {
             _logger.LogInformation("Processing feeds for {account}...", azdoAccount);
@@ -93,7 +82,7 @@ await Parallel.ForEachAsync(
                     try
                     {
                         cancellationToken.ThrowIfCancellationRequested();
-                        await feedCleaner.CleanFeedAsync(feed, packagesInReleaseFeeds);
+                        await feedCleaner.CleanFeedAsync(feed);
                     }
                     catch (Exception e)
                     {
@@ -111,55 +100,4 @@ await Parallel.ForEachAsync(
                 azdoAccount);
         }
     }
-
-    /// <summary>
-    /// Get a mapping of feed -> (package, versions) for the release feeds so it
-    /// can be easily queried whether a version of a package is in a feed.
-    /// </summary>
-    /// <returns>Mapping of packages to versions for the release feeds.</returns>
-    private async Task<PackagesInReleaseFeeds> GetPackagesForReleaseFeedsAsync()
-    {
-        var packagesWithVersionsInReleaseFeeds = new PackagesInReleaseFeeds();
-        IEnumerable<ReleasePackageFeed> dotnetManagedFeeds = Options.ReleasePackageFeeds;
-        foreach ((string account, string project, string feedName) in dotnetManagedFeeds)
-        {
-            string readableFeedURL = ComputeAzureArtifactsNuGetFeedUrl(feedName, account, project);
-
-            packagesWithVersionsInReleaseFeeds[readableFeedURL] = await GetPackageVersionsForFeedAsync(account, project, feedName);
-        }
-        return packagesWithVersionsInReleaseFeeds;
-    }
-
-    /// <summary>
-    /// Construct a nuget feed URL for an Azure DevOps Artifact feed
-    /// </summary>
-    /// <param name="feedName">Name of the feed</param>
-    /// <param name="account">Azure DevOps Account where the feed is hosted</param>
-    /// <param name="project">Optional project for the feed.</param>
-    /// <returns>Url of the form https://pkgs.dev.azure.com/account/project/_packaging/feedName/nuget/v3/index.json </returns>
-    private static string ComputeAzureArtifactsNuGetFeedUrl(string feedName, string account, string project = "")
-    {
-        string projectSection = string.IsNullOrEmpty(project) ? "" : $"{project}/";
-        return $"https://pkgs.dev.azure.com/{account}/{projectSection}_packaging/{feedName}/nuget/v3/index.json";
-    }
-
-    /// <summary>
-    /// Gets a Mapping of package -> versions for an Azure DevOps feed.
-    /// </summary>
-    /// <param name="azdoClient">Azure DevOps client.</param>
-    /// <param name="account">Azure DevOps account.</param>
-    /// <param name="project">Azure DevOps project the feed is hosted in.</param>
-    /// <param name="feedName">Name of the feed</param>
-    /// <returns>Dictionary where the key is the package name, and the value is a HashSet of the versions of the package in the feed</returns>
-    private async Task<Dictionary<string, HashSet<string>>> GetPackageVersionsForFeedAsync(string account, string project, string feedName)
-    {
-        var packagesWithVersions = new Dictionary<string, HashSet<string>>();
-        List<AzureDevOpsPackage> packagesInFeed = await _azureDevOpsClient.GetPackagesForFeedAsync(account, project, feedName);
-        foreach (AzureDevOpsPackage package in packagesInFeed)
-        {
-            packagesWithVersions.Add(package.Name, new HashSet<string>(StringComparer.OrdinalIgnoreCase));
-            packagesWithVersions[package.Name].UnionWith(package.Versions?.Where(v => !v.IsDeleted).Select(v => v.Version) ?? []);
-        }
-        return packagesWithVersions;
-    }
 }
diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerOptions.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerOptions.cs
@@ -7,9 +7,5 @@ public class FeedCleanerOptions
 {
     public required bool Enabled { get; set; }
 
-    public required List<ReleasePackageFeed> ReleasePackageFeeds { get; set; }
-
     public required List<string> AzdoAccounts { get; set; }
 }
-
-public record ReleasePackageFeed(string Account, string Project, string Name);
diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.json b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.json
@@ -7,28 +7,6 @@
         }
     },
     "FeedCleaner": {
-        "Enabled": false,
-        "ReleasePackageFeeds": [
-            {
-                "Account": "dnceng",
-                "Project": "public",
-                "Name": "dotnet3"
-            },
-            {
-                "Account": "dnceng",
-                "Project": "public",
-                "Name": "dotnet3.1"
-            },
-            {
-                "Account": "dnceng",
-                "Project": "public",
-                "Name": "dotnet5"
-            },
-            {
-                "Account": "dnceng",
-                "Project": "public",
-                "Name": "dotnet-tools"
-            }
-        ]
+        "Enabled": false
     }
 }
diff --git a/test/ProductConstructionService.FeedCleaner.Tests/FeedCleanerTests.cs b/test/ProductConstructionService.FeedCleaner.Tests/FeedCleanerTests.cs

Original file line number	Diff line number	Diff line change
`@@ -7,9 +7,5 @@ public class FeedCleanerOptions`
`7`	`7`	`{`
`8`	`8`	`public required bool Enabled { get; set; }`
`9`	`9`
`10`		`- public required List<ReleasePackageFeed> ReleasePackageFeeds { get; set; }`
`11`		`-`
`12`	`10`	`public required List<string> AzdoAccounts { get; set; }`
`13`	`11`	`}`
`14`		`-`
`15`		`-public record ReleasePackageFeed(string Account, string Project, string Name);`