[Rollout] Production rollout 2025-04-25 (#4742)

<!-- Link the GitHub or AzDO issue this pull request is associated with.
Please copy and paste the full URL rather than using the
dotnet/arcade-services# syntax -->
#4741

Author: adamzip

Directory.Packages.props

@@ -94,6 +94,7 @@
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
+    <PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.1" />

src/Microsoft.DotNet.Darc/DarcLib/Microsoft.DotNet.DarcLib.csproj

@@ -16,7 +16,7 @@
     <PackageReference Include="Microsoft.DotNet.Internal.Logging" />
     <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.Http" />
-    <PackageReference Include="Microsoft.Extensions.Logging" />
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="NuGet.Packaging" />

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VersionFileCodeFlowUpdater.cs

@@ -4,13 +4,15 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.DotNet.DarcLib.Helpers;
 using Microsoft.DotNet.DarcLib.Models;
 using Microsoft.DotNet.DarcLib.Models.Darc;
 using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
 using Microsoft.DotNet.ProductConstructionService.Client.Models;
+using Microsoft.Extensions.FileSystemGlobbing;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.Services.Common;
 using NuGet.Versioning;
@@ -90,6 +92,8 @@ public async Task<VersionFileUpdateResult> TryMergingBranchAndUpdateDependencies
             targetBranch,
             cancellationToken);
 
+        var excludedAssetsMatcher = GetExcludedAssetsMatcher(excludedAssets);
+
         if (mergeSuccessful)
         {
             try
@@ -99,7 +103,7 @@ public async Task<VersionFileUpdateResult> TryMergingBranchAndUpdateDependencies
                     targetRepo,
                     targetBranch,
                     build,
-                    excludedAssets,
+                    excludedAssetsMatcher,
                     lastFlow,
                     currentFlow,
                     cancellationToken);
@@ -124,7 +128,7 @@ public async Task<VersionFileUpdateResult> TryMergingBranchAndUpdateDependencies
             build,
             headBranch,
             targetBranch,
-            excludedAssets,
+            excludedAssetsMatcher,
             headBranchExisted,
             cancellationToken);
     }
@@ -203,7 +207,7 @@ private async Task<VersionFileUpdateResult> ResolveVersionFileConflicts(
         Build build,
         string headBranch,
         string branchToMerge,
-        IReadOnlyCollection<string>? excludedAssets,
+        Matcher? excludedAssetsMatcher,
         bool headBranchExisted,
         CancellationToken cancellationToken)
     {
@@ -258,7 +262,7 @@ async Task AbortMerge()
                 repo,
                 branchToMerge,
                 build,
-                excludedAssets,
+                excludedAssetsMatcher,
                 lastFlow,
                 (Backflow)currentFlow,
                 cancellationToken);
@@ -290,7 +294,7 @@ private async Task<List<DependencyUpdate>> BackflowDependenciesAndToolset(
         ILocalGitRepo targetRepo,
         string targetBranch,
         Build build,
-        IReadOnlyCollection<string>? excludedAssets,
+        Matcher? excludedAssetsMatcher,
         Codeflow lastFlow,
         Backflow currentFlow,
         CancellationToken cancellationToken)
@@ -316,17 +320,17 @@ private async Task<List<DependencyUpdate>> BackflowDependenciesAndToolset(
         var currentVmrDependencies = await GetVmrDependencies(vmr, mappingName, currentFlow.VmrSha);
 
         List<DependencyUpdate> repoChanges = ComputeChanges(
-            excludedAssets,
+            excludedAssetsMatcher,
             previousRepoDependencies,
             currentRepoDependencies);
 
         List<DependencyUpdate> vmrChanges = ComputeChanges(
-            excludedAssets,
+            excludedAssetsMatcher,
             previousVmrDependencies,
             currentVmrDependencies);
 
         List<AssetData> buildAssets = build.Assets
-            .Where(a => excludedAssets is null || !excludedAssets.Contains(a.Name))
+            .Where(a => !IsExcludedAsset(a.Name, excludedAssetsMatcher))
             .Select(a => new AssetData(a.NonShipping)
             {
                 Name = a.Name,
@@ -341,7 +345,7 @@ private async Task<List<DependencyUpdate>> BackflowDependenciesAndToolset(
             .Concat(vmrChanges.Select(c => c.From?.Name ?? c.To.Name))
             .Concat(repoChanges.Select(c => c.From?.Name ?? c.To.Name))
             .Distinct()
-            .Except(excludedAssets ?? [])
+            .Where(dep => !IsExcludedAsset(dep, excludedAssetsMatcher))
             .ToHashSet();
 
         var versionUpdates = new List<DependencyDetail>();
@@ -605,20 +609,39 @@ await targetRepo.CommitAsync(
         ];
     }
 
-    private static List<DependencyUpdate> ComputeChanges(IReadOnlyCollection<string>? excludedAssets, VersionDetails before, VersionDetails after)
+    private static Matcher? GetExcludedAssetsMatcher(IReadOnlyCollection<string>? excludedAssets)
+    {
+        if (excludedAssets == null || excludedAssets.Count == 0)
+        {
+            return null;
+        }
+        var matcher = new Matcher();
+        matcher.AddIncludePatterns(excludedAssets);
+        return matcher;
+    }
+
+    private static bool IsExcludedAsset(string asset, Matcher? excludedAssetsMatcher)
     {
-        bool IsExcluded(DependencyDetail dep) => excludedAssets?.Contains(dep.Name) ?? false;
+        if (excludedAssetsMatcher == null)
+        {
+            return false;
+        }
 
+        return excludedAssetsMatcher.Match(asset).HasMatches;
+    }
+
+    private static List<DependencyUpdate> ComputeChanges(Matcher? excludedAssetsMatcher, VersionDetails before, VersionDetails after)
+    {
         var dependencyChanges = before.Dependencies
-            .Where(dep => !IsExcluded(dep))
+            .Where(dep => !IsExcludedAsset(dep.Name, excludedAssetsMatcher))
             .Select(dep => new DependencyUpdate()
             {
                 From = dep,
             })
             .ToList();
 
         // Pair dependencies with the same name
-        foreach (var dep in after.Dependencies.Where(dep => !IsExcluded(dep)))
+        foreach (var dep in after.Dependencies.Where(dep => !IsExcludedAsset(dep.Name, excludedAssetsMatcher)))
         {
             var existing = dependencyChanges.FirstOrDefault(d => d.From?.Name == dep.Name);
             if (existing != null)

test/Microsoft.DotNet.DarcLib.Tests/VirtualMonoRepo/VersionFileCodeFlowUpdaterTests.cs

@@ -164,6 +164,8 @@ public async Task VersionsAreMergedInBackflowAfterForwardFlowTest()
         _versionDetails[$"repo/{lastFlow.RepoSha}"] = new VersionDetails(
             [
                 CreateDependency("Package.From.Build", "1.0.0", LastVmrSha),
+                CreateDependency("Package.Excluded.From.Backflow", "1.0.0", LastVmrSha),
+                CreateDependency("Package.Also.Excluded.From.Backflow", "1.0.0", LastVmrSha),
                 CreateDependency("Package.Removed.In.Repo", "1.0.0", LastVmrSha),
                 CreateDependency("Package.Updated.In.Both", "1.0.0", LastVmrSha),
                 CreateDependency("Package.Removed.In.VMR", "1.0.0", LastVmrSha), // Will be removed in VMR
@@ -174,6 +176,8 @@ public async Task VersionsAreMergedInBackflowAfterForwardFlowTest()
         _versionDetails[$"repo/{TargetBranch}"] = new VersionDetails(
             [
                 CreateDependency("Package.From.Build", "1.0.1", LastVmrSha), // Updated
+                CreateDependency("Package.Excluded.From.Backflow", "1.0.0", LastVmrSha),
+                CreateDependency("Package.Also.Excluded.From.Backflow", "1.0.0", LastVmrSha),
                 CreateDependency("Package.Updated.In.Both", "1.0.3", LastVmrSha), // Updated (vmr updated to 3.0.0)
                 CreateDependency("Package.Added.In.Repo", "1.0.0", LastVmrSha), // Added
                 CreateDependency("Package.Added.In.Both", "2.2.2", LastVmrSha), // Added in both
@@ -191,6 +195,8 @@ public async Task VersionsAreMergedInBackflowAfterForwardFlowTest()
         _versionDetails[$"vmr/{CurrentVmrSha}"] = new VersionDetails(
             [
                 CreateDependency("Package.From.Build", "1.0.0", LastVmrSha),
+                CreateDependency("Package.Excluded.From.Backflow", "1.0.0", LastVmrSha),
+                CreateDependency("Package.Also.Excluded.From.Backflow", "1.0.0", LastVmrSha),
                 CreateDependency("Package.Removed.In.Repo", "1.0.0", LastVmrSha),
                 CreateDependency("Package.Updated.In.Both", "3.0.0", LastVmrSha), // Updated (repo updated to 1.0.3)
                 CreateDependency("Package.Added.In.VMR", "2.0.0", LastVmrSha), // Added
@@ -202,6 +208,8 @@ public async Task VersionsAreMergedInBackflowAfterForwardFlowTest()
         var build = CreateNewBuild(CurrentVmrSha,
         [
             ("Package.From.Build", "1.0.5"),
+            ("Package.Excluded.From.Backflow", "1.0.2"),
+            ("Package.Also.Excluded.From.Backflow", "1.0.2"),
             ("Another.Package.From.Build", "1.0.5"),
             ("Yet.Another.Package.From.Build", "1.0.5")
         ]);
@@ -215,13 +223,17 @@ public async Task VersionsAreMergedInBackflowAfterForwardFlowTest()
         //   - Package.Removed.In.VMR - removed in VMR (and thus in repo)
         //   - Package.Added.In.Repo: 1.0.0 - added in repo, so already there
         //   - Package.Added.In.VMR - added in VMR, so it was just added in the repo (not getting updated)
+        //   - Package.Excluded.From.Backflow - excluded from backflow
+        //   - Package.Also.Excluded.From.Backflow - excluded from backflow
         await TestConflictResolver(
             build,
             lastFlow,
             currentFlow,
             expectedDependencies:
             [
                 ("Package.From.Build", "1.0.5"),
+                ("Package.Excluded.From.Backflow", "1.0.0"),
+                ("Package.Also.Excluded.From.Backflow", "1.0.0"),
                 ("Package.Updated.In.Both", "3.0.0"),
                 ("Package.Added.In.Repo", "1.0.0"),
                 ("Package.Added.In.VMR", "2.0.0"),
@@ -233,7 +245,8 @@ await TestConflictResolver(
                 new("Package.From.Build", "1.0.1", "1.0.5"),
                 new("Package.Updated.In.Both", "1.0.3", "3.0.0"),
             ],
-            headBranchExisted: false);
+            headBranchExisted: false,
+            excludedAssets: ["Package.Excluded.From.Backflow", "Package.Also.*"]);
 
         // Now we will add a new dependency to the PR branch
         // We will change a dependency in the repo too
@@ -260,6 +273,8 @@ await TestConflictResolver(
 
         build = CreateNewBuild(newVmrSha, [..build.Assets.Select(a => (a.Name, "1.0.6"))]);
 
+        // This time, don't exclude the assets from an update.
+        // We should see the updates.
         await TestConflictResolver(
             build,
             currentFlow,
@@ -268,6 +283,8 @@ await TestConflictResolver(
             [
                 // Same as before
                 ("Package.From.Build", "1.0.6"),
+                ("Package.Excluded.From.Backflow", "1.0.6"),
+                ("Package.Also.Excluded.From.Backflow", "1.0.6"),
                 ("Package.Updated.In.Both", "3.0.0"),
                 ("Package.Added.In.Repo", "1.0.0"),
                 ("Package.Added.In.VMR", "2.0.0"),
@@ -281,9 +298,12 @@ await TestConflictResolver(
             [
                 new("New.Package.In.Vmr", null, To: "4.0.0"),
                 new("New.Package.In.Repo", null, To: "4.0.0"),
+                new("Package.Excluded.From.Backflow", "1.0.0", To: "1.0.6"),
+                new("Package.Also.Excluded.From.Backflow", "1.0.0", To: "1.0.6"),
                 new("Package.From.Build", "1.0.5", To: "1.0.6"),
             ],
-            headBranchExisted: true);
+            headBranchExisted: true,
+            excludedAssets: []);
     }
 
     // Tests a case when conflicting updates were made in the repo and VMR.
@@ -327,7 +347,8 @@ public async Task ConflictingChangesThrowTest()
             currentFlow,
             [],
             [],
-            headBranchExisted: false);
+            headBranchExisted: false,
+            excludedAssets: []);
 
         await action.Should().ThrowAsync<ConflictingDependencyUpdateException>();
     }
@@ -338,7 +359,8 @@ private async Task TestConflictResolver(
         Backflow currentFlow,
         (string Name, string Version)[] expectedDependencies,
         ExpectedUpdate[] expectedUpdates,
-        bool headBranchExisted)
+        bool headBranchExisted,
+        string[] excludedAssets)
     {
         var gitFileChanges = new GitFileContentContainer();
         _dependencyFileManager
@@ -376,7 +398,7 @@ private async Task TestConflictResolver(
             build,
             PrBranch,
             TargetBranch,
-            excludedAssets: [],
+            excludedAssets: excludedAssets,
             headBranchExisted,
             cancellationToken);