C#: Make the restore sources project/solution specific.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -119,7 +119,7 @@ public HashSet<AssemblyLookupLocation> Restore()
             compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", CheckNugetFeedResponsiveness ? "1" : "0"));
 
             HashSet<string> explicitFeeds = [];
-            string? explicitNugetSources = null;
+            HashSet<string> reachableFeeds = [];
 
             try
             {
@@ -137,9 +137,11 @@ public HashSet<AssemblyLookupLocation> Restore()
                         compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString()));
                     }
 
-                    var timeout = CheckSpecifiedFeeds(explicitFeeds, out var reachableFeeds);
-                    var allReachable = explicitFeeds.Count == reachableFeeds.Count;
-                    EmitUnreachableFeedsDiagnostics(allReachable);
+                    var timeout = CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds);
+                    reachableFeeds.UnionWith(reachableExplicitFeeds);
+
+                    var allExplicitReachable = explicitFeeds.Count == reachableExplicitFeeds.Count;
+                    EmitUnreachableFeedsDiagnostics(allExplicitReachable);
 
                     if (timeout)
                     {
@@ -154,16 +156,6 @@ public HashSet<AssemblyLookupLocation> Restore()
                     // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific).
                     CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds);
                     reachableFeeds.UnionWith(reachableInheritedFeeds);
-
-                    // If feed responsiveness is being checked, we only want to use the feeds that are reachable (note this set includes private
-                    // registry feeds if they are reachable).
-                    explicitNugetSources = MakeRestoreSourcesArgument(reachableFeeds);
-                }
-                else if (HasPrivateRegistryFeeds)
-                {
-                    // If private registries are configured they need to be included as sources for the restore, which requires that
-                    // they are provided as source arguments for the restore. The private registries are included in the `allFeeds` set.
-                    explicitNugetSources = MakeRestoreSourcesArgument(allFeeds);
                 }
 
                 using (var nuget = new NugetExeWrapper(fileProvider, legacyPackageDirectory, logger, IsDefaultFeedReachable))
@@ -206,9 +198,9 @@ public HashSet<AssemblyLookupLocation> Restore()
             }
 
             // Restore project dependencies with `dotnet restore`.
-            var restoredProjects = RestoreSolutions(explicitNugetSources, out var container);
+            var restoredProjects = RestoreSolutions(reachableFeeds, out var container);
             var projects = fileProvider.Projects.Except(restoredProjects);
-            RestoreProjects(projects, explicitNugetSources, out var containers);
+            RestoreProjects(projects, reachableFeeds, out var containers);
 
             var dependencies = containers.Flatten(container);
 
@@ -313,7 +305,7 @@ private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNu
         /// Populates dependencies with the relevant dependencies from the assets files generated by the restore.
         /// Returns a list of projects that are up to date with respect to restore.
         /// </summary>
-        private IEnumerable<string> RestoreSolutions(string? nugetSources, out DependencyContainer dependencies)
+        private IEnumerable<string> RestoreSolutions(HashSet<string> reachableFeeds, out DependencyContainer dependencies)
         {
             var successCount = 0;
             var nugetSourceFailures = 0;
@@ -326,6 +318,7 @@ private IEnumerable<string> RestoreSolutions(string? nugetSources, out Dependenc
             var projects = fileProvider.Solutions.SelectMany(solution =>
                 {
                     logger.LogInfo($"Restoring solution {solution}...");
+                    var nugetSources = MakeRestoreSourcesArgument(solution, reachableFeeds);
                     var res = dotnet.Restore(new(solution, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows));
                     if (res.Success)
                     {
@@ -350,7 +343,7 @@ private IEnumerable<string> RestoreSolutions(string? nugetSources, out Dependenc
             return projects;
         }
 
-        private string? MakeRestoreSourcesArgument(IEnumerable<string> feeds)
+        private string FeedsToRestoreArgument(IEnumerable<string> feeds)
         {
             // If there are no feeds, we want to override any default feeds that `dotnet restore` would use by passing a dummy source argument.
             if (!feeds.Any())
@@ -369,14 +362,46 @@ private IEnumerable<string> RestoreSolutions(string? nugetSources, out Dependenc
             return feedArgs.ToString();
         }
 
+        /// <summary>
+        /// Constructs the list of NuGet sources to use for this restore.
+        // (1) Use the feeds we get from `dotnet nuget list source`
+        // (2) Use private registries, if they are configured
+        /// </summary>
+        /// <param name="path">Path to project/solution</param>
+        /// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
+        /// <returns>A string representing the NuGet sources argument for the restore command.</returns>
+        private string? MakeRestoreSourcesArgument(string path, HashSet<string> reachableFeeds)
+        {
+            // Do not construct an set of explicit NuGet sources to use for restore.
+            if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
+            {
+                return null;
+            }
+
+            // Find the path specific feeds.
+            var folder = GetDirectoryName(path);
+            var feedsToConsider = folder is not null ? GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder)).ToHashSet() : [];
+
+            if (HasPrivateRegistryFeeds)
+            {
+                feedsToConsider.UnionWith(PrivateRegistryFeeds);
+            }
+
+            var feedsToUse = CheckNugetFeedResponsiveness
+                ? feedsToConsider.Where(reachableFeeds.Contains)
+                : feedsToConsider;
+
+            return FeedsToRestoreArgument(feedsToUse);
+        }
+
         /// <summary>
         /// Executes `dotnet restore` on all projects in projects.
         /// This is done in parallel for performance reasons.
         /// Populates dependencies with the relative paths to the assets files generated by the restore.
         /// </summary>
         /// <param name="projects">A list of paths to project files.</param>
-        /// <param name="explicitRestoreSources">The explicit restore sources argument.</param>
-        private void RestoreProjects(IEnumerable<string> projects, string? explicitRestoreSources, out ConcurrentBag<DependencyContainer> dependencies)
+        /// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
+        private void RestoreProjects(IEnumerable<string> projects, HashSet<string> reachableFeeds, out ConcurrentBag<DependencyContainer> dependencies)
         {
             var successCount = 0;
             var nugetSourceFailures = 0;
@@ -393,7 +418,8 @@ private void RestoreProjects(IEnumerable<string> projects, string? explicitResto
                 foreach (var project in projectGroup)
                 {
                     logger.LogInfo($"Restoring project {project}...");
-                    var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: explicitRestoreSources, TargetWindows: isWindows));
+                    var nugetSources = MakeRestoreSourcesArgument(project, reachableFeeds);
+                    var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows));
                     assets.AddDependenciesRange(res.AssetsFilePaths);
                     lock (sync)
                     {
@@ -898,6 +924,19 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
             }
         }
 
+        private string? GetDirectoryName(string path)
+        {
+            try
+            {
+                return new FileInfo(path).Directory?.FullName;
+            }
+            catch (Exception exc)
+            {
+                logger.LogWarning($"Failed to get directory of '{path}': {exc}");
+            }
+            return null;
+        }
+
         private (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
         {
             var nugetConfigs = fileProvider.NugetConfigs;
@@ -968,18 +1007,7 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
             {
                 // We don't have to get the feeds from each of the folders from below, it would be enought to check the folders that recursively contain the others.
                 var nugetConfigFeeds = nugetConfigs
-                    .Select(config =>
-                    {
-                        try
-                        {
-                            return new FileInfo(config).Directory?.FullName;
-                        }
-                        catch (Exception exc)
-                        {
-                            logger.LogWarning($"Failed to get directory of '{config}': {exc}");
-                        }
-                        return null;
-                    })
+                    .Select(GetDirectoryName)
                     .Where(folder => folder != null)
                     .SelectMany(folder => GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder!)))
                     .ToHashSet();