Allow to disable test expansion on implementations of ITestDataSource

src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs

@@ -411,6 +411,16 @@ private static bool ProcessITestDataSourceTests(UnitTestElement test, Reflection
     {
         foreach (ITestDataSource dataSource in testDataSources)
         {
+            // When reaching this point the global setup is to expand the parameterized tests but each data source
+            // could decide not to expand so we need to respect it.
+            if (dataSource is IExpandableDataSource { ExpandDataSource: false })
+            {
+                // TODO: Improve multi-source design!
+                // Ideally we would want to consider each data source separately but when one source cannot be expanded,
+                // we will run all sources from the given method so we need to bail-out "globally".
+                return false;
+            }
+
             IEnumerable<object?[]>? data;
 
             // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests.

src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs

@@ -11,7 +11,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting;
 /// Attribute to define in-line data for a test method.
 /// </summary>
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-public class DataRowAttribute : Attribute, ITestDataSource
+public class DataRowAttribute : Attribute, ITestDataSource, IExpandableDataSource
 {
     /// <summary>
     /// Initializes a new instance of the <see cref="DataRowAttribute"/> class.
@@ -55,6 +55,8 @@ public DataRowAttribute(string?[]? stringArrayData)
     /// </summary>
     public string? DisplayName { get; set; }
 
+    bool IExpandableDataSource.ExpandDataSource => true;
+
     /// <inheritdoc />
     public IEnumerable<object?[]> GetData(MethodInfo methodInfo) => [Data];
 

src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs

@@ -33,7 +33,7 @@ public enum DynamicDataSourceType
 /// Attribute to define dynamic data for a test method.
 /// </summary>
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo
+public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, IExpandableDataSource
 {
     private readonly string _dynamicDataSourceName;
     private readonly DynamicDataSourceType _dynamicDataSourceType;
@@ -84,6 +84,12 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar
     /// </summary>
     public Type? DynamicDataDisplayNameDeclaringType { get; set; }
 
+    /// <summary>
+    /// Gets or sets a value indicating whether the data source should be expanded, allowing to treat each data row as a separate test.
+    /// The default value is <c>true</c>.
+    /// </summary>
+    public bool ExpandDataSource { get; set; } = true;
+
     /// <inheritdoc />
     public IEnumerable<object[]> GetData(MethodInfo methodInfo) => DynamicDataProvider.Instance.GetData(_dynamicDataDeclaringType, _dynamicDataSourceType, _dynamicDataSourceName, methodInfo);
 

src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryAttribute.cs

@@ -15,7 +15,23 @@ public class TestDataSourceDiscoveryAttribute : Attribute
     /// <param name="discoveryOption">
     /// The <see cref="TestDataSourceDiscoveryOption"/> to use when discovering <see cref="ITestDataSource"/> tests.
     /// </param>
-    public TestDataSourceDiscoveryAttribute(TestDataSourceDiscoveryOption discoveryOption) => DiscoveryOption = discoveryOption;
+    public TestDataSourceDiscoveryAttribute(TestDataSourceDiscoveryOption discoveryOption)
+        => DiscoveryOption = discoveryOption;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TestDataSourceDiscoveryAttribute"/> class.
+    /// Allows to control parameterized tests expansion. When expanded, each data source entry is considered as a
+    /// different test. Otherwise, multiple results are associated with the same test.
+    /// </summary>
+    /// <param name="expandDataSource">Define whether or not to expand data source during discovery.</param>
+    /// <remarks>
+    /// When a test is expanded, the associated data are serialized using DataContractSerializer which could cause issue
+    /// if your data is not serializable.
+    /// </remarks>
+    public TestDataSourceDiscoveryAttribute(bool expandDataSource)
+        => DiscoveryOption = expandDataSource
+            ? TestDataSourceDiscoveryOption.DuringDiscovery
+            : TestDataSourceDiscoveryOption.DuringExecution;
 
     /// <summary>
     /// Gets the discovery option.

src/TestFramework/TestFramework/Interfaces/IExpandableDataSource.cs

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestTools.UnitTesting;
+
+/// <summary>
+/// Defines whether the data source can be expanded or not.
+/// The expansion represents the capability to treat each data row as its own test which will impact test results
+/// and UI representation of the test.
+/// </summary>
+public interface IExpandableDataSource
+{
+    bool ExpandDataSource { get; }
+}

src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt

@@ -1 +1,6 @@
 #nullable enable
+Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.ExpandDataSource.get -> bool
+Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.ExpandDataSource.set -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.IExpandableDataSource
+Microsoft.VisualStudio.TestTools.UnitTesting.IExpandableDataSource.ExpandDataSource.get -> bool
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceDiscoveryAttribute.TestDataSourceDiscoveryAttribute(bool expandDataSource) -> void

test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs

@@ -1,7 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System.Collections.Immutable;
+
 using Microsoft.MSTestV2.CLIAutomation;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
 namespace MSTest.IntegrationTests;
 
@@ -21,8 +24,8 @@ public void CustomTestDataSourceTests()
         string assemblyPath = GetAssetFullPath(TestAssetName);
 
         // Act
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1");
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
 
         // Assert
         VerifyE2E.ContainsTestsPassed(testResults, "CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)");
@@ -34,8 +37,8 @@ public void AssertExtensibilityTests()
         string assemblyPath = GetAssetFullPath(TestAssetName);
 
         // Act
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest");
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
 
         // Assert
         VerifyE2E.ContainsTestsPassed(testResults, "BasicAssertExtensionTest", "ChainedAssertExtensionTest");
@@ -48,8 +51,8 @@ public void ExecuteCustomTestExtensibilityTests()
         string assemblyPath = GetAssetFullPath(TestAssetName);
 
         // Act
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)");
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
 
         // Assert
         VerifyE2E.ContainsTestsPassed(
@@ -75,8 +78,8 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests()
         string assemblyPath = GetAssetFullPath(TestAssetName);
 
         // Act
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2");
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
 
         // Assert
         VerifyE2E.TestsPassed(
@@ -94,4 +97,24 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests()
             "CustomTestMethod2 (\"C\")",
             "CustomTestMethod2 (\"C\")");
     }
+
+    public void WhenUsingCustomITestDataSourceWithExpansionDisabled_RespectSetting()
+    {
+        // Arrange
+        string assemblyPath = GetAssetFullPath(TestAssetName);
+
+        // Act
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "CustomDisableExpansionTestDataSourceTestMethod1");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
+
+        // Assert
+        Verify(testCases.Length == 1);
+
+        VerifyE2E.TestsPassed(
+            testResults,
+            "CustomDisableExpansionTestDataSourceTestMethod1 (1,2,3)",
+            "CustomDisableExpansionTestDataSourceTestMethod1 (4,5,6)");
+
+        VerifyE2E.TestsFailed(testResults);
+    }
 }

test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs

@@ -1,7 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System.Collections.Immutable;
+
 using Microsoft.MSTestV2.CLIAutomation;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
 namespace MSTest.IntegrationTests;
 
@@ -15,8 +18,8 @@ public void ExecuteDynamicDataTests()
         string assemblyPath = GetAssetFullPath(TestAssetName);
 
         // Act
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath);
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DynamicDataTests");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
 
         // Assert
         VerifyE2E.TestsPassed(
@@ -62,8 +65,8 @@ public void ExecuteDynamicDataTestsWithCategoryFilter()
         string assemblyPath = GetAssetFullPath(TestAssetName);
 
         // Act
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DynamicDataWithCategory");
-        System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DynamicDataWithCategory");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
 
         // Assert
         VerifyE2E.ContainsTestsPassed(
@@ -73,4 +76,33 @@ public void ExecuteDynamicDataTestsWithCategoryFilter()
 
         VerifyE2E.FailedTestCount(testResults, 0);
     }
+
+    public void ExecuteNonExpandableDynamicDataTests()
+    {
+        // Arrange
+        string assemblyPath = GetAssetFullPath(TestAssetName);
+
+        // Act
+        ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DisableExpansionTests");
+        ImmutableArray<TestResult> testResults = RunTests(testCases);
+
+        // Assert
+        Verify(testCases.Length == 5);
+
+        VerifyE2E.TestsPassed(
+            testResults,
+            "TestPropertySourceOnCurrentType (1,a)",
+            "TestPropertySourceOnCurrentType (2,b)",
+            "TestMethodSourceOnCurrentType (1,a)",
+            "TestMethodSourceOnCurrentType (2,b)",
+            "TestPropertySourceOnDifferentType (3,c)",
+            "TestPropertySourceOnDifferentType (3,c)",
+            "TestMethodSourceOnDifferentType (4,d)",
+            "TestMethodSourceOnDifferentType (4,d)",
+            "TestPropertyWithTwoSourcesButOnlyOneDisablingExpansion (1,a)",
+            "TestPropertyWithTwoSourcesButOnlyOneDisablingExpansion (2,b)",
+            "TestPropertyWithTwoSourcesButOnlyOneDisablingExpansion (3,c)",
+            "TestPropertyWithTwoSourcesButOnlyOneDisablingExpansion (4,d)");
+        VerifyE2E.FailedTestCount(testResults, 0);
+    }
 }

test/IntegrationTests/TestAssets/DynamicDataTestProject/DisableExpansionTests.cs

@@ -0,0 +1,60 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace DynamicDataTestProject;
+
+[TestClass]
+public sealed class DisableExpansionTests
+{
+    [TestMethod]
+    [DynamicData(nameof(PropertySource), ExpandDataSource = false)]
+    public void TestPropertySourceOnCurrentType(int a, string s)
+    {
+    }
+
+    [TestMethod]
+    [DynamicData(nameof(MethodSource), DynamicDataSourceType.Method, ExpandDataSource = false)]
+    public void TestMethodSourceOnCurrentType(int a, string s)
+    {
+    }
+
+    [TestMethod]
+    [DynamicData(nameof(PropertySource), typeof(DataSourceHelper), ExpandDataSource = false)]
+    public void TestPropertySourceOnDifferentType(int a, string s)
+    {
+    }
+
+    [TestMethod]
+    [DynamicData(nameof(MethodSource), typeof(DataSourceHelper), DynamicDataSourceType.Method, ExpandDataSource = false)]
+    public void TestMethodSourceOnDifferentType(int a, string s)
+    {
+    }
+
+    [TestMethod]
+    [DynamicData(nameof(PropertySource), ExpandDataSource = false)]
+    [DynamicData(nameof(PropertySource), typeof(DataSourceHelper))]
+    public void TestPropertyWithTwoSourcesButOnlyOneDisablingExpansion(int a, string s)
+    {
+    }
+
+    private static IEnumerable<object[]> PropertySource => MethodSource();
+
+    private static IEnumerable<object[]> MethodSource()
+    {
+        yield return new object[] { 1, "a" };
+        yield return new object[] { 2, "b" };
+    }
+}
+
+public class DataSourceHelper
+{
+    public static IEnumerable<object[]> PropertySource => MethodSource();
+
+    public static IEnumerable<object[]> MethodSource()
+    {
+        yield return new object[] { 3, "c" };
+        yield return new object[] { 4, "d" };
+    }
+}

test/IntegrationTests/TestAssets/FxExtensibilityTestProject/TestDataSourceExTests.cs

@@ -20,6 +20,12 @@ public void CustomTestDataSourceTestMethod1(int a, int b, int c)
         Assert.AreEqual(2, b % 3);
         Assert.AreEqual(0, c % 3);
     }
+
+    [TestMethod]
+    [CustomDisableExpansionTestDataSource]
+    public void CustomDisableExpansionTestDataSourceTestMethod1(int a, int b, int c)
+    {
+    }
 }
 
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
@@ -37,3 +43,13 @@ public class CustomEmptyTestDataSourceAttribute : Attribute, ITestDataSource
 
     public string GetDisplayName(MethodInfo methodInfo, object[] data) => data != null ? string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)) : null;
 }
+
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
+public class CustomDisableExpansionTestDataSourceAttribute : Attribute, ITestDataSource, IExpandableDataSource
+{
+    bool IExpandableDataSource.ExpandDataSource => false;
+
+    public IEnumerable<object[]> GetData(MethodInfo methodInfo) => [[1, 2, 3], [4, 5, 6]];
+
+    public string GetDisplayName(MethodInfo methodInfo, object[] data) => data != null ? string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)) : null;
+}