Move UI tests to Windows 11 (#1866)

Calculator's UI tests currently run on Server 2022 VM images. However, the production app is primarily developed for Windows 11. This change moves the UI test jobs to run on Windows 11, so we can run our tests in an environment that's closer to what real users will experience.

build/pipelines/azure-pipelines.release.yaml

@@ -36,6 +36,16 @@ jobs:
     useReleaseAppxmanifest: true
     condition: not(eq(variables['Build.Reason'], 'PullRequest'))
 
+- template: ./templates/run-ui-tests.yaml
+  parameters:
+    platform: x64
+    runsettingsFileName: CalculatorUITests.release.runsettings
+
+- template: ./templates/run-ui-tests.yaml
+  parameters:
+    platform: x86
+    runsettingsFileName: CalculatorUITests.release.runsettings
+
 - template: ./templates/run-unit-tests.yaml
   parameters:
     platform: x64

build/pipelines/templates/run-ui-tests.yaml

@@ -12,17 +12,26 @@ jobs:
   condition: succeeded()
   pool:
     ${{ if eq(parameters.isOSSBuild, true) }}:
-      name: EssentialExperiencesOpenSource-windows-2022
+      name: EssentialExperiencesOpenSource-Win11
     ${{ if eq(parameters.isOSSBuild, false) }}:
-      name: EssentialExperiences-windows-2022
+      name: EssentialExperiences-Win11
   variables:
     skipComponentGovernanceDetection: true
   steps:
-  - checkout: none
+  - checkout: self
+    fetchDepth: 1
 
-  - powershell: Set-DisplayResolution -Width 1920 -Height 1080 -Force	
+  - task: PowerShell@2
-    displayName: Set resolution to 1920x1080	
+    displayName: Turn off animation effects
-    continueOnError: true	
+    inputs:
+      filePath: $(Build.SourcesDirectory)\build\scripts\TurnOffAnimationEffects.ps1
+
+  - task: ScreenResolutionUtility@1
+    displayName: Set resolution to 1920x1080
+    inputs:
+      displaySettings: 'specific'
+      width: 1920
+      height: 1080
 
   - task: DownloadBuildArtifacts@0
     displayName: Download MsixBundle and CalculatorUITests
@@ -52,4 +61,4 @@ jobs:
       platform: ${{ parameters.platform }}
       configuration: Release
       ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-        testFiltercriteria: Priority=0 
+        testFiltercriteria: Priority=0

build/scripts/TurnOffAnimationEffects.ps1

@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+<#
+.SYNOPSIS
+    Disables animations on the system. Equivalent to turning off the "Animation effects" setting in the Windows Settings app.
+#>
+
+Add-Type -AssemblyName System.Runtime.WindowsRuntime
+$asTaskGeneric = ([System.WindowsRuntimeSystemExtensions].GetMethods() | ? { $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1' })[0]
+
+Function WaitForAsyncAction($WinRtTask, $ResultType) {
+    $asTask = $asTaskGeneric.MakeGenericMethod($ResultType)
+    $task = $asTask.Invoke($null, @($WinRtTask))
+    $task.GetAwaiter().GetResult()
+}
+
+[Windows.UI.ViewManagement.Core.UISettingsController,Windows.UI.ViewManagement.Core,ContentType=WindowsRuntime] | Out-Null
+
+$controller = WaitForAsyncAction ([Windows.UI.ViewManagement.Core.UISettingsController]::RequestDefaultAsync())  ([Windows.UI.ViewManagement.Core.UISettingsController])
+
+$controller.SetAnimationsEnabled($false)

src/Calculator/Package.appxmanifest

@@ -8,7 +8,7 @@
         <Logo>Assets\CalculatorStoreLogo.png</Logo>
     </Properties>
     <Dependencies>
-        <TargetDeviceFamily Name="Windows.Universal" MinVersion="10.0.18362.0" MaxVersionTested="10.0.22000.0" />
+        <TargetDeviceFamily Name="Windows.Universal" MinVersion="10.0.22000.0" MaxVersionTested="10.0.22000.0" />
     </Dependencies>
     <Resources>
         <Resource Language="x-generate" />

src/CalculatorUITestFramework/CalculatorUITestFramework.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Appium.WebDriver" Version="4.3.1" />

src/CalculatorUITestFramework/ProgrammerOperatorsPanel.cs

@@ -172,17 +172,17 @@ namespace CalculatorUITestFramework
             else if (source.Contains("dwordButton"))
             {
                 DWordButton.Click();
-                WordButton.Click();
+                ResetWordSize();
-                ByteButton.Click();
             }
             else if (source.Contains("wordButton"))
             {
                 WordButton.Click();
-                ByteButton.Click();
+                ResetWordSize();
             }
             else if (source.Contains("byteButton"))
             {
                 ByteButton.Click();
+                ResetWordSize();
             }
             else
             {

src/CalculatorUITestFramework/StandardAoTCalculatorPage.cs

@@ -95,40 +95,29 @@ namespace CalculatorUITestFramework
             return ToolTip.Text;
         }
 
-        ///// <summary>
+        /// <summary>
-        ///// Checks in AoT (Keep on top) button is present
+        /// Checks in AoT (Keep on top) button is present
-        ///// </summary>
+        /// </summary>
-        public string GetAoTPresence()
+        public bool IsKeepOnTopButtonPresent()
         {
-            bool AoTPresent;
             string source = this.session.PageSource;
-            if (source.Contains("Keep on top"))
+            return source.Contains("Keep on top");
-            {
-                AoTPresent = true;
-            }
-            else
-            {
-                AoTPresent = false;
-            }
-            return AoTPresent.ToString();
         }
 
-        ///// <summary>
+        /// <summary>
-        ///// Checks Standard calculator to see if it's in AoT (Keep on top)
+        /// Checks Standard calculator to see if it's in AoT (Keep on top)
-        ///// </summary>
+        /// </summary>
-        public string AoTModeCheck()
+        public bool IsInAlwaysOnTopMode()
         {
-            bool InAoTMode;
             string source = this.session.PageSource;
             if ((source.Contains("Keep on top")) && (source.Contains("Header")))
             {
-                InAoTMode = false;
+                return false;
             }
             else
             {
-                InAoTMode = true;
+                return true;
             }
-            return InAoTMode.ToString();
         }
 
         /// <summary>
@@ -143,11 +132,11 @@ namespace CalculatorUITestFramework
         }
 
         /// <summary>
-        /// Increases the size of the window until History label for the History panel is visible
+        /// Increases the height of the window until invert button is visible
         /// </summary>
         private void GrowWindowToShowInvertButton(int height)
         {
-            if (height > 1000)
+            if (height > 700)
             {
                 throw new NotFoundException("Could not find the Invert Button");
             }
@@ -158,45 +147,7 @@ namespace CalculatorUITestFramework
                 WinAppDriver.Instance.CalculatorSession.Manage().Window.Size = new Size(width, height);
                 //give window time to render new size
                 System.Threading.Thread.Sleep(10);
-                GrowWindowToShowInvertButton(width + 100);
+                GrowWindowToShowInvertButton(height + 100);
-            }
-        }
-        /// <summary>
-        /// If the Invert button is not displayed, resize the window
-        /// Two attempts are made, the the button is not found a "not found" exception is thrown
-        /// </summary>
-        public void ResizeAoTWindowToDiplayInvertButton()
-        {
-            Point newWindowPostion = new Point(8, 8);
-            WinAppDriver.Instance.CalculatorSession.Manage().Window.Position = newWindowPostion;
-            string source0 = this.session.PageSource;
-            if (source0.Contains("invertButton"))
-            {
-                return;
-            }
-            else
-            {
-                Size newWindowSize = new Size(502, 502);
-                WinAppDriver.Instance.CalculatorSession.Manage().Window.Size = newWindowSize;
-                string source1 = this.session.PageSource;
-                if (source1.Contains("invertButton"))
-                {
-                    return;
-                }
-                else
-                {
-                    Size newWindowSize2 = new Size(750, 750);
-                    WinAppDriver.Instance.CalculatorSession.Manage().Window.Size = newWindowSize2;
-                }
-                string source2 = this.session.PageSource;
-                if (source2.Contains("invertButton"))
-                {
-                    return;
-                }
-                else
-                {
-                    throw new NotFoundException("Could not find the Invert Button");
-                }
             }
         }
     }

src/CalculatorUITestFramework/UnitConverterPage.cs

@@ -68,11 +68,9 @@ namespace CalculatorUITestFramework
         {
             CalculatorApp.ClickOnWindow();
             UnitConverterOperators.Units1.SendKeys(OpenQA.Selenium.Keys.Home);
-            UnitConverterOperators.Units1.SendKeys(OpenQA.Selenium.Keys.Enter);
 
             CalculatorApp.ClickOnWindow();
             UnitConverterOperators.Units2.SendKeys(OpenQA.Selenium.Keys.Home);
-            UnitConverterOperators.Units2.SendKeys(OpenQA.Selenium.Keys.Enter);
 
             CalculatorApp.ClickOnWindow();
         }

src/CalculatorUITests/CalculatorUITests.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
   <ItemGroup>

src/CalculatorUITests/StandardModeFunctionalTests.cs

@@ -181,10 +181,10 @@ namespace CalculatorUITests
             Assert.IsTrue(historyFlyoutItems[1].GetExpression().Equals("4 × 5=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
             page.HistoryPanel.ResizeWindowToDisplayHistoryLabel();
             var historyItems = page.HistoryPanel.GetAllHistoryListViewItems();
-            Assert.IsTrue(historyFlyoutItems[0].GetValue().Equals("3.333333333333333", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[0].GetValue().Equals("3.333333333333333", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
-            Assert.IsTrue(historyFlyoutItems[0].GetExpression().Equals("20 ÷ 6=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[0].GetExpression().Equals("20 ÷ 6=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
-            Assert.IsTrue(historyFlyoutItems[1].GetValue().Equals("20", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[1].GetValue().Equals("20", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
-            Assert.IsTrue(historyFlyoutItems[1].GetExpression().Equals("4 × 5=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[1].GetExpression().Equals("4 × 5=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
             page.HistoryPanel.ClearHistoryButton.Click();
             Assert.IsNotNull(WinAppDriver.Instance.CalculatorSession.FindElementByAccessibilityId("HistoryEmpty")); //verifies the History panel's clear history button
         }
@@ -666,7 +666,6 @@ namespace CalculatorUITests
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
             page.StandardAoTCalculatorPage.NavigateToStandardMode();
-
         }
 
         [TestMethod]
@@ -724,26 +723,22 @@ namespace CalculatorUITests
         {
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.ScientificCalculator);
             Assert.AreEqual("Scientific", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.GetAoTPresence());
 
             CalculatorApp.EnsureCalculatorHasFocus();
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.ProgrammerCalculator);
             Assert.AreEqual("Programmer", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.GetAoTPresence());
 
             CalculatorApp.EnsureCalculatorHasFocus();
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.DateCalculator);
             Assert.AreEqual("Date calculation", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.GetAoTPresence());
 
             CalculatorApp.EnsureCalculatorHasFocus();
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.StandardCalculator);
             Assert.AreEqual("Standard", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.GetAoTPresence());
         }
 
         [TestMethod]
@@ -751,12 +746,11 @@ namespace CalculatorUITests
         public void AoT_ErrorMessage_ResultUndefined()
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
-            page.StandardAoTCalculatorPage.ResizeAoTWindowToDisplayInvertButton();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
+
             page.StandardOperators.DivideButton.Click();
             page.StandardOperators.NumberPad.Num0Button.Click();
             page.StandardOperators.EqualButton.Click();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.AoTModeCheck());
             Assert.AreEqual("Result is undefined", page.CalculatorResults.GetAoTCalculatorResultText());
         }
 
@@ -765,11 +759,11 @@ namespace CalculatorUITests
         public void AoT_ErrorMessage_CannotDivideByZero()
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
             page.StandardAoTCalculatorPage.ResizeAoTWindowToDisplayInvertButton();
+
             page.StandardOperators.ClearButton.Click();
             page.StandardOperators.InvertButton.Click();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.AoTModeCheck());
             Assert.AreEqual("Cannot divide by zero", page.CalculatorResults.GetAoTCalculatorResultText());
         }
 
@@ -778,16 +772,16 @@ namespace CalculatorUITests
         public void AoT_ErrorMessage_MessageRetentionUponExitingAoT()
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
-            page.StandardAoTCalculatorPage.ResizeAoTWindowToDisplayInvertButton();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
-            page.StandardOperators.ClearButton.Click();
+
-            page.StandardOperators.InvertButton.Click();
+            page.StandardOperators.DivideButton.Click();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
+            page.StandardOperators.NumberPad.Num0Button.Click();
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.AoTModeCheck());
+            page.StandardOperators.EqualButton.Click();
-            Assert.AreEqual("Cannot divide by zero", page.CalculatorResults.GetAoTCalculatorResultText());
+            Assert.AreEqual("Result is undefined", page.CalculatorResults.GetAoTCalculatorResultText());
+
             page.StandardAoTCalculatorPage.NavigateToStandardMode();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.AoTModeCheck());
+            Assert.AreEqual("Result is undefined", page.CalculatorResults.GetCalculatorResultText());
-            Assert.AreEqual("Cannot divide by zero", page.CalculatorResults.GetCalculatorResultText());
         }
 
         #endregion