Delta client

doc/Settings.md

@@ -465,3 +465,15 @@ Enables numbered choices for ambiguous single-package `install`, `show`, and `do
 ```
 
 `--disable-interactivity` and redirected input or output still prevent prompting.
+
+### deltaIndex
+
+This feature changes how the client acquires updates to a pre-indexed source. Instead of downloading the whole index every time, it downloads a delta describing only what has changed since a periodically published baseline, and combines the two locally. The baseline is downloaded only when the source does not already have the one the delta was built against.
+
+The data presented by the source is the same either way; only how it is acquired differs. A source that does not publish a delta is unaffected, and the client falls back to downloading the full index.
+
+```json
+  "experimentalFeatures": {
+        "deltaIndex": true
+  },
+```

schemas/JSON/settings/settings.schema.0.2.json

@@ -348,6 +348,11 @@
           "description": "Enable support for some commands to resume",
           "type": "boolean",
           "default": false
+        },
+        "deltaIndex": {
+          "description": "Enable acquiring source index updates as a delta against a baseline, rather than downloading the full index",
+          "type": "boolean",
+          "default": false
         }
       }
     },

src/AppInstallerCLICore/Command.h

@@ -5,11 +5,11 @@
 #include "ExecutionContext.h"
 #include "Invocation.h"
 #include "Resources.h"
+#include <AppInstallerLanguageUtilities.h>
 #include <winget/UserSettings.h>
 #include <winget/ExperimentalFeature.h>
 #include <winget/GroupPolicy.h>
 
-#include <initializer_list>
 #include <memory>
 #include <optional>
 #include <ostream>
@@ -138,20 +138,6 @@ namespace AppInstaller::CLI
         std::string m_commandArguments;
     };
 
-    template <typename Container>
-    Container InitializeFromMoveOnly(std::initializer_list<typename Container::value_type> il)
-    {
-        using Value = typename Container::value_type;
-        Container result;
-
-        for (const auto& v : il)
-        {
-            result.emplace_back(std::move(*const_cast<Value*>(&v)));
-        }
-
-        return result;
-    }
-
     void ExecuteWithoutLoggingSuccess(Execution::Context& context, Command* command);
 
     int Execute(Execution::Context& context, std::unique_ptr<Command>& command);

src/AppInstallerCLIE2ETests/Constants.cs

@@ -53,6 +53,29 @@ public class Constants
         public const string RestTestSourceUrl = @"https://localhost:5001/TestKit/TestData/TestRestSource";
         public const string RestTestSourceType = "Microsoft.Rest";
 
+        // The delta capable test source. It is published separately from the source above so that a
+        // client that knows nothing about deltas keeps seeing exactly what it saw before.
+        public const string DeltaTestSourceName = @"TestDeltaSource";
+        public const string DeltaTestSourceDirectory = @"deltaSource";
+        public const string DeltaTestSourceUrl = @"https://localhost:5001/TestKit/" + DeltaTestSourceDirectory;
+        public const string DeltaTestSourceIdentityName = @"WingetE2EDelta.Tests";
+        public const string DeltaTestSourceDeltaIdentityName = @"WingetE2EDelta.Tests.Delta";
+        public const string DeltaTestSourceBaselineIdentityName = @"WingetE2EDelta.Tests.Baseline";
+        public const string DeltaTestSourceIdentifier = DeltaTestSourceIdentityName + "_" + AICLIPackagePublisherHash;
+        public const string DeltaTestSourceDeltaIdentifier = DeltaTestSourceDeltaIdentityName + "_" + AICLIPackagePublisherHash;
+        public const string DeltaTestSourceBaselineIdentifier = DeltaTestSourceBaselineIdentityName + "_" + AICLIPackagePublisherHash;
+
+        // The baseline is published older than the full index, because a baseline is captured once
+        // and then described by a succession of deltas.
+        public const string DeltaTestSourceVersion = @"2020.805.713.335";
+        public const string DeltaTestSourceBaselineVersion = @"2020.805.713.334";
+        public const string DeltaTestSourceBaselineRelativePath = @"baseline/source2.msix";
+
+        // Left out of the baseline, so that it exists only in the delta. Finding it is what tells a
+        // merged result apart from either half read on its own.
+        public const string DeltaOnlyManifestFileName = @"TestExampleInstaller.yaml";
+        public const string DeltaOnlyPackageIdentifier = @"AppInstallerTest.TestExampleInstaller";
+
         public const string AICLIPackageFamilyName = "WinGetDevCLI_8wekyb3d8bbwe";
         public const string AICLIPackageName = "WinGetDevCLI";
         public const string AICLIPackagePublisherHash = "8wekyb3d8bbwe";

src/AppInstallerCLIE2ETests/DeltaSource.cs

@@ -0,0 +1,247 @@
+// -----------------------------------------------------------------------------
+// <copyright file="DeltaSource.cs" company="Microsoft Corporation">
+//     Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+// </copyright>
+// -----------------------------------------------------------------------------
+
+namespace AppInstallerCLIE2ETests
+{
+    using AppInstallerCLIE2ETests.Helpers;
+    using NUnit.Framework;
+
+    /// <summary>
+    /// Tests the delta index against the deployed package mechanism.
+    ///
+    /// Unit tests cover the shape of a delta source and the merging of a delta with its baseline,
+    /// but they run unpackaged, so they can only ever reach the local file store. Everything here
+    /// is about the half they structurally cannot touch: deploying the packages, keeping three
+    /// separate deployed identities apart, and the clean up that follows a removal.
+    /// </summary>
+    public class DeltaSource : BaseCommand
+    {
+        /// <summary>
+        /// The experimental feature, as it is named in the settings file.
+        /// </summary>
+        private const string DeltaFeatureName = "deltaIndex";
+
+        /// <summary>
+        /// A package that is in the baseline, and so is not one the delta has to supply.
+        /// </summary>
+        private const string BaselinePackageIdentifier = Constants.PortableExePackageId;
+
+        /// <summary>
+        /// One time set up.
+        /// </summary>
+        [OneTimeSetUp]
+        public void OneTimeSetup()
+        {
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true);
+        }
+
+        /// <summary>
+        /// One time tear down.
+        /// </summary>
+        [OneTimeTearDown]
+        public void OneTimeTeardown()
+        {
+            RemoveDeltaSource();
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false);
+        }
+
+        /// <summary>
+        /// Test set up.
+        /// </summary>
+        [SetUp]
+        public void Setup()
+        {
+            RemoveDeltaSource();
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true);
+        }
+
+        /// <summary>
+        /// Adding a delta capable source acquires the delta and the baseline rather than the full
+        /// index, and what it then presents is the two of them merged.
+        /// </summary>
+        [Test]
+        public void DeltaSourceAdd()
+        {
+            AddDeltaSource();
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed");
+            Assert.That(
+                GetDeployedPackageVersion(Constants.DeltaTestSourceBaselineIdentityName),
+                Is.EqualTo(Constants.DeltaTestSourceBaselineVersion),
+                "The baseline should have been deployed at its own version");
+
+            // The baseline is a package of its own, so a source on the delta holds no full index at
+            // all. Were the two to share an identity this could not be asked.
+            Assert.That(
+                IsPackageDeployed(Constants.DeltaTestSourceIdentityName),
+                Is.False,
+                "The full index should not have been acquired");
+
+            // This package was left out of the baseline, so it exists only in the delta.
+            RequireFound(Constants.DeltaOnlyPackageIdentifier);
+
+            // And a package that is in the baseline is still there, so the delta did not displace it.
+            RequireFound(BaselinePackageIdentifier);
+        }
+
+        /// <summary>
+        /// A source added while the feature was off holds the full index, and goes on holding it.
+        ///
+        /// Which packages a source is made of is established when it is added and recorded in its
+        /// details; an update reads those details and never probes for anything else. So turning
+        /// the feature on does not move an existing source onto the delta, and the source has to be
+        /// re-added for that. This is deliberate -- see the no-migration decision -- and it is
+        /// asserted here because nothing about it is obvious from the outside.
+        /// </summary>
+        [Test]
+        public void DeltaSourceNotAdoptedByExistingFullIndexSource()
+        {
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false);
+
+            AddDeltaSource();
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "No delta should be involved while the feature is off");
+            Assert.That(
+                GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName),
+                Is.EqualTo(Constants.DeltaTestSourceVersion),
+                "The full index should be deployed at its own version");
+
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true);
+
+            var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}");
+            Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+            Assert.That(updateResult.StdOut, Does.Contain("Done"));
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "The update should not have acquired a delta");
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.False, "The update should not have acquired a baseline");
+            Assert.That(
+                GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName),
+                Is.EqualTo(Constants.DeltaTestSourceVersion),
+                "The source should still be served by the full index");
+
+            // The full index holds everything, so what the source presents is unchanged.
+            RequireFound(Constants.DeltaOnlyPackageIdentifier);
+            RequireFound(BaselinePackageIdentifier);
+        }
+
+        /// <summary>
+        /// Disabling delta after adding a source acquires the full index and keeps it usable.
+        /// </summary>
+        /// <param name="updateSource">Whether to explicitly update before searching.</param>
+        [TestCase(false)]
+        [TestCase(true)]
+        public void DeltaSourceFeatureDisabled(bool updateSource)
+        {
+            AddDeltaSource();
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True);
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.True);
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.False);
+            RequireFound(Constants.DeltaOnlyPackageIdentifier);
+            RequireFound(BaselinePackageIdentifier);
+
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false);
+
+            if (updateSource)
+            {
+                var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}");
+                Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+                Assert.That(updateResult.StdOut, Does.Contain("Done"));
+            }
+
+            RequireFound(Constants.DeltaOnlyPackageIdentifier);
+            RequireFound(BaselinePackageIdentifier);
+            Assert.That(
+                GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName),
+                Is.EqualTo(Constants.DeltaTestSourceVersion),
+                "The source should be served by the full index with delta disabled");
+
+            var removeResult = TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}");
+            Assert.That(removeResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.False);
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False);
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.False);
+        }
+
+        /// <summary>
+        /// Removing the source takes both deployed packages with it. A delta source leaves more
+        /// behind than an ordinary one, so there is more to get wrong.
+        /// </summary>
+        [Test]
+        public void DeltaSourceRemove()
+        {
+            AddDeltaSource();
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True);
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.True);
+
+            var removeResult = TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}");
+            Assert.That(removeResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+            Assert.That(removeResult.StdOut, Does.Contain("Done"));
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "The delta package should have been removed");
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.False, "The baseline package should have been removed");
+
+            var listResult = TestCommon.RunAICLICommand("source list", $"-n {Constants.DeltaTestSourceName}");
+            Assert.That(listResult.ExitCode, Is.EqualTo(Constants.ErrorCode.ERROR_SOURCE_NAME_DOES_NOT_EXIST));
+        }
+
+        /// <summary>
+        /// Adds the delta source, requiring that it succeed.
+        /// </summary>
+        private static void AddDeltaSource()
+        {
+            var result = TestCommon.RunAICLICommand("source add", $"{Constants.DeltaTestSourceName} {Constants.DeltaTestSourceUrl} --trust-level trusted");
+            Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+            Assert.That(result.StdOut, Does.Contain("Done"));
+        }
+
+        /// <summary>
+        /// Removes the delta source, tolerating its not having been added.
+        /// </summary>
+        private static void RemoveDeltaSource()
+        {
+            TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}");
+        }
+
+        /// <summary>
+        /// Requires that the delta source presents the given package.
+        /// </summary>
+        /// <param name="packageIdentifier">The package identifier.</param>
+        private static void RequireFound(string packageIdentifier)
+        {
+            var result = TestCommon.RunAICLICommand("search", $"{packageIdentifier} --source {Constants.DeltaTestSourceName}");
+            Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK), $"{packageIdentifier} should be found");
+            Assert.That(result.StdOut, Does.Contain(packageIdentifier));
+        }
+
+        /// <summary>
+        /// Whether a package with the given identity name is deployed for the current user.
+        /// </summary>
+        /// <param name="identityName">The package identity name.</param>
+        /// <returns>True if it is deployed.</returns>
+        private static bool IsPackageDeployed(string identityName)
+        {
+            return !string.IsNullOrEmpty(GetDeployedPackageVersion(identityName));
+        }
+
+        /// <summary>
+        /// Gets the version of a deployed package, or an empty string when it is not deployed.
+        /// </summary>
+        /// <param name="identityName">The package identity name.</param>
+        /// <returns>The version, or an empty string.</returns>
+        private static string GetDeployedPackageVersion(string identityName)
+        {
+            // The name has to match exactly, which matters here because the delta's identity name
+            // has the full index's as a prefix.
+            var result = TestCommon.RunCommandWithResult(
+                "powershell",
+                $"-Command \"(Get-AppxPackage | Where-Object {{ $_.Name -ceq '{identityName}' }} | Select-Object -First 1).Version\"");
+
+            return result.StdOut.Trim();
+        }
+    }
+}

src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs

@@ -192,6 +192,20 @@ public static void GenerateE2ESource()
                 {
                     CertFile = testParams.PackageCertificatePath,
                 },
+                Delta = new ()
+                {
+                    RelativeDirectory = Constants.DeltaTestSourceDirectory,
+                    IdentityName = Constants.DeltaTestSourceIdentityName,
+                    DeltaIdentityName = Constants.DeltaTestSourceDeltaIdentityName,
+                    BaselineIdentityName = Constants.DeltaTestSourceBaselineIdentityName,
+                    Version = Constants.DeltaTestSourceVersion,
+                    BaselineVersion = Constants.DeltaTestSourceBaselineVersion,
+                    BaselineRelativePath = Constants.DeltaTestSourceBaselineRelativePath,
+                    ManifestsNotInBaseline = new ()
+                    {
+                        Constants.DeltaOnlyManifestFileName,
+                    },
+                },
             };
 
             WinGetLocalSource.CreateLocalSource(e2eSource);

src/AppInstallerCLIE2ETests/TestData/localsource.json

@@ -53,5 +53,17 @@
   "Signature": {
     "CertFile": "%TestSigningCert_PfxPath%",
     "Password": "%TestSigningCert_Password%"
+  },
+  "Delta": {
+    "RelativeDirectory": "deltaSource",
+    "IdentityName": "WingetE2EDelta.Tests",
+    "DeltaIdentityName": "WingetE2EDelta.Tests.Delta",
+    "BaselineIdentityName": "WingetE2EDelta.Tests.Baseline",
+    "Version": "2020.805.713.335",
+    "BaselineVersion": "2020.805.713.334",
+    "BaselineRelativePath": "baseline/source2.msix",
+    "ManifestsNotInBaseline": [
+      "TestExampleInstaller.yaml"
+    ]
   }
 }

src/AppInstallerCLITests/AppInstallerCLITests.vcxproj

@@ -218,6 +218,7 @@
     <ClInclude Include="TestConfiguration.h" />
     <ClInclude Include="TestRestRequestHandler.h" />
     <ClInclude Include="TestHooks.h" />
+    <ClInclude Include="TestMsixPackage.h" />
     <ClInclude Include="TestSettings.h" />
     <ClInclude Include="TestSource.h" />
     <ClInclude Include="WorkflowCommon.h" />
@@ -338,6 +339,7 @@
     <ClCompile Include="TableOutput.cpp" />
     <ClCompile Include="TestCertificates.cpp" />
     <ClCompile Include="TestCommon.cpp" />
+    <ClCompile Include="TestMsixPackage.cpp" />
     <ClCompile Include="WorkflowCommon.cpp" />
     <ClCompile Include="WorkflowGroupPolicy.cpp" />
     <ClCompile Include="Yaml.cpp" />