diff --git a/Configuration.props b/Configuration.props index 38f3dcf8526..bf10d52af15 100644 --- a/Configuration.props +++ b/Configuration.props @@ -93,7 +93,6 @@ $(MSBuildThisFileDirectory)external\sqlite $(MSBuildThisFileDirectory)external\libunwind $(BootstrapOutputDirectory)\libunwind - $(MSBuildThisFileDirectory)external\lz4 $(MSBuildThisFileDirectory) $(MSBuildThisFileDirectory)src-ThirdParty\ armeabi-v7a;x86 @@ -165,7 +164,6 @@ $([System.IO.Path]::GetFullPath ('$(SqliteSourceDirectory)')) $([System.IO.Path]::GetFullPath ('$(LibUnwindSourceDirectory)')) $([System.IO.Path]::GetFullPath ('$(LibUnwindGeneratedHeadersDirectory)')) - $([System.IO.Path]::GetFullPath ('$(LZ4SourceDirectory)')) net10.0 diff --git a/Xamarin.Android.Build.Tasks.sln b/Xamarin.Android.Build.Tasks.sln index a3ec688acea..4bef4d8fbc9 100644 --- a/Xamarin.Android.Build.Tasks.sln +++ b/Xamarin.Android.Build.Tasks.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 16.0.29920.165 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Build.Tasks", "src\Xamarin.Android.Build.Tasks\Xamarin.Android.Build.Tasks.csproj", "{3F1F2F50-AF1A-4A5A-BEDB-193372F068D7}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Android.Build.Tasks", "src\Microsoft.Android.Build.Tasks\Microsoft.Android.Build.Tasks.csproj", "{5DB5BB32-F094-46C8-9C81-4FFEC0BAF5D2}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Build.Tests", "src\Xamarin.Android.Build.Tasks\Tests\Xamarin.Android.Build.Tests\Xamarin.Android.Build.Tests.csproj", "{53E4ABF0-1085-45F9-B964-DCAE4B819998}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.ProjectTools", "src\Xamarin.Android.Build.Tasks\Tests\Xamarin.ProjectTools\Xamarin.ProjectTools.csproj", "{2DD1EE75-6D8D-4653-A800-0A24367F7F38}" @@ -44,6 +46,10 @@ Global {3F1F2F50-AF1A-4A5A-BEDB-193372F068D7}.Debug|Any CPU.Build.0 = Debug|Any CPU {3F1F2F50-AF1A-4A5A-BEDB-193372F068D7}.Release|Any CPU.ActiveCfg = Release|Any CPU {3F1F2F50-AF1A-4A5A-BEDB-193372F068D7}.Release|Any CPU.Build.0 = Release|Any CPU + {5DB5BB32-F094-46C8-9C81-4FFEC0BAF5D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5DB5BB32-F094-46C8-9C81-4FFEC0BAF5D2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5DB5BB32-F094-46C8-9C81-4FFEC0BAF5D2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5DB5BB32-F094-46C8-9C81-4FFEC0BAF5D2}.Release|Any CPU.Build.0 = Release|Any CPU {53E4ABF0-1085-45F9-B964-DCAE4B819998}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {53E4ABF0-1085-45F9-B964-DCAE4B819998}.Debug|Any CPU.Build.0 = Debug|Any CPU {53E4ABF0-1085-45F9-B964-DCAE4B819998}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index 03f06938fb5..f60bdbec6e7 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -159,6 +159,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.Build.Debug EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "fastdevtools", "tools\fastdev\fastdevtools.csproj", "{4723786E-4BB5-4939-BBFD-3C2D0EAE236B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Android.Build.Tasks", "src\Microsoft.Android.Build.Tasks\Microsoft.Android.Build.Tasks.csproj", "{4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "xamarin-android-tools", "xamarin-android-tools", "{BB3C8E1B-31A2-1C4E-0B3F-36B3F475D6AA}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{65925F91-7BB8-AC56-CADB-FB3DD28A5B5B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1021,6 +1027,18 @@ Global {4723786E-4BB5-4939-BBFD-3C2D0EAE236B}.Release|x64.Build.0 = Release|Any CPU {4723786E-4BB5-4939-BBFD-3C2D0EAE236B}.Release|x86.ActiveCfg = Release|Any CPU {4723786E-4BB5-4939-BBFD-3C2D0EAE236B}.Release|x86.Build.0 = Release|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Debug|x64.ActiveCfg = Debug|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Debug|x64.Build.0 = Debug|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Debug|x86.ActiveCfg = Debug|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Debug|x86.Build.0 = Debug|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Release|Any CPU.Build.0 = Release|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Release|x64.ActiveCfg = Release|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Release|x64.Build.0 = Release|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Release|x86.ActiveCfg = Release|Any CPU + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1095,6 +1113,9 @@ Global {79BBD075-BC73-4D35-8DE6-BE70C9AFC708} = {FFCF518F-2A4A-40A2-9174-2EE13B76C723} {624C58EB-57CF-4754-95EF-E9893E584296} = {FFCF518F-2A4A-40A2-9174-2EE13B76C723} {4723786E-4BB5-4939-BBFD-3C2D0EAE236B} = {FFCF518F-2A4A-40A2-9174-2EE13B76C723} + {4E5C5846-CBFD-4695-B1F0-D48DEC0AF50B} = {FFCF518F-2A4A-40A2-9174-2EE13B76C723} + {BB3C8E1B-31A2-1C4E-0B3F-36B3F475D6AA} = {05C3B1D6-A4CE-4534-A9E4-E9117591ADF7} + {65925F91-7BB8-AC56-CADB-FB3DD28A5B5B} = {BB3C8E1B-31A2-1C4E-0B3F-36B3F475D6AA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {53A1F287-EFB2-4D97-A4BB-4A5E145613F6} diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index 9c5e22940f6..b655fa49620 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -150,6 +150,8 @@ <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Microsoft.Android.Sdk.Bindings.Gradle.targets" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Xamarin.Android.Build.Tasks.dll" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Xamarin.Android.Build.Tasks.pdb" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Microsoft.Android.Build.Tasks.dll" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Microsoft.Android.Build.Tasks.pdb" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Microsoft.Android.Sdk.TrimmableTypeMap.dll" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Microsoft.Android.Sdk.TrimmableTypeMap.pdb" /> <_MSBuildFiles Include="@(_LocalizationLanguages->'$(MicrosoftAndroidSdkOutDir)%(Identity)\Microsoft.Android.Build.BaseTasks.resources.dll')" /> diff --git a/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj b/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj new file mode 100644 index 00000000000..5cf7aa3a070 --- /dev/null +++ b/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj @@ -0,0 +1,39 @@ + + + + + + $(DotNetTargetFramework) + Microsoft.Android.Tasks + enable + Nullable + $(MicrosoftAndroidSdkOutDir) + false + true + ..\..\product.snk + + + + + + + + + + + + Xamarin.Android.Tasks.Properties.Resources + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs b/src/Microsoft.Android.Build.Tasks/Tasks/CompressAssemblies.cs similarity index 66% rename from src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs rename to src/Microsoft.Android.Build.Tasks/Tasks/CompressAssemblies.cs index 039cfc0626c..9633ca36013 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs +++ b/src/Microsoft.Android.Build.Tasks/Tasks/CompressAssemblies.cs @@ -1,15 +1,19 @@ #nullable enable -using System; using System.Collections.Generic; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; +using Properties = Xamarin.Android.Tasks.Properties; -namespace Xamarin.Android.Tasks; +namespace Microsoft.Android.Tasks; /// -/// Compresses assemblies using LZ4 compression before placing them in the APK. +/// Compresses assemblies using Zstandard compression before placing them in the APK. /// Note this is independent of whether they are stored compressed with ZIP in the APK. -/// Our runtime bits will LZ4 decompress them at assembly load time. +/// Our runtime bits will Zstd decompress them at assembly load time. +/// +/// This task lives in Microsoft.Android.Build.Tasks.dll (net11.0) because it uses +/// System.IO.Compression.ZstandardEncoder, which is not available in netstandard2.0. +/// It is imported with <UsingTask ... Runtime="NET" TaskFactory="TaskHostFactory" />. /// public class CompressAssemblies : AndroidTask { @@ -26,7 +30,7 @@ public override bool RunTask () var failed_assemblies = new List (); foreach (var assembly in AssembliesToCompress) { - MonoAndroidHelper.LogIfReferenceAssembly (assembly, Log); + ReferenceAssemblyChecker.LogIfReferenceAssembly (assembly, Log); if (!assembly.TryGetRequiredMetadata ("AssembliesToCompress", "DestinationPath", Log, out var destination_path)) break; @@ -38,8 +42,8 @@ public override bool RunTask () Log.LogCodedError ("XA5303", Properties.Resources.XA5303, descriptor_index_string, assembly.ItemSpec); break; } - - if (!AssemblyCompression.TryCompress (Log, assembly.ItemSpec, destination_path, descriptor_index)) { + + if (!AssemblyCompressor.TryCompress (Log, assembly.ItemSpec, destination_path, descriptor_index)) { failed_assemblies.Add (assembly); continue; } diff --git a/src/Microsoft.Android.Build.Tasks/Utilities/AssemblyCompressor.cs b/src/Microsoft.Android.Build.Tasks/Utilities/AssemblyCompressor.cs new file mode 100644 index 00000000000..1cea691324c --- /dev/null +++ b/src/Microsoft.Android.Build.Tasks/Utilities/AssemblyCompressor.cs @@ -0,0 +1,99 @@ +#nullable enable +using System; +using System.Buffers; +using System.IO; +using System.IO.Compression; + +using Microsoft.Build.Utilities; + +namespace Microsoft.Android.Tasks; + +/// +/// Compresses assemblies with Zstandard before they are placed in the AssemblyStore. +/// The native runtime decompresses them at assembly load time. The 12-byte header +/// (magic / descriptor index / uncompressed length) is read back by the runtime and by +/// the diagnostic tools; the reader-side helpers live in AssemblyCompression in +/// Xamarin.Android.Build.Tasks. +/// +static class AssemblyCompressor +{ + const uint CompressedDataMagic = 0x535A4158; // 'XAZS', little-endian + + static readonly ArrayPool bytePool = ArrayPool.Shared; + + enum CompressionResult + { + Success, + EncodingFailed, + } + + public static bool TryCompress (TaskLoggingHelper log, string sourceAssembly, string destinationAssembly, uint descriptorIndex) + { + CompressionResult result = Compress (sourceAssembly, destinationAssembly, descriptorIndex); + + if (result != CompressionResult.Success) { + log.LogMessage ($"Failed to compress {sourceAssembly}"); + return false; + } + + return true; + } + + static CompressionResult Compress (string sourcePath, string outputFilePath, uint descriptorIndex) + { + var outputDirectory = Path.GetDirectoryName (outputFilePath); + if (string.IsNullOrEmpty (outputDirectory)) + throw new ArgumentException ("must not be null or empty", nameof (outputFilePath)); + + Directory.CreateDirectory (outputDirectory); + + var fi = new FileInfo (sourcePath); + if (!fi.Exists) + throw new InvalidOperationException ($"File '{sourcePath}' does not exist"); + + int fileSize = checked ((int) fi.Length); + + byte[]? sourceBytes = null; + byte[]? destBytes = null; + try { + sourceBytes = bytePool.Rent (fileSize); + int bytesRead = 0; + using (var fs = File.Open (sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { + while (bytesRead < fileSize) { + int read = fs.Read (sourceBytes, bytesRead, fileSize - bytesRead); + if (read == 0) + break; + + bytesRead += read; + } + } + + if (bytesRead != fileSize) + return CompressionResult.EncodingFailed; + + long maxOutputSize = ZstandardEncoder.GetMaxCompressedLength (bytesRead); + if (maxOutputSize <= 0 || maxOutputSize > int.MaxValue) + return CompressionResult.EncodingFailed; + + destBytes = bytePool.Rent ((int) maxOutputSize); + if (!ZstandardEncoder.TryCompress (sourceBytes.AsSpan (0, bytesRead), destBytes, out int encodedLength)) + return CompressionResult.EncodingFailed; + + using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) + using (var bw = new BinaryWriter (fs)) { + bw.Write (CompressedDataMagic); // magic + bw.Write (descriptorIndex); // index into runtime array of descriptors + bw.Write (checked ((uint) fi.Length)); // file size before compression + bw.Write (destBytes, 0, encodedLength); + bw.Flush (); + } + } finally { + if (sourceBytes != null) + bytePool.Return (sourceBytes); + if (destBytes != null) + bytePool.Return (destBytes); + } + + return CompressionResult.Success; + } +} diff --git a/src/Microsoft.Android.Build.Tasks/Utilities/ReferenceAssemblyChecker.cs b/src/Microsoft.Android.Build.Tasks/Utilities/ReferenceAssemblyChecker.cs new file mode 100644 index 00000000000..2f25455b624 --- /dev/null +++ b/src/Microsoft.Android.Build.Tasks/Utilities/ReferenceAssemblyChecker.cs @@ -0,0 +1,70 @@ +#nullable enable +using System.IO; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Properties = Xamarin.Android.Tasks.Properties; + +namespace Microsoft.Android.Tasks; + +/// +/// Minimal reference-assembly detection for . This mirrors +/// MonoAndroidHelper.LogIfReferenceAssembly in Xamarin.Android.Build.Tasks, but is +/// duplicated here to keep this net11.0 assembly self-contained. +/// +static class ReferenceAssemblyChecker +{ + public static bool LogIfReferenceAssembly (ITaskItem assembly, TaskLoggingHelper log) + { + if (IsReferenceAssembly (assembly.ItemSpec, log)) { + log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); + return true; + } + + return false; + } + + static bool IsReferenceAssembly (string assembly, TaskLoggingHelper log) + { + using var stream = File.OpenRead (assembly); + using var pe = new PEReader (stream); + if (!pe.HasMetadata) { + log.LogDebugMessage ($"Skipping non-.NET assembly: {assembly}"); + return false; + } + + var reader = pe.GetMetadataReader (); + var assemblyDefinition = reader.GetAssemblyDefinition (); + foreach (var handle in assemblyDefinition.GetCustomAttributes ()) { + var attribute = reader.GetCustomAttribute (handle); + var attributeName = GetCustomAttributeFullName (reader, attribute); + if (attributeName == "System.Runtime.CompilerServices.ReferenceAssemblyAttribute") + return true; + } + + return false; + } + + static string? GetCustomAttributeFullName (MetadataReader reader, CustomAttribute attribute) + { + switch (attribute.Constructor.Kind) { + case HandleKind.MemberReference: { + var ctor = reader.GetMemberReference ((MemberReferenceHandle) attribute.Constructor); + if (ctor.Parent.Kind != HandleKind.TypeReference) + return null; + var type = reader.GetTypeReference ((TypeReferenceHandle) ctor.Parent); + return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name); + } + case HandleKind.MethodDefinition: { + var ctor = reader.GetMethodDefinition ((MethodDefinitionHandle) attribute.Constructor); + var type = reader.GetTypeDefinition (ctor.GetDeclaringType ()); + return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name); + } + default: + return null; + } + } +} diff --git a/src/Microsoft.Android.Build.Tasks/Utilities/TaskItemExtensions.cs b/src/Microsoft.Android.Build.Tasks/Utilities/TaskItemExtensions.cs new file mode 100644 index 00000000000..c214d6fe0ca --- /dev/null +++ b/src/Microsoft.Android.Build.Tasks/Utilities/TaskItemExtensions.cs @@ -0,0 +1,27 @@ +#nullable enable +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Properties = Xamarin.Android.Tasks.Properties; + +namespace Microsoft.Android.Tasks; + +/// +/// Minimal helpers for , duplicated +/// here to keep this net11.0 assembly self-contained (the full versions live in +/// ITaskItemExtensions in Xamarin.Android.Build.Tasks). +/// +static class TaskItemExtensions +{ + public static bool TryGetRequiredMetadata (this ITaskItem item, string itemName, string name, TaskLoggingHelper log, out string value) + { + value = item.GetMetadata (name); + + if (string.IsNullOrWhiteSpace (value)) { + log.LogCodedError ("XA4234", Properties.Resources.XA4234, itemName, item.ItemSpec, name); + return false; + } + + return true; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc index a00fd898add..6ad0abc158a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc @@ -8,16 +8,16 @@ "Size": 402352 }, "lib/arm64-v8a/libassembly-store.so": { - "Size": 3229120 + "Size": 2717688 }, "lib/arm64-v8a/libclrjit.so": { - "Size": 2835128 + "Size": 2836936 }, "lib/arm64-v8a/libcoreclr.so": { - "Size": 4891056 + "Size": 4890752 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 1324320 + "Size": 1264112 }, "lib/arm64-v8a/libSystem.Globalization.Native.so": { "Size": 72112 @@ -32,7 +32,7 @@ "Size": 168080 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 20984 + "Size": 20968 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -59,5 +59,5 @@ "Size": 1904 } }, - "PackageSize": 7312827 + "PackageSize": 7271867 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc index f156471b0da..042ee6a45db 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc @@ -32,16 +32,16 @@ "Size": 2396 }, "lib/arm64-v8a/libassembly-store.so": { - "Size": 14137920 + "Size": 11658648 }, "lib/arm64-v8a/libclrjit.so": { - "Size": 2804464 + "Size": 2824392 }, "lib/arm64-v8a/libcoreclr.so": { - "Size": 4872088 + "Size": 4890432 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 1325808 + "Size": 1265504 }, "lib/arm64-v8a/libSystem.Globalization.Native.so": { "Size": 72112 @@ -56,7 +56,7 @@ "Size": 168080 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 147616 + "Size": 147824 }, "META-INF/androidx.activity_activity.version": { "Size": 6 @@ -2234,5 +2234,5 @@ "Size": 794696 } }, - "PackageSize": 20778573 + "PackageSize": 20786765 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index ae852f42feb..5a8467a1ce1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -1,133 +1,23 @@ #nullable enable using System; using System.Collections.Generic; -using System.Buffers; using System.IO; -using K4os.Compression.LZ4; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -using Xamarin.Android.Tasks; using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { + /// + /// Reader-side helpers for Zstandard-compressed AssemblyStore assemblies. The actual + /// compression is performed by the CompressAssemblies task in + /// Microsoft.Android.Build.Tasks.dll (net11.0), which uses + /// System.IO.Compression.ZstandardEncoder. + /// class AssemblyCompression { - public enum CompressionResult - { - Success, - InputTooBig, - EncodingFailed, - } - - public sealed class AssemblyData - { - public string? SourcePath { get; internal set; } - public uint DescriptorIndex { get; internal set; } - - public string? DestinationPath; - public uint SourceSize; - public uint DestinationSize; - - public AssemblyData (string sourcePath, uint descriptorIndex) - { - SetData (sourcePath, descriptorIndex); - } - - public void SetData (string sourcePath, uint descriptorIndex) - { - if (sourcePath.IsNullOrEmpty ()) - throw new ArgumentException ("must not be null or empty", nameof (sourcePath)); - SourcePath = sourcePath; - DescriptorIndex = descriptorIndex; - } - } - - const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian - - static readonly ArrayPool bytePool = ArrayPool.Shared; - - static CompressionResult Compress (AssemblyData data, string outputFilePath) - { - if (data == null) - throw new ArgumentNullException (nameof (data)); - - var outputDirectory = Path.GetDirectoryName (outputFilePath); - - if (outputDirectory.IsNullOrEmpty ()) - throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); - - Directory.CreateDirectory (outputDirectory); - - var fi = new FileInfo (data.SourcePath); - if (!fi.Exists) - throw new InvalidOperationException ($"File '{data.SourcePath}' does not exist"); - - data.DestinationPath = outputFilePath; - data.SourceSize = checked((uint)fi.Length); - - int bytesRead; - byte[]? sourceBytes = null; - byte[]? destBytes = null; - try { - int fileSize = checked((int)fi.Length); - sourceBytes = bytePool.Rent (fileSize); - using (var fs = File.Open (data.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - bytesRead = fs.Read (sourceBytes, 0, fileSize); - } - - destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (bytesRead)); - int encodedLength = LZ4Codec.Encode (sourceBytes, 0, bytesRead, destBytes, 0, destBytes.Length, LZ4Level.L12_MAX); - if (encodedLength < 0) - return CompressionResult.EncodingFailed; - - data.DestinationSize = (uint)encodedLength; - using (var fs = File.Open (data.DestinationPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - using (var bw = new BinaryWriter (fs)) { - bw.Write (CompressedDataMagic); // magic - bw.Write (data.DescriptorIndex); // index into runtime array of descriptors - bw.Write (checked((uint)fi.Length)); // file size before compression - - bw.Write (destBytes, 0, encodedLength); - bw.Flush (); - } - } - } finally { - if (sourceBytes != null) - bytePool.Return (sourceBytes); - if (destBytes != null) - bytePool.Return (destBytes); - } - - return CompressionResult.Success; - } - - public static bool TryCompress (TaskLoggingHelper log, string sourceAssembly, string destinationAssembly, uint descriptorIndex) - { - AssemblyData compressedAssembly = new AssemblyData (sourceAssembly, descriptorIndex); - CompressionResult result = Compress (compressedAssembly, destinationAssembly); - - if (result != CompressionResult.Success) { - switch (result) { - case CompressionResult.EncodingFailed: - log.LogMessage ($"Failed to compress {sourceAssembly}"); - break; - case CompressionResult.InputTooBig: - log.LogMessage ($"Input assembly {sourceAssembly} exceeds maximum input size"); - break; - default: - log.LogMessage ($"Unknown error compressing {sourceAssembly}"); - break; - } - - return false; - } - - return true; - } - // Gets the descriptor index for the specified assembly from the compressed assembly info public static bool TryGetDescriptorIndex (TaskLoggingHelper log, ITaskItem assembly, IDictionary> compressedAssembliesInfo, out uint descriptorIndex) { @@ -154,7 +44,7 @@ public static bool TryGetDescriptorIndex (TaskLoggingHelper log, ITaskItem assem public static string GetCompressedAssemblyOutputPath (ITaskItem assembly, string compressedOutputDir) { var assemblyOutputDir = GetCompressedAssemblyOutputDirectory (assembly, compressedOutputDir); - return Path.Combine (assemblyOutputDir, $"{Path.GetFileName (assembly.ItemSpec)}.lz4"); + return Path.Combine (assemblyOutputDir, $"{Path.GetFileName (assembly.ItemSpec)}.zst"); } static string GetCompressedAssemblyOutputDirectory (ITaskItem assembly, string compressedOutputDir) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index 590ca553c7d..a8c38d99a29 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -34,7 +34,7 @@ public AssemblyStoreAssemblyInfo (string sourceFilePath, ITaskItem assembly, boo throw new InvalidOperationException ("Internal error: info without assembly name"); } - if (name.EndsWith (".lz4", StringComparison.OrdinalIgnoreCase)) { + if (name.EndsWith (".zst", StringComparison.OrdinalIgnoreCase)) { name = Path.GetFileNameWithoutExtension (name); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index 4773d0cd1c1..141bab21947 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -125,7 +125,6 @@ public NativeRuntimeComponents (ITaskItem[]? monoComponents) new AndroidArchive ("libruntime-base-common-release.a"), new AndroidArchive ("libruntime-base-release.a"), new AndroidArchive ("libxa-java-interop-release.a"), - new AndroidArchive ("libxa-lz4-release.a"), new AndroidArchive ("libxa-shared-bits-release.a"), new AndroidArchive ("libxamarin-startup-release.a"), diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index ea3cf49c141..03084bd6c37 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -44,7 +44,8 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. - + + @@ -2150,7 +2151,7 @@ because xbuild doesn't support framework reference assemblies. This dependency chain should be addressed in a separate PR. --> -#if defined (HAVE_LZ4) -#include -#endif - #include #include #include #include #include #include +#include using namespace xamarin::android; @@ -26,7 +23,7 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co uint8_t *assembly_data = nullptr; uint32_t assembly_data_size = 0; -#if defined (HAVE_LZ4) && defined (RELEASE) +#if defined (RELEASE) auto header = reinterpret_cast(e.image_data); if (header->magic == COMPRESSED_DATA_MAGIC) { log_debug (LOG_ASSEMBLY, "Decompressing assembly '{}' from the assembly store"sv, name); @@ -114,20 +111,20 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co } const char *data_start = pointer_add(e.image_data, sizeof(CompressedAssemblyHeader)); - int ret = LZ4_decompress_safe (data_start, reinterpret_cast(data_buffer), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); + size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ret < 0) { + if (ZSTD_isError (ret)) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( - "Decompression of assembly {} failed with code {}"sv, + "Decompression of assembly {} failed: {}"sv, name, - ret + ZSTD_getErrorName (ret) ) ); } - if (static_cast(ret) != cad.uncompressed_file_size) { + if (ret != cad.uncompressed_file_size) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( @@ -147,7 +144,7 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); } else -#endif // def HAVE_LZ4 && def RELEASE +#endif // def RELEASE { log_debug (LOG_ASSEMBLY, "Assembly '{}' is not compressed in the assembly store"sv, name); diff --git a/src/native/clr/include/xamarin-app.hh b/src/native/clr/include/xamarin-app.hh index c2140e2c882..39fbb97f822 100644 --- a/src/native/clr/include/xamarin-app.hh +++ b/src/native/clr/include/xamarin-app.hh @@ -9,7 +9,7 @@ #include static constexpr uint64_t FORMAT_TAG = 0x00045E6972616D58; // 'Xmari^XY' where XY is the format version -static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian +static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x535A4158; // 'XAZS', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian // The highest bit of assembly store version is a 64-bit ABI flag diff --git a/src/native/common/include/runtime-base/timing-internal.hh b/src/native/common/include/runtime-base/timing-internal.hh index 31099b86322..5bd95f11af4 100644 --- a/src/native/common/include/runtime-base/timing-internal.hh +++ b/src/native/common/include/runtime-base/timing-internal.hh @@ -430,7 +430,7 @@ namespace xamarin::android { switch (kind) { case TimingEventKind::AssemblyDecompression: - append_desc ("LZ4 decompression time for "sv); + append_desc ("Zstd decompression time for "sv); return; case TimingEventKind::AssemblyLoad: diff --git a/src/native/common/include/runtime-base/zstd.hh b/src/native/common/include/runtime-base/zstd.hh new file mode 100644 index 00000000000..bbab4c91471 --- /dev/null +++ b/src/native/common/include/runtime-base/zstd.hh @@ -0,0 +1,18 @@ +// Dear Emacs, this is a -*- C++ -*- header +#pragma once + +#include + +// +// Minimal declarations for the Zstandard decompression functions that are exported by +// `libSystem.IO.Compression.Native`, which ships in the .NET runtime pack and is linked +// into our runtime. The assembly store compresses assemblies with Zstd at build time and +// we decompress them here at load time. +// +// We declare only the few entry points we need instead of pulling in `zstd.h`. +// +extern "C" { + size_t ZSTD_decompress (void *dst, size_t dst_capacity, const void *src, size_t compressed_size) noexcept; + unsigned ZSTD_isError (size_t code) noexcept; + const char* ZSTD_getErrorName (size_t code) noexcept; +} diff --git a/src/native/common/lz4/CMakeLists.txt b/src/native/common/lz4/CMakeLists.txt deleted file mode 100644 index 57c8a717d01..00000000000 --- a/src/native/common/lz4/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -set(LIB_NAME xa-lz4) -set(LIB_ALIAS xa::lz4) - -set(LZ4_SRC_DIR "${EXTERNAL_DIR}/lz4/lib") -set(LZ4_INCLUDE_DIR ${LZ4_SRC_DIR}) - -set(LZ4_SOURCES - ${LZ4_SRC_DIR}/lz4.c -) - -add_library( - ${LIB_NAME} - STATIC - ${LZ4_SOURCES} -) -set_static_library_suffix(${LIB_NAME}) - -add_library(${LIB_ALIAS} ALIAS ${LIB_NAME}) - -target_compile_definitions( - ${LIB_NAME} - PRIVATE - # Ugly, but this is the only way to change LZ4 symbols visibility without modifying lz4.h - "LZ4LIB_VISIBILITY=__attribute__ ((visibility (\"hidden\")))" - XXH_NAMESPACE=LZ4_ -) - -target_include_directories( - ${LIB_NAME} - PUBLIC - "$" -) - -if(DEBUG_BUILD) - set_target_properties( - ${LIB_NAME} - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - ) -endif() - -xa_add_compile_definitions(${LIB_NAME}) diff --git a/src/native/mono/monodroid/CMakeLists.txt b/src/native/mono/monodroid/CMakeLists.txt index 888b87d3d98..43da300d837 100644 --- a/src/native/mono/monodroid/CMakeLists.txt +++ b/src/native/mono/monodroid/CMakeLists.txt @@ -152,7 +152,6 @@ macro(lib_target_options TARGET_NAME) ${TARGET_NAME} PRIVATE HAVE_CONFIG_H - HAVE_LZ4 JI_DLL_EXPORT JI_NO_VISIBILITY MONO_DLL_EXPORT @@ -231,7 +230,7 @@ macro(lib_target_options TARGET_NAME) xa::runtime-base xa::java-interop xa::pinvoke-override-precompiled - xa::lz4 + -lSystem.IO.Compression.Native -lmonosgen-2.0 -landroid -llog diff --git a/src/native/mono/monodroid/embedded-assemblies.cc b/src/native/mono/monodroid/embedded-assemblies.cc index c3aa3488aa8..08099f65324 100644 --- a/src/native/mono/monodroid/embedded-assemblies.cc +++ b/src/native/mono/monodroid/embedded-assemblies.cc @@ -15,10 +15,6 @@ #include #include -#if defined (HAVE_LZ4) -#include -#endif - #include #include #include @@ -37,6 +33,7 @@ #include "startup-aware-lock.hh" #include #include +#include using namespace xamarin::android; using namespace xamarin::android::internal; @@ -82,7 +79,7 @@ EmbeddedAssemblies::set_assembly_data_and_size (uint8_t* source_assembly_data, u [[gnu::always_inline]] void EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[maybe_unused]] const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { -#if defined (HAVE_LZ4) && defined (RELEASE) +#if defined (RELEASE) auto header = reinterpret_cast(data); if (header->magic == COMPRESSED_DATA_MAGIC) { if (compressed_assembly_count == 0) [[unlikely]] { @@ -156,20 +153,20 @@ EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[mayb } const char *data_start = pointer_add(data, sizeof(CompressedAssemblyHeader)); - int ret = LZ4_decompress_safe (data_start, reinterpret_cast(data_buffer), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); + size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ret < 0) { + if (ZSTD_isError (ret)) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( - "Decompression of assembly {} failed with code {}", + "Decompression of assembly {} failed: {}", optional_string (name), - ret + ZSTD_getErrorName (ret) ) ); } - if (static_cast(ret) != cad.uncompressed_file_size) { + if (ret != cad.uncompressed_file_size) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( @@ -185,7 +182,7 @@ EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[mayb set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); } else -#endif // def HAVE_LZ4 && def RELEASE +#endif // def RELEASE { set_assembly_data_and_size (data, data_size, assembly_data, assembly_data_size); } diff --git a/src/native/mono/xamarin-app-stub/xamarin-app.hh b/src/native/mono/xamarin-app-stub/xamarin-app.hh index c0e3d3012d6..2a21c1e827b 100644 --- a/src/native/mono/xamarin-app-stub/xamarin-app.hh +++ b/src/native/mono/xamarin-app-stub/xamarin-app.hh @@ -12,7 +12,7 @@ #include static constexpr uint64_t FORMAT_TAG = 0x00035E6972616D58; // 'Xmari^XY' where XY is the format version -static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian +static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x535A4158; // 'XAZS', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian // The highest bit of assembly store version is a 64-bit ABI flag diff --git a/src/native/native.targets b/src/native/native.targets index 01a2522e38c..80c87e42131 100644 --- a/src/native/native.targets +++ b/src/native/native.targets @@ -36,7 +36,6 @@ <_ConfigureRuntimesInputs Include="CMakeLists.txt" /> <_ConfigureRuntimesInputs Include="common\java-interop\CMakeLists.txt" /> <_ConfigureRuntimesInputs Include="common\libunwind\CMakeLists.txt" /> - <_ConfigureRuntimesInputs Include="common\lz4\CMakeLists.txt" /> <_ConfigureRuntimesInputs Include="common\runtime-base\CMakeLists.txt" /> <_ConfigureRuntimesOutputs Include="@(AndroidSupportedTargetJitAbi->'$(FlavorIntermediateOutputPath)\%(AndroidRID)-Debug\CMakeCache.txt')" /> @@ -217,7 +216,6 @@ <_RuntimeSources Include="common\archive-dso-stub\*.cc" /> <_RuntimeSources Include="common\include\**\*.hh" /> <_RuntimeSources Include="common\runtime-base\*.cc" /> - <_RuntimeSources Include="$(LZ4SourceFullPath)\lib\lz4.c;$(LZ4SourceFullPath)\lib\lz4.h" /> diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj index ec4c90fecb1..1a760876dfc 100644 --- a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -17,7 +17,6 @@ - diff --git a/tools/assembly-store-reader/assembly-store-reader.csproj b/tools/assembly-store-reader/assembly-store-reader.csproj index 9b0e604b1da..c2b4c58ff21 100644 --- a/tools/assembly-store-reader/assembly-store-reader.csproj +++ b/tools/assembly-store-reader/assembly-store-reader.csproj @@ -17,7 +17,6 @@ - diff --git a/tools/decompress-assemblies/decompress-assemblies.csproj b/tools/decompress-assemblies/decompress-assemblies.csproj index 1f789958973..cf59128cbc2 100644 --- a/tools/decompress-assemblies/decompress-assemblies.csproj +++ b/tools/decompress-assemblies/decompress-assemblies.csproj @@ -4,7 +4,7 @@ Microsoft Corporation 2021 Microsoft Corporation 0.0.1 - $(DotNetStableTargetFramework) + $(DotNetTargetFramework) false Xamarin.Android.Tools.DecompressAssemblies decompress-assemblies @@ -22,7 +22,6 @@ - diff --git a/tools/decompress-assemblies/main.cs b/tools/decompress-assemblies/main.cs index ba7d9ad4b2c..5efe0837a7e 100644 --- a/tools/decompress-assemblies/main.cs +++ b/tools/decompress-assemblies/main.cs @@ -2,15 +2,15 @@ using System.Buffers; using System.IO; -using K4os.Compression.LZ4; using Xamarin.Tools.Zip; using Xamarin.Android.AssemblyStore; +using ZstandardDecoder = System.IO.Compression.ZstandardDecoder; namespace Xamarin.Android.Tools.DecompressAssemblies { class App { - const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian + const uint CompressedDataMagic = 0x535A4158; // 'XAZS', little-endian static readonly ArrayPool bytePool = ArrayPool.Shared; @@ -30,8 +30,8 @@ static bool UncompressDLL (Stream inputStream, string fileName, string filePath, Console.WriteLine ($"Processing {fileName}"); // - // LZ4 compressed assembly header format: - // uint magic; // 0x5A4C4158; 'XALZ', little-endian + // Zstd compressed assembly header format: + // uint magic; // 0x535A4158; 'XAZS', little-endian // uint descriptor_index; // Index into an internal assembly descriptor table // uint uncompressed_length; // Size of assembly, uncompressed // @@ -46,9 +46,12 @@ static bool UncompressDLL (Stream inputStream, string fileName, string filePath, reader.Read (sourceBytes, 0, inputLength); byte[] assemblyBytes = bytePool.Rent ((int)decompressedLength); - int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength); + int decoded = ZstandardDecoder.TryDecompress ( + sourceBytes.AsSpan (0, inputLength), + assemblyBytes.AsSpan (0, (int)decompressedLength), + out int bytesWritten) ? bytesWritten : -1; if (decoded != (int)decompressedLength) { - Console.Error.WriteLine ($" Failed to decompress LZ4 data of {fileName} (decoded: {decoded})"); + Console.Error.WriteLine ($" Failed to decompress Zstd data of {fileName} (decoded: {decoded})"); retVal = false; } else { string? outputDir = Path.GetDirectoryName (outputFile); diff --git a/tools/tmt/ApkManagedTypeResolver.cs b/tools/tmt/ApkManagedTypeResolver.cs index 5f519e07275..444efcb187d 100644 --- a/tools/tmt/ApkManagedTypeResolver.cs +++ b/tools/tmt/ApkManagedTypeResolver.cs @@ -2,16 +2,16 @@ using System.Collections.Generic; using System.IO; -using K4os.Compression.LZ4; using Mono.Cecil; using Xamarin.Android.AssemblyStore; using Xamarin.Tools.Zip; +using ZstandardDecoder = System.IO.Compression.ZstandardDecoder; namespace tmt { class ApkManagedTypeResolver : ManagedTypeResolver { - const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian + const uint CompressedDataMagic = 0x535A4158; // 'XAZS', little-endian readonly Dictionary? individualAssemblies; readonly Dictionary? blobAssemblies; @@ -130,11 +130,12 @@ Stream PrepStream (Stream stream) protected override AssemblyDefinition ReadAssembly (string assemblyPath) { byte[]? assemblyBytes = null; + byte[]? sourceBytes = null; Stream stream = GetAssemblyStream (assemblyPath); // - // LZ4 compressed assembly header format: - // uint magic; // 0x5A4C4158; 'XALZ', little-endian + // Zstd compressed assembly header format: + // uint magic; // 0x535A4158; 'XAZS', little-endian // uint descriptor_index; // Index into an internal assembly descriptor table // uint uncompressed_length; // Size of assembly, uncompressed // @@ -145,18 +146,32 @@ protected override AssemblyDefinition ReadAssembly (string assemblyPath) uint decompressedLength = reader.ReadUInt32 (); int inputLength = (int)(stream.Length - 12); - byte[] sourceBytes = Utilities.BytePool.Rent (inputLength); - reader.Read (sourceBytes, 0, inputLength); - - assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength); - int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength); - if (decoded != (int)decompressedLength) { - throw new InvalidOperationException ($"Failed to decompress LZ4 data of {assemblyPath} (decoded: {decoded})"); + try { + sourceBytes = Utilities.BytePool.Rent (inputLength); + ReadFully (reader, sourceBytes, inputLength); + + assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength); + int decoded = ZstandardDecoder.TryDecompress ( + sourceBytes.AsSpan (0, inputLength), + assemblyBytes.AsSpan (0, (int)decompressedLength), + out int bytesWritten) ? bytesWritten : -1; + if (decoded != (int)decompressedLength) { + throw new InvalidOperationException ($"Failed to decompress Zstd data of {assemblyPath} (decoded: {decoded})"); + } + } catch { + if (assemblyBytes != null) { + Utilities.BytePool.Return (assemblyBytes); + assemblyBytes = null; + } + throw; + } finally { + if (sourceBytes != null) { + Utilities.BytePool.Return (sourceBytes); + sourceBytes = null; + } } - Utilities.BytePool.Return (sourceBytes); } - if (assemblyBytes != null) { stream.Close (); stream.Dispose (); @@ -168,5 +183,17 @@ protected override AssemblyDefinition ReadAssembly (string assemblyPath) return AssemblyDefinition.ReadAssembly (stream); } + + static void ReadFully (BinaryReader reader, byte[] destination, int count) + { + int totalRead = 0; + while (totalRead < count) { + int read = reader.Read (destination, totalRead, count - totalRead); + if (read <= 0) + throw new EndOfStreamException ("Unexpected end of stream while reading compressed assembly."); + + totalRead += read; + } + } } } diff --git a/tools/tmt/tmt.csproj b/tools/tmt/tmt.csproj index bf31f337de7..4c1ba65babd 100644 --- a/tools/tmt/tmt.csproj +++ b/tools/tmt/tmt.csproj @@ -4,7 +4,7 @@ Microsoft Corporation 2020 Microsoft Corporation 0.0.1 - $(DotNetStableTargetFramework) + $(DotNetTargetFramework) false ../../bin/$(Configuration)/bin/typemap-tool Exe @@ -25,7 +25,6 @@ -