From 29cd24d57e893d4d3173b96233a55c5fe507c168 Mon Sep 17 00:00:00 2001 From: Alexander Zekelin Date: Thu, 28 May 2026 14:06:24 +0200 Subject: [PATCH] feat: add 'data pkg cleanup' to delete records from a CMT package --- README.md | 3 + docs/configuration-migration.md | 43 +- docs/data-plane.md | 4 + .../Dataverse/IDataPackageService.cs | 66 +++ .../DataPackageCleanupCliCommand.cs | 187 ++++++++ .../DataPackageCliCommand.cs | 2 +- .../Sdk/DataPackageReader.cs | 302 +++++++++++++ .../DataverseDataPackageService.Cleanup.cs | 419 ++++++++++++++++++ .../Services/DataverseDataPackageService.cs | 2 +- .../DataPackageCleanupTests.cs | 59 +++ .../Data/DataPackageCleanupCliCommandTests.cs | 160 +++++++ .../Dataverse/DataPackageReaderTests.cs | 172 +++++++ 12 files changed, 1416 insertions(+), 3 deletions(-) create mode 100644 src/TALXIS.CLI.Features.Data/DataPackageCleanupCliCommand.cs create mode 100644 src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/DataPackageReader.cs create mode 100644 src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.Cleanup.cs create mode 100644 tests/TALXIS.CLI.IntegrationTests/DataPackageCleanupTests.cs create mode 100644 tests/TALXIS.CLI.Tests/Data/DataPackageCleanupCliCommandTests.cs create mode 100644 tests/TALXIS.CLI.Tests/Dataverse/DataPackageReaderTests.cs diff --git a/README.md b/README.md index c79092cd..8aaab78c 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,9 @@ txc data pkg import ./data-package \ --prefetch-limit 100 # pre-cache record lookups txc data pkg convert --input export.xlsx --output data.xml + +# Tear down records inserted by a previous import (handy for CI test teardown): +txc data pkg cleanup ./data-package --yes ``` See [docs/configuration-migration.md](docs/configuration-migration.md) for the full deep-dive into CMT internals, deduplication logic, and tuning strategies. diff --git a/docs/configuration-migration.md b/docs/configuration-migration.md index 729ba4fd..030beef1 100644 --- a/docs/configuration-migration.md +++ b/docs/configuration-migration.md @@ -20,6 +20,7 @@ The **Configuration Migration Tool (CMT)** is a Microsoft utility for migrating | Safety-check override | ✅ `--override-safety-checks` | ❌ | | Prefetch tuning | ✅ `--prefetch-limit` | ❌ | | XLSX → CMT XML conversion | ✅ `txc data package convert` | ❌ | +| Cleanup of records produced by a package | ✅ `txc data package cleanup` | ❌ | | Authentication | `txc` profiles | PAC auth profiles | ### When to Use CMT @@ -29,6 +30,7 @@ Use CMT / `txc data package` when you need to: - Move **reference/configuration data** (currencies, business units, security roles, option-set seed data) between environments. - Preserve **record GUIDs** across environments so that lookups and relationships remain intact. - Automate data seeding in CI/CD pipelines. +- Tear down test data inserted by a previous package import — see [`txc data package cleanup`](#cleanup---delete-records-produced-by-a-package). - Migrate **file columns** and **image columns** between environments. For **bulk transactional data** or **ETL workloads**, consider the Dataverse Import Data Wizard, Azure Data Factory, or SSIS instead. @@ -155,6 +157,45 @@ txc data package import data.zip \ --profile staging ``` +### Cleanup — delete records produced by a package + +Tear down everything a previous import created. Useful in automated tests where you spin up a fixture before each suite and remove it after. + +``` +txc data package cleanup [options] +``` + +| Argument / Option | Alias | Required | Default | Description | +|---|---|---|---|---| +| `` *(argument)* | — | **Yes** | — | Path to the CMT data package (`.zip` file or folder containing `data.xml` and `data_schema.xml`). | +| `--connection-count ` | — | No | `1` | Open N parallel `ServiceClient` connections; entities are sharded across them. Higher values speed up cleanup of many small entities at the cost of more concurrent throttle pressure. | +| `--batch-size ` | — | No | `200` | How many `DeleteRequest` messages to send per `ExecuteMultiple` batch. Lower is safer, higher is faster. | +| `--dry-run` | — | No | `false` | Parse the package and report what would be deleted without issuing any `DeleteRequest`. | +| `--missing-action ` | — | No | `by-natural-key` | What to do when a record can't be deleted by its GUID: `by-natural-key` (look it up via `primarynamefield` + every `updateCompare="true"` field), `skip` (count as not-found), or `fail` (abort the run). | +| `--continue-on-error` | — | No | `true` | Keep going after the first per-record failure. Set to `false` to abort on the first error. | +| `--yes` | — | No | `false` | Required in non-interactive sessions because the command is destructive. | +| `--allow-production` | — | No | `false` | Inherited; required when the target profile is detected as Production. | +| `--profile ` | `-p` | No | *(active profile)* | Profile name to resolve. | +| `--verbose` | — | No | `false` | Emit verbose logging for this invocation. | + +**How it works:** + +1. The package is parsed (folder or `.zip`) and entities are processed in the **reverse** of the schema's `` so children come down before their parents. +2. For each entity, `DeleteRequest` messages are batched through `ExecuteMultiple`. Records the server can't find by GUID fall through to a `QueryExpression` lookup against the schema's natural-key columns (the `primarynamefield` plus every field marked `updateCompare="true"`); an exact single match is then deleted. +3. Any `` blocks in `data.xml` are issued as `DisassociateRequest`s before the endpoint records are deleted, so cleanup also works when only one side of the N:N relationship lives in the package. + +**Example — clean up test data after an integration test:** + +```bash +txc data pkg cleanup ./fixtures/seed-data --profile dev --yes +``` + +**Example — preview without touching the environment:** + +```bash +txc data pkg cleanup ./data.zip --dry-run --profile dev --yes +``` + ### `txc data package convert` Convert tables from an XLSX file to CMT data package XML. @@ -839,7 +880,7 @@ This preserves the hour/minute/second component while shifting the date. ### `deleteBeforeAdd` is dead code -The CMT API accepts a `deleteBeforeAdd` parameter that is supposed to delete all existing records before importing. **This parameter exists in the code but is never actually executed** — the delete logic is unreachable. Do not rely on it. If you need a clean slate, truncate the target entity manually before import. +The CMT API accepts a `deleteBeforeAdd` parameter that is supposed to delete all existing records before importing. **This parameter exists in the code but is never actually executed** — the delete logic is unreachable. Do not rely on it. If you need a clean slate, use [`txc data package cleanup`](#cleanup---delete-records-produced-by-a-package) to tear down a previous import, or truncate the target entity manually. ### Image columns work despite documentation diff --git a/docs/data-plane.md b/docs/data-plane.md index e624a637..a352153b 100644 --- a/docs/data-plane.md +++ b/docs/data-plane.md @@ -210,6 +210,9 @@ For schema-driven dataset migration — exporting a curated slice of configurati txc data pkg export --schema ./data_schema.xml --output ./data-package --export-files txc data pkg import ./data-package txc data pkg convert --input export.xlsx --output data.xml + +# Tear down everything a previous import created (CI teardown, test fixtures): +txc data pkg cleanup ./data-package --yes ``` See [configuration-migration.md](configuration-migration.md) for the full deep-dive: deduplication logic, batching, parallel channels, prefetch tuning, and other options not exposed by PAC CLI or the CMT GUI. @@ -227,6 +230,7 @@ See [configuration-migration.md](configuration-migration.md) for the full deep-d | All-or-nothing semantics (rollback on any failure) | `record … --stage` × N, then `txc env changeset apply --strategy transaction` | | Heterogeneous mix, no rollback, but want a single round-trip | `record … --stage` × N, then `txc env changeset apply --strategy batch` | | Schema-driven dataset migration between environments | `txc data pkg export` / `import` (CMT) | +| Tear down records inserted by a previous CMT import (CI test teardown) | `txc data pkg cleanup` | --- diff --git a/src/TALXIS.CLI.Core/Contracts/Dataverse/IDataPackageService.cs b/src/TALXIS.CLI.Core/Contracts/Dataverse/IDataPackageService.cs index 18e56aeb..315e9bee 100644 --- a/src/TALXIS.CLI.Core/Contracts/Dataverse/IDataPackageService.cs +++ b/src/TALXIS.CLI.Core/Contracts/Dataverse/IDataPackageService.cs @@ -16,6 +16,56 @@ public sealed record DataPackageExportResult( string? ErrorMessage, bool InteractiveAuthRequired); +/// +/// What to do when a record listed in the package cannot be deleted by its +/// primary-key GUID (server returns ObjectDoesNotExist). +/// +public enum DataPackageCleanupMissingAction +{ + /// Look the record up by the schema's natural-key columns and delete that match if found. + ByNaturalKey = 0, + /// Count the record as not-found and move on. + Skip = 1, + /// Abort the run on the first miss. + Fail = 2, +} + +/// +/// Per-call tuning options for . +/// +public sealed record DataPackageCleanupOptions( + int BatchSize, + int ConnectionCount, + bool DryRun, + DataPackageCleanupMissingAction MissingAction, + bool ContinueOnError); + +/// +/// Per-entity cleanup statistics returned by . +/// +public sealed record DataPackageCleanupEntityResult( + string EntityLogicalName, + int Total, + int DeletedByGuid, + int DeletedByNaturalKey, + int NotFound, + int Errors, + IReadOnlyList ErrorMessages); + +/// +/// Outcome returned by . +/// +public sealed record DataPackageCleanupResult( + bool Succeeded, + string? ErrorMessage, + bool InteractiveAuthRequired, + IReadOnlyList EntityResults, + int TotalDeletedByGuid, + int TotalDeletedByNaturalKey, + int TotalNotFound, + int TotalErrors, + int M2mDisassociations); + /// /// Imports and exports Configuration-Migration-Tool (CMT) data packages /// for the Dataverse environment referenced by a profile. Hides subprocess @@ -41,4 +91,20 @@ Task ExportAsync( bool exportFiles, bool verbose, CancellationToken ct); + + /// + /// Deletes every record described by a CMT data package from the live + /// environment referenced by . Entities are + /// processed in reverse <entityImportOrder>. Deletes first + /// dispatch by the record's GUID (<record id>); on + /// ObjectDoesNotExist the schema's primary-name field and every + /// updateCompare="true" field are used as a natural-key fallback + /// per . + /// + Task CleanupAsync( + string? profileName, + string dataPackagePath, + DataPackageCleanupOptions options, + bool verbose, + CancellationToken ct); } diff --git a/src/TALXIS.CLI.Features.Data/DataPackageCleanupCliCommand.cs b/src/TALXIS.CLI.Features.Data/DataPackageCleanupCliCommand.cs new file mode 100644 index 00000000..6068dd5b --- /dev/null +++ b/src/TALXIS.CLI.Features.Data/DataPackageCleanupCliCommand.cs @@ -0,0 +1,187 @@ +using System.ComponentModel; +using System.Text.Json; +using DotMake.CommandLine; +using Microsoft.Extensions.Logging; +using TALXIS.CLI.Core; +using TALXIS.CLI.Core.Abstractions; +using TALXIS.CLI.Core.Contracts.Dataverse; +using TALXIS.CLI.Core.DependencyInjection; +using TALXIS.CLI.Logging; + +namespace TALXIS.CLI.Features.Data; + +[CliDestructive("Permanently deletes every record listed in the CMT data package from the target Dataverse environment.")] +[CliLongRunning] +[CliWorkflow("data-operations")] +[CliCommand( + Name = "cleanup", + Description = "Deletes every record contained in a CMT data package from the LIVE Dataverse environment referenced by the active profile. Intended for tearing down test data inserted by a previous 'data package import'." +)] +public class DataPackageCleanupCliCommand : ProfiledCliCommand, IDestructiveCommand +{ + protected override ILogger Logger { get; } = TxcLoggerFactory.CreateLogger(nameof(DataPackageCleanupCliCommand)); + + [CliArgument(Description = "Path to the CMT data package (.zip file or folder containing data.xml and data_schema.xml)")] + public required string Data { get; set; } + + [CliOption(Name = "--connection-count", Description = "How many parallel connections to open. Entities are sharded across connections — higher values speed up cleanup of many small entities at the cost of more concurrent throttle pressure.", Required = false)] + [DefaultValue(1)] + public int ConnectionCount { get; set; } = 1; + + [CliOption(Name = "--batch-size", Description = "How many DeleteRequest messages to send per ExecuteMultiple batch. Lower is safer, higher is faster.", Required = false)] + [DefaultValue(200)] + public int BatchSize { get; set; } = 200; + + [CliOption(Name = "--dry-run", Description = "Parse the package and report what would be deleted without issuing any DeleteRequest.", Required = false)] + [DefaultValue(false)] + public bool DryRun { get; set; } + + [CliOption(Name = "--missing-action", Description = "What to do when a record can't be deleted by its GUID: by-natural-key (look it up via primary-name + updateCompare fields), skip (count as not-found), or fail (abort the run).", Required = false)] + [DefaultValue("by-natural-key")] + public string MissingAction { get; set; } = "by-natural-key"; + + [CliOption(Name = "--continue-on-error", Description = "Keep going after the first per-record failure. Default true — set to false to abort on the first error.", Required = false)] + [DefaultValue(true)] + public bool ContinueOnError { get; set; } = true; + + [CliOption(Name = "--yes", Description = "Skip interactive confirmation for this destructive operation.", Required = false)] + public bool Yes { get; set; } + + protected override async Task ExecuteAsync() + { + if (string.IsNullOrWhiteSpace(Data)) + { + Logger.LogError("A path to a CMT data package (.zip or folder) must be provided."); + return ExitValidationError; + } + + if (!File.Exists(Data) && !Directory.Exists(Data)) + { + Logger.LogError("Data package not found: {DataPath}", Data); + return ExitValidationError; + } + + if (BatchSize <= 0) + { + Logger.LogError("--batch-size must be greater than zero (got {BatchSize}).", BatchSize); + return ExitValidationError; + } + + if (ConnectionCount <= 0) + { + Logger.LogError("--connection-count must be greater than zero (got {ConnectionCount}).", ConnectionCount); + return ExitValidationError; + } + + if (!TryParseMissingAction(MissingAction, out var missingAction)) + { + Logger.LogError("--missing-action must be one of: by-natural-key, skip, fail (got '{Value}').", MissingAction); + return ExitValidationError; + } + + var service = TxcServices.Get(); + var options = new DataPackageCleanupOptions(BatchSize, ConnectionCount, DryRun, missingAction, ContinueOnError); + + var result = await service.CleanupAsync(Profile, Data, options, Verbose, CancellationToken.None).ConfigureAwait(false); + + if (result.InteractiveAuthRequired) + { + Logger.LogError("Interactive authentication is required. Run 'txc config auth login' for profile '{Profile}' and retry.", Profile ?? "(default)"); + OutputFormatter.WriteResult("failed", "Interactive authentication required.", exitCode: ExitError); + return ExitError; + } + + if (result.ErrorMessage is not null) + { + Logger.LogError("{ErrorMessage}", result.ErrorMessage); + OutputFormatter.WriteResult("failed", result.ErrorMessage, exitCode: ExitError); + return ExitError; + } + + EmitReport(result); + + if (!result.Succeeded) + { + Logger.LogError( + "Data package cleanup completed with errors. Deleted: {DeletedByGuid} by id, {DeletedByNaturalKey} by natural key. Not found: {NotFound}. Errors: {Errors}.", + result.TotalDeletedByGuid, result.TotalDeletedByNaturalKey, result.TotalNotFound, result.TotalErrors); + OutputFormatter.WriteResult("failed", $"{result.TotalErrors} record(s) failed to delete.", exitCode: ExitError); + return ExitError; + } + + var summary = DryRun + ? $"Dry run: would delete {result.TotalDeletedByGuid} record(s) and disassociate {result.M2mDisassociations} M:N pair(s)." + : $"Deleted {result.TotalDeletedByGuid + result.TotalDeletedByNaturalKey} record(s) ({result.TotalDeletedByNaturalKey} via natural-key fallback). {result.TotalNotFound} not found. Disassociated {result.M2mDisassociations} M:N pair(s)."; + OutputFormatter.WriteResult("succeeded", summary); + return ExitSuccess; + } + + private void EmitReport(DataPackageCleanupResult result) + { + if (!OutputContext.IsJson) + { + foreach (var entity in result.EntityResults) + { + if (entity.Total == 0) + continue; + Logger.LogInformation( + "{Entity}: {Total} record(s) — deleted {DeletedByGuid} by id, {DeletedByNaturalKey} by natural key, {NotFound} not found, {Errors} error(s).", + entity.EntityLogicalName, entity.Total, entity.DeletedByGuid, entity.DeletedByNaturalKey, entity.NotFound, entity.Errors); + foreach (var message in entity.ErrorMessages) + Logger.LogWarning(" {ErrorMessage}", message); + } + return; + } + + var payload = new + { + entities = result.EntityResults.Select(e => new + { + entity = e.EntityLogicalName, + total = e.Total, + deletedByGuid = e.DeletedByGuid, + deletedByNaturalKey = e.DeletedByNaturalKey, + notFound = e.NotFound, + errors = e.Errors, + errorMessages = e.ErrorMessages, + }).ToArray(), + totals = new + { + deletedByGuid = result.TotalDeletedByGuid, + deletedByNaturalKey = result.TotalDeletedByNaturalKey, + notFound = result.TotalNotFound, + errors = result.TotalErrors, + m2mDisassociations = result.M2mDisassociations, + }, + dryRun = DryRun, + }; + OutputFormatter.WriteData(payload); + } + + /// + /// Pure helper kept for test coverage: parses the textual value passed to + /// --missing-action into the corresponding enum. + /// + public static bool TryParseMissingAction(string? value, out DataPackageCleanupMissingAction action) + { + switch (value?.Trim().ToLowerInvariant()) + { + case null: + case "": + case "by-natural-key": + case "natural-key": + case "naturalkey": + action = DataPackageCleanupMissingAction.ByNaturalKey; + return true; + case "skip": + action = DataPackageCleanupMissingAction.Skip; + return true; + case "fail": + action = DataPackageCleanupMissingAction.Fail; + return true; + default: + action = DataPackageCleanupMissingAction.ByNaturalKey; + return false; + } + } +} diff --git a/src/TALXIS.CLI.Features.Data/DataPackageCliCommand.cs b/src/TALXIS.CLI.Features.Data/DataPackageCliCommand.cs index 4865924a..7430adad 100644 --- a/src/TALXIS.CLI.Features.Data/DataPackageCliCommand.cs +++ b/src/TALXIS.CLI.Features.Data/DataPackageCliCommand.cs @@ -6,7 +6,7 @@ namespace TALXIS.CLI.Features.Data; Name = "package", Alias = "pkg", Description = "Configuration migration tool (CMT) for moving data between different environments", - Children = new[] { typeof(DataPackageImportCliCommand), typeof(DataPackageExportCliCommand) }, + Children = new[] { typeof(DataPackageImportCliCommand), typeof(DataPackageExportCliCommand), typeof(DataPackageConvertCliCommand), typeof(DataPackageCleanupCliCommand) }, ShortFormAutoGenerate = CliNameAutoGenerate.None )] public class DataPackageCliCommand diff --git a/src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/DataPackageReader.cs b/src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/DataPackageReader.cs new file mode 100644 index 00000000..e42654e2 --- /dev/null +++ b/src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/DataPackageReader.cs @@ -0,0 +1,302 @@ +using System.IO.Compression; +using System.Xml.Linq; + +namespace TALXIS.CLI.Platform.Dataverse.Application.Sdk; + +/// +/// Parsed contents of a CMT data package — just enough to drive cleanup. +/// We deliberately do not reuse the legacy CMT parser here because (a) it +/// runs in a subprocess and (b) it loads the whole record set as +/// EntityCollections, which is more work than we need. +/// +internal sealed record DataPackageContents( + IReadOnlyList EntityImportOrder, + IReadOnlyDictionary Schemas, + IReadOnlyDictionary> Records, + IReadOnlyList M2mAssociations); + +/// +/// Schema slice for a single entity: the primary-id column plus the +/// natural-key fields used as the dedup fallback during cleanup. +/// +internal sealed record DataPackageEntitySchema( + string LogicalName, + string PrimaryIdField, + string? PrimaryNameField, + IReadOnlyList NaturalKeyFields); + +/// +/// Single record as it appears in data.xml. +/// +internal sealed record DataPackageRecordRow( + Guid Id, + IReadOnlyDictionary Fields); + +/// +/// One source/target pair drawn from a <m2mrelationship> block. +/// One row per target id (denormalised so callers can issue +/// DisassociateRequest per pair). +/// +internal sealed record DataPackageM2mAssociation( + string RelationshipName, + string SourceEntity, + Guid SourceId, + string TargetEntity, + Guid TargetId); + +/// +/// Loads a CMT package from a folder or a .zip archive into an +/// in-memory . Pure parsing — no IO beyond +/// reading the two XML files. +/// +internal static class DataPackageReader +{ + private const string DataFileName = "data.xml"; + private const string SchemaFileName = "data_schema.xml"; + + public static DataPackageContents Load(string path) + { + if (string.IsNullOrWhiteSpace(path)) + throw new ArgumentException("Data package path is required.", nameof(path)); + + XDocument schemaDoc; + XDocument dataDoc; + + if (Directory.Exists(path)) + { + var schemaFile = Path.Combine(path, SchemaFileName); + var dataFile = Path.Combine(path, DataFileName); + if (!File.Exists(schemaFile) || !File.Exists(dataFile)) + { + throw new InvalidOperationException( + $"Folder '{path}' does not contain required CMT files ({DataFileName} and {SchemaFileName})."); + } + schemaDoc = XDocument.Load(schemaFile); + dataDoc = XDocument.Load(dataFile); + } + else if (File.Exists(path)) + { + using var zip = ZipFile.OpenRead(path); + schemaDoc = LoadEntryAsXml(zip, SchemaFileName, path); + dataDoc = LoadEntryAsXml(zip, DataFileName, path); + } + else + { + throw new InvalidOperationException($"Data package not found: '{path}'."); + } + + var importOrder = ParseEntityImportOrder(schemaDoc); + var schemas = ParseSchemas(schemaDoc); + var m2mDefs = ParseM2mRelationshipDefinitions(schemaDoc); + var records = ParseRecords(dataDoc); + var m2mAssociations = ParseM2mAssociations(dataDoc, m2mDefs); + + return new DataPackageContents(importOrder, schemas, records, m2mAssociations); + } + + private static XDocument LoadEntryAsXml(ZipArchive zip, string name, string archivePath) + { + var entry = FindEntry(zip, name) + ?? throw new InvalidOperationException( + $"Archive '{archivePath}' does not contain required CMT file '{name}'."); + using var stream = entry.Open(); + return XDocument.Load(stream); + } + + private static ZipArchiveEntry? FindEntry(ZipArchive zip, string name) + { + foreach (var entry in zip.Entries) + { + if (string.Equals(entry.Name, name, StringComparison.OrdinalIgnoreCase)) + return entry; + } + return null; + } + + private static IReadOnlyList ParseEntityImportOrder(XDocument schema) + { + var root = schema.Root; + if (root is null) return Array.Empty(); + + var orderElement = root.Element("entityImportOrder"); + if (orderElement is null) return Array.Empty(); + + return orderElement.Elements("entityName") + .Select(e => e.Value?.Trim()) + .Where(n => !string.IsNullOrWhiteSpace(n)) + .Select(n => n!) + .ToList(); + } + + private static IReadOnlyDictionary ParseSchemas(XDocument schema) + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + var root = schema.Root; + if (root is null) return dict; + + foreach (var entity in root.Elements("entity")) + { + var name = (string?)entity.Attribute("name"); + if (string.IsNullOrWhiteSpace(name)) continue; + + var primaryId = (string?)entity.Attribute("primaryidfield") ?? string.Empty; + var primaryName = (string?)entity.Attribute("primarynamefield"); + + var naturalKey = new List(); + if (!string.IsNullOrWhiteSpace(primaryName)) + naturalKey.Add(primaryName!); + + var fieldsRoot = entity.Element("fields"); + if (fieldsRoot is not null) + { + foreach (var field in fieldsRoot.Elements("field")) + { + var fname = (string?)field.Attribute("name"); + if (string.IsNullOrWhiteSpace(fname)) continue; + + var update = ParseBool((string?)field.Attribute("updateCompare")); + if (update && !naturalKey.Contains(fname!, StringComparer.OrdinalIgnoreCase)) + naturalKey.Add(fname!); + } + } + + dict[name!] = new DataPackageEntitySchema(name!, primaryId, primaryName, naturalKey); + } + + return dict; + } + + private static IReadOnlyDictionary ParseM2mRelationshipDefinitions(XDocument schema) + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + var root = schema.Root; + if (root is null) return dict; + + foreach (var entity in root.Elements("entity")) + { + var sourceEntity = (string?)entity.Attribute("name"); + if (string.IsNullOrWhiteSpace(sourceEntity)) continue; + + var relationships = entity.Element("relationships"); + if (relationships is null) continue; + + foreach (var rel in relationships.Elements("relationship")) + { + var manyToMany = ParseBool((string?)rel.Attribute("manyToMany")); + if (!manyToMany) continue; + + var relName = (string?)rel.Attribute("name"); + if (string.IsNullOrWhiteSpace(relName)) continue; + + var target = (string?)rel.Attribute("m2mTargetEntity"); + if (string.IsNullOrWhiteSpace(target)) continue; + + // Keep the first occurrence — both endpoints often declare the + // same relationship, but the source-side declaration is the one + // whose block in data.xml lists the targets. + dict.TryAdd(relName!, new M2mRelationshipDef(relName!, sourceEntity!, target!)); + } + } + + return dict; + } + + private static IReadOnlyDictionary> ParseRecords(XDocument data) + { + var dict = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var root = data.Root; + if (root is null) return dict; + + foreach (var entity in root.Elements("entity")) + { + var name = (string?)entity.Attribute("name"); + if (string.IsNullOrWhiteSpace(name)) continue; + + var records = new List(); + var recordsRoot = entity.Element("records"); + if (recordsRoot is not null) + { + foreach (var record in recordsRoot.Elements("record")) + { + var idAttr = (string?)record.Attribute("id"); + if (!Guid.TryParse(idAttr, out var id)) continue; + + var fields = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var field in record.Elements("field")) + { + var fname = (string?)field.Attribute("name"); + if (string.IsNullOrWhiteSpace(fname)) continue; + var fvalue = (string?)field.Attribute("value"); + fields[fname!] = fvalue; + } + + records.Add(new DataPackageRecordRow(id, fields)); + } + } + + dict[name!] = records; + } + + return dict; + } + + private static IReadOnlyList ParseM2mAssociations( + XDocument data, IReadOnlyDictionary m2mDefs) + { + var list = new List(); + var root = data.Root; + if (root is null) return list; + + foreach (var entity in root.Elements("entity")) + { + var sourceEntity = (string?)entity.Attribute("name"); + if (string.IsNullOrWhiteSpace(sourceEntity)) continue; + + var m2mRoot = entity.Element("m2mrelationships"); + if (m2mRoot is null) continue; + + foreach (var m2m in m2mRoot.Elements("m2mrelationship")) + { + var relName = (string?)m2m.Attribute("m2mrelationshipname"); + var sourceIdAttr = (string?)m2m.Attribute("sourceid"); + if (string.IsNullOrWhiteSpace(relName)) continue; + if (!Guid.TryParse(sourceIdAttr, out var sourceId)) continue; + + // The relationship's target entity is recorded only in the schema. + // Without a definition we can't issue a disassociate — skip silently; + // the legacy CMT importer behaves the same way. + if (!m2mDefs.TryGetValue(relName!, out var def)) continue; + + // The source endpoint in the schema may not match the entity that + // is hosting the data block (e.g. reflexive N:N). Prefer the + // hosting entity as the source so DisassociateRequest gets the + // correct EntityReference type. + var resolvedSource = string.Equals(def.SourceEntity, sourceEntity, StringComparison.OrdinalIgnoreCase) + ? def.SourceEntity + : sourceEntity!; + var resolvedTarget = string.Equals(def.SourceEntity, sourceEntity, StringComparison.OrdinalIgnoreCase) + ? def.TargetEntity + : def.SourceEntity; + + var targetsRoot = m2m.Element("targetids"); + if (targetsRoot is null) continue; + + foreach (var t in targetsRoot.Elements("targetid")) + { + if (!Guid.TryParse(t.Value, out var targetId)) continue; + list.Add(new DataPackageM2mAssociation( + relName!, resolvedSource, sourceId, resolvedTarget, targetId)); + } + } + } + + return list; + } + + private static bool ParseBool(string? value) + => !string.IsNullOrWhiteSpace(value) + && (string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) + || value == "1"); + + private sealed record M2mRelationshipDef(string Name, string SourceEntity, string TargetEntity); +} diff --git a/src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.Cleanup.cs b/src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.Cleanup.cs new file mode 100644 index 00000000..41caa84f --- /dev/null +++ b/src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.Cleanup.cs @@ -0,0 +1,419 @@ +using Microsoft.Identity.Client; +using Microsoft.PowerPlatform.Dataverse.Client; +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Messages; +using Microsoft.Xrm.Sdk.Query; +using TALXIS.CLI.Core.Abstractions; +using TALXIS.CLI.Core.Contracts.Dataverse; +using TALXIS.CLI.Platform.Dataverse.Application.Sdk; +using TALXIS.CLI.Platform.Dataverse.Runtime; + +namespace TALXIS.CLI.Platform.Dataverse.Application.Services; + +internal sealed partial class DataverseDataPackageService +{ + /// + /// Dataverse OrganizationServiceFault.ErrorCode for "record does not exist". + /// + private const int ObjectDoesNotExistErrorCode = -2147220969; + + public async Task CleanupAsync( + string? profileName, + string dataPackagePath, + DataPackageCleanupOptions options, + bool verbose, + CancellationToken ct) + { + try + { + await DataverseCommandBridge.PrimeTokenAsync(profileName, ct).ConfigureAwait(false); + } + catch (MsalUiRequiredException) + { + return EmptyCleanupResult(interactiveAuthRequired: true); + } + catch (Exception ex) when (ex is ConfigurationResolutionException or InvalidOperationException or NotSupportedException) + { + return EmptyCleanupResult(errorMessage: ex.Message); + } + + DataPackageContents contents; + try + { + contents = DataPackageReader.Load(Path.GetFullPath(dataPackagePath)); + } + catch (Exception ex) when (ex is InvalidOperationException or FileNotFoundException or ArgumentException) + { + return EmptyCleanupResult(errorMessage: ex.Message); + } + + var connectionCount = Math.Max(1, options.ConnectionCount); + var connections = new List(connectionCount); + try + { + for (int i = 0; i < connectionCount; i++) + { + connections.Add(await DataverseCommandBridge.ConnectAsync(profileName, ct).ConfigureAwait(false)); + } + + var primary = connections[0].Client; + + int m2mCount = await ProcessM2mAsync(primary, contents.M2mAssociations, options, ct).ConfigureAwait(false); + + var order = BuildCleanupOrder(contents); + var entityResults = await ProcessEntitiesAsync(connections, contents, order, options, ct).ConfigureAwait(false); + + var succeeded = entityResults.All(r => r.Errors == 0); + return new DataPackageCleanupResult( + Succeeded: succeeded, + ErrorMessage: null, + InteractiveAuthRequired: false, + EntityResults: entityResults, + TotalDeletedByGuid: entityResults.Sum(r => r.DeletedByGuid), + TotalDeletedByNaturalKey: entityResults.Sum(r => r.DeletedByNaturalKey), + TotalNotFound: entityResults.Sum(r => r.NotFound), + TotalErrors: entityResults.Sum(r => r.Errors), + M2mDisassociations: m2mCount); + } + finally + { + foreach (var c in connections) + c.Dispose(); + } + } + + /// + /// Returns the entity order to process during cleanup: the schema's + /// <entityImportOrder> reversed, plus any entity with records + /// that wasn't listed in import order (appended in the order they appear + /// in data.xml). + /// + internal static IReadOnlyList BuildCleanupOrder(DataPackageContents contents) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var order = new List(); + + for (int i = contents.EntityImportOrder.Count - 1; i >= 0; i--) + { + var name = contents.EntityImportOrder[i]; + if (seen.Add(name)) + order.Add(name); + } + + foreach (var name in contents.Records.Keys) + { + if (seen.Add(name)) + order.Add(name); + } + + return order; + } + + private async Task ProcessM2mAsync( + ServiceClient client, + IReadOnlyList associations, + DataPackageCleanupOptions options, + CancellationToken ct) + { + if (associations.Count == 0) + return 0; + + if (options.DryRun) + return associations.Count; + + int processed = 0; + var batchSize = Math.Max(1, options.BatchSize); + + // Group by (relationship, source) so we can disassociate many targets in one request. + var groups = associations + .GroupBy(a => (a.RelationshipName, a.SourceEntity, a.SourceId, a.TargetEntity)) + .ToList(); + + var requests = new OrganizationRequestCollection(); + foreach (var group in groups) + { + var sourceRef = new EntityReference(group.Key.SourceEntity, group.Key.SourceId); + var relatedRefs = new EntityReferenceCollection(); + foreach (var pair in group) + relatedRefs.Add(new EntityReference(group.Key.TargetEntity, pair.TargetId)); + + requests.Add(new DisassociateRequest + { + Target = sourceRef, + RelatedEntities = relatedRefs, + Relationship = new Relationship(group.Key.RelationshipName), + }); + + processed += relatedRefs.Count; + + if (requests.Count >= batchSize) + { + await ExecuteMultipleIgnoringMissingAsync(client, requests, options.ContinueOnError, ct).ConfigureAwait(false); + requests = new OrganizationRequestCollection(); + } + } + + if (requests.Count > 0) + await ExecuteMultipleIgnoringMissingAsync(client, requests, options.ContinueOnError, ct).ConfigureAwait(false); + + return processed; + } + + private static async Task ExecuteMultipleIgnoringMissingAsync( + ServiceClient client, + OrganizationRequestCollection requests, + bool continueOnError, + CancellationToken ct) + { + await client.ExecuteAsync(new ExecuteMultipleRequest + { + Settings = new ExecuteMultipleSettings + { + ContinueOnError = continueOnError, + ReturnResponses = true, + }, + Requests = requests, + }, ct).ConfigureAwait(false); + } + + private async Task> ProcessEntitiesAsync( + IReadOnlyList connections, + DataPackageContents contents, + IReadOnlyList order, + DataPackageCleanupOptions options, + CancellationToken ct) + { + var results = new List(); + var resultsLock = new object(); + + if (connections.Count > 1) + { + var indexCounter = -1; + await Parallel.ForEachAsync( + order, + new ParallelOptions { MaxDegreeOfParallelism = connections.Count, CancellationToken = ct }, + async (entityName, innerCt) => + { + var idx = Interlocked.Increment(ref indexCounter); + var client = connections[idx % connections.Count].Client; + var r = await CleanupEntityAsync(client, entityName, contents, options, innerCt).ConfigureAwait(false); + lock (resultsLock) + results.Add(r); + }).ConfigureAwait(false); + } + else + { + var primary = connections[0].Client; + foreach (var entityName in order) + { + ct.ThrowIfCancellationRequested(); + var r = await CleanupEntityAsync(primary, entityName, contents, options, ct).ConfigureAwait(false); + results.Add(r); + if (!options.ContinueOnError && r.Errors > 0) + break; + } + } + + return results; + } + + private static async Task CleanupEntityAsync( + ServiceClient client, + string entityName, + DataPackageContents contents, + DataPackageCleanupOptions options, + CancellationToken ct) + { + if (!contents.Records.TryGetValue(entityName, out var records) || records.Count == 0) + { + return new DataPackageCleanupEntityResult(entityName, 0, 0, 0, 0, 0, Array.Empty()); + } + + contents.Schemas.TryGetValue(entityName, out var schema); + + int deletedByGuid = 0; + int deletedByNaturalKey = 0; + int notFound = 0; + int errors = 0; + var errorMessages = new List(); + + var batchSize = Math.Max(1, options.BatchSize); + + for (int offset = 0; offset < records.Count; offset += batchSize) + { + var chunk = records.Skip(offset).Take(batchSize).ToList(); + + if (options.DryRun) + { + deletedByGuid += chunk.Count; + continue; + } + + var requests = new OrganizationRequestCollection(); + foreach (var r in chunk) + requests.Add(new DeleteRequest { Target = new EntityReference(entityName, r.Id) }); + + ExecuteMultipleResponse response; + try + { + response = (ExecuteMultipleResponse)await client.ExecuteAsync(new ExecuteMultipleRequest + { + Settings = new ExecuteMultipleSettings + { + ContinueOnError = true, + ReturnResponses = true, + }, + Requests = requests, + }, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + errors += chunk.Count; + errorMessages.Add(ex.Message); + if (!options.ContinueOnError) + return new DataPackageCleanupEntityResult( + entityName, records.Count, deletedByGuid, deletedByNaturalKey, notFound, errors, errorMessages); + continue; + } + + for (int i = 0; i < chunk.Count; i++) + { + var record = chunk[i]; + var item = response.Responses.FirstOrDefault(r => r.RequestIndex == i); + if (item is null || item.Fault is null) + { + deletedByGuid++; + continue; + } + + var fault = item.Fault; + if (fault.ErrorCode == ObjectDoesNotExistErrorCode) + { + var fallback = await TryNaturalKeyDeleteAsync(client, entityName, record, schema, options, ct).ConfigureAwait(false); + switch (fallback.Outcome) + { + case NaturalKeyOutcome.DeletedByNaturalKey: + deletedByNaturalKey++; + break; + case NaturalKeyOutcome.NotFound: + notFound++; + if (options.MissingAction == DataPackageCleanupMissingAction.Fail) + { + errors++; + errorMessages.Add($"{entityName} {record.Id}: not found by id or natural key."); + return new DataPackageCleanupEntityResult( + entityName, records.Count, deletedByGuid, deletedByNaturalKey, notFound, errors, errorMessages); + } + break; + case NaturalKeyOutcome.Error: + errors++; + if (fallback.ErrorMessage is not null) + errorMessages.Add($"{entityName} {record.Id}: {fallback.ErrorMessage}"); + if (!options.ContinueOnError) + return new DataPackageCleanupEntityResult( + entityName, records.Count, deletedByGuid, deletedByNaturalKey, notFound, errors, errorMessages); + break; + } + } + else + { + errors++; + errorMessages.Add($"{entityName} {record.Id}: {fault.Message}"); + if (!options.ContinueOnError) + return new DataPackageCleanupEntityResult( + entityName, records.Count, deletedByGuid, deletedByNaturalKey, notFound, errors, errorMessages); + } + } + } + + return new DataPackageCleanupEntityResult( + entityName, records.Count, deletedByGuid, deletedByNaturalKey, notFound, errors, errorMessages); + } + + private static async Task TryNaturalKeyDeleteAsync( + ServiceClient client, + string entityName, + DataPackageRecordRow record, + DataPackageEntitySchema? schema, + DataPackageCleanupOptions options, + CancellationToken ct) + { + if (options.MissingAction != DataPackageCleanupMissingAction.ByNaturalKey) + return new NaturalKeyResult(NaturalKeyOutcome.NotFound, null); + + if (schema is null) + return new NaturalKeyResult(NaturalKeyOutcome.NotFound, null); + + var primaryId = schema.PrimaryIdField; + if (string.IsNullOrWhiteSpace(primaryId)) + return new NaturalKeyResult(NaturalKeyOutcome.NotFound, null); + + // Build conditions from every natural-key column that has a value on the record. + var conditions = new List(); + foreach (var fieldName in schema.NaturalKeyFields) + { + if (!record.Fields.TryGetValue(fieldName, out var value) || string.IsNullOrEmpty(value)) + continue; + conditions.Add(new ConditionExpression(fieldName, ConditionOperator.Equal, value)); + } + + if (conditions.Count == 0) + return new NaturalKeyResult(NaturalKeyOutcome.NotFound, null); + + var query = new QueryExpression(entityName) + { + ColumnSet = new ColumnSet(primaryId), + TopCount = 2, + }; + foreach (var c in conditions) + query.Criteria.Conditions.Add(c); + + EntityCollection matches; + try + { + matches = await client.RetrieveMultipleAsync(query, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + return new NaturalKeyResult(NaturalKeyOutcome.Error, ex.Message); + } + + if (matches.Entities.Count == 0) + return new NaturalKeyResult(NaturalKeyOutcome.NotFound, null); + + if (matches.Entities.Count > 1) + return new NaturalKeyResult(NaturalKeyOutcome.Error, "ambiguous natural-key match."); + + try + { + await client.DeleteAsync(entityName, matches.Entities[0].Id, ct).ConfigureAwait(false); + return new NaturalKeyResult(NaturalKeyOutcome.DeletedByNaturalKey, null); + } + catch (Exception ex) + { + return new NaturalKeyResult(NaturalKeyOutcome.Error, ex.Message); + } + } + + private static DataPackageCleanupResult EmptyCleanupResult( + bool interactiveAuthRequired = false, string? errorMessage = null) + => new( + Succeeded: false, + ErrorMessage: errorMessage, + InteractiveAuthRequired: interactiveAuthRequired, + EntityResults: Array.Empty(), + TotalDeletedByGuid: 0, + TotalDeletedByNaturalKey: 0, + TotalNotFound: 0, + TotalErrors: 0, + M2mDisassociations: 0); + + private enum NaturalKeyOutcome + { + DeletedByNaturalKey, + NotFound, + Error, + } + + private sealed record NaturalKeyResult(NaturalKeyOutcome Outcome, string? ErrorMessage); +} diff --git a/src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.cs b/src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.cs index c311d25d..8d6f3214 100644 --- a/src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.cs +++ b/src/TALXIS.CLI.Platform.Dataverse.Application/Services/DataverseDataPackageService.cs @@ -7,7 +7,7 @@ namespace TALXIS.CLI.Platform.Dataverse.Application.Services; -internal sealed class DataverseDataPackageService : IDataPackageService +internal sealed partial class DataverseDataPackageService : IDataPackageService { public async Task ImportAsync( string? profileName, diff --git a/tests/TALXIS.CLI.IntegrationTests/DataPackageCleanupTests.cs b/tests/TALXIS.CLI.IntegrationTests/DataPackageCleanupTests.cs new file mode 100644 index 00000000..b6111c85 --- /dev/null +++ b/tests/TALXIS.CLI.IntegrationTests/DataPackageCleanupTests.cs @@ -0,0 +1,59 @@ +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace TALXIS.CLI.IntegrationTests; + +/// +/// Black-box exit-code tests for txc data pkg cleanup. These don't +/// connect to a live environment — they just confirm the command is wired +/// into the tree and rejects obviously bad inputs at the validation stage. +/// +[Collection("Sequential")] +public class DataPackageCleanupTests +{ + [Fact] + public async Task Cleanup_Help_ExitsZero() + { + var result = await CliRunner.RunRawAsync(["data", "pkg", "cleanup", "--help"]); + + Assert.Equal(0, result.ExitCode); + Assert.Contains("cleanup", result.Output, StringComparison.OrdinalIgnoreCase); + Assert.Contains("--yes", result.Output, StringComparison.OrdinalIgnoreCase); + Assert.Contains("--missing-action", result.Output, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Cleanup_NonExistentPath_ReturnsValidationError() + { + var result = await CliRunner.RunRawAsync( + ["data", "pkg", "cleanup", + Path.Combine(Path.GetTempPath(), "txc-no-such-package-" + Guid.NewGuid().ToString("N")), + "--yes"]); + + Assert.NotEqual(0, result.ExitCode); + } + + [Fact] + public async Task Cleanup_UnknownMissingAction_ReturnsValidationError() + { + var folder = Path.Combine(Path.GetTempPath(), "txc-cleanup-bad-flag-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + try + { + File.WriteAllText(Path.Combine(folder, "data_schema.xml"), ""); + File.WriteAllText(Path.Combine(folder, "data.xml"), ""); + + var result = await CliRunner.RunRawAsync( + ["data", "pkg", "cleanup", folder, + "--missing-action", "magic", + "--yes"]); + + Assert.NotEqual(0, result.ExitCode); + } + finally + { + Directory.Delete(folder, recursive: true); + } + } +} diff --git a/tests/TALXIS.CLI.Tests/Data/DataPackageCleanupCliCommandTests.cs b/tests/TALXIS.CLI.Tests/Data/DataPackageCleanupCliCommandTests.cs new file mode 100644 index 00000000..087ad6a4 --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataPackageCleanupCliCommandTests.cs @@ -0,0 +1,160 @@ +using System.IO; +using TALXIS.CLI.Core.Contracts.Dataverse; +using TALXIS.CLI.Features.Data; +using TALXIS.CLI.Platform.Dataverse.Application.Sdk; +using TALXIS.CLI.Platform.Dataverse.Application.Services; +using Xunit; + +namespace TALXIS.CLI.Tests.Data; + +/// +/// Validates input-validation paths on +/// and the pure helpers it exposes. These tests deliberately stop before any +/// service call so they don't depend on a live Dataverse connection. +/// +public class DataPackageCleanupCliCommandTests +{ + private const int ExitValidationError = 2; + + [Fact] + public async Task RunAsync_MissingPath_ReturnsValidationError() + { + var cmd = new DataPackageCleanupCliCommand { Data = " ", Yes = true }; + + var exit = await cmd.RunAsync(); + + Assert.Equal(ExitValidationError, exit); + } + + [Fact] + public async Task RunAsync_NonExistentPath_ReturnsValidationError() + { + var cmd = new DataPackageCleanupCliCommand + { + Data = Path.Combine(Path.GetTempPath(), "txc-tests-cleanup-missing-" + Guid.NewGuid().ToString("N")), + Yes = true, + }; + + var exit = await cmd.RunAsync(); + + Assert.Equal(ExitValidationError, exit); + } + + [Fact] + public async Task RunAsync_BadBatchSize_ReturnsValidationError() + { + var tempFolder = CreateValidPackageFolder(); + try + { + var cmd = new DataPackageCleanupCliCommand + { + Data = tempFolder, + Yes = true, + BatchSize = 0, + }; + + var exit = await cmd.RunAsync(); + + Assert.Equal(ExitValidationError, exit); + } + finally + { + Directory.Delete(tempFolder, recursive: true); + } + } + + [Fact] + public async Task RunAsync_BadConnectionCount_ReturnsValidationError() + { + var tempFolder = CreateValidPackageFolder(); + try + { + var cmd = new DataPackageCleanupCliCommand + { + Data = tempFolder, + Yes = true, + ConnectionCount = 0, + }; + + var exit = await cmd.RunAsync(); + + Assert.Equal(ExitValidationError, exit); + } + finally + { + Directory.Delete(tempFolder, recursive: true); + } + } + + [Fact] + public async Task RunAsync_UnknownMissingAction_ReturnsValidationError() + { + var tempFolder = CreateValidPackageFolder(); + try + { + var cmd = new DataPackageCleanupCliCommand + { + Data = tempFolder, + Yes = true, + MissingAction = "magic", + }; + + var exit = await cmd.RunAsync(); + + Assert.Equal(ExitValidationError, exit); + } + finally + { + Directory.Delete(tempFolder, recursive: true); + } + } + + [Theory] + [InlineData("by-natural-key", DataPackageCleanupMissingAction.ByNaturalKey)] + [InlineData("NATURAL-KEY", DataPackageCleanupMissingAction.ByNaturalKey)] + [InlineData("skip", DataPackageCleanupMissingAction.Skip)] + [InlineData("Skip", DataPackageCleanupMissingAction.Skip)] + [InlineData("fail", DataPackageCleanupMissingAction.Fail)] + [InlineData(null, DataPackageCleanupMissingAction.ByNaturalKey)] + public void TryParseMissingAction_Accepts(string? value, DataPackageCleanupMissingAction expected) + { + var ok = DataPackageCleanupCliCommand.TryParseMissingAction(value, out var action); + Assert.True(ok); + Assert.Equal(expected, action); + } + + [Fact] + public void TryParseMissingAction_RejectsUnknownValues() + { + var ok = DataPackageCleanupCliCommand.TryParseMissingAction("???", out _); + Assert.False(ok); + } + + [Fact] + public void BuildCleanupOrder_ReversesImportOrderAndAppendsExtraEntities() + { + var contents = new DataPackageContents( + EntityImportOrder: new[] { "businessunit", "account", "contact" }, + Schemas: new Dictionary(), + Records: new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + ["account"] = Array.Empty(), + ["contact"] = Array.Empty(), + ["lead"] = Array.Empty(), + }, + M2mAssociations: Array.Empty()); + + var order = DataverseDataPackageService.BuildCleanupOrder(contents); + + Assert.Equal(new[] { "contact", "account", "businessunit", "lead" }, order); + } + + private static string CreateValidPackageFolder() + { + var folder = Path.Combine(Path.GetTempPath(), "txc-tests-cleanup-pkg-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "data_schema.xml"), ""); + File.WriteAllText(Path.Combine(folder, "data.xml"), ""); + return folder; + } +} diff --git a/tests/TALXIS.CLI.Tests/Dataverse/DataPackageReaderTests.cs b/tests/TALXIS.CLI.Tests/Dataverse/DataPackageReaderTests.cs new file mode 100644 index 00000000..e0441c45 --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Dataverse/DataPackageReaderTests.cs @@ -0,0 +1,172 @@ +using System.IO; +using System.IO.Compression; +using TALXIS.CLI.Platform.Dataverse.Application.Sdk; +using Xunit; + +namespace TALXIS.CLI.Tests.Dataverse; + +/// +/// Pure-parser tests for . Uses tiny synthetic +/// data.xml + data_schema.xml fixtures so the tests don't depend on a live +/// environment. +/// +public class DataPackageReaderTests : IDisposable +{ + private readonly string _tempRoot; + + public DataPackageReaderTests() + { + _tempRoot = Path.Combine(Path.GetTempPath(), "txc-tests", "data-package-reader-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempRoot); + } + + public void Dispose() + { + try { Directory.Delete(_tempRoot, recursive: true); } + catch { /* best-effort cleanup */ } + } + + [Fact] + public void Load_FromFolder_ParsesEntityOrderAndSchemaAndRecords() + { + var folder = WriteFixture("pkg-folder"); + + var contents = DataPackageReader.Load(folder); + + Assert.Equal(new[] { "account", "contact" }, contents.EntityImportOrder); + + Assert.True(contents.Schemas.ContainsKey("account")); + var accountSchema = contents.Schemas["account"]; + Assert.Equal("accountid", accountSchema.PrimaryIdField); + Assert.Equal("name", accountSchema.PrimaryNameField); + Assert.Contains("name", accountSchema.NaturalKeyFields); + Assert.Contains("accountnumber", accountSchema.NaturalKeyFields); + + Assert.True(contents.Records.ContainsKey("account")); + Assert.Equal(2, contents.Records["account"].Count); + Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), contents.Records["account"][0].Id); + Assert.Equal("Contoso", contents.Records["account"][0].Fields["name"]); + } + + [Fact] + public void Load_FromZip_ParsesSameAsFolder() + { + var folder = WriteFixture("pkg-zip-source"); + var zipPath = Path.Combine(_tempRoot, "package.zip"); + ZipFile.CreateFromDirectory(folder, zipPath); + + var contents = DataPackageReader.Load(zipPath); + + Assert.Equal(new[] { "account", "contact" }, contents.EntityImportOrder); + Assert.Equal(2, contents.Records["account"].Count); + } + + [Fact] + public void Load_FolderMissingFiles_Throws() + { + var folder = Path.Combine(_tempRoot, "empty"); + Directory.CreateDirectory(folder); + + Assert.Throws(() => DataPackageReader.Load(folder)); + } + + [Fact] + public void Load_NonexistentPath_Throws() + { + Assert.Throws(() => DataPackageReader.Load(Path.Combine(_tempRoot, "nope"))); + } + + [Fact] + public void Load_M2mAssociations_AreResolvedAgainstSchema() + { + var folder = WriteFixture("pkg-m2m"); + var contents = DataPackageReader.Load(folder); + + Assert.Single(contents.M2mAssociations); + var assoc = contents.M2mAssociations[0]; + Assert.Equal("account_lead_association", assoc.RelationshipName); + Assert.Equal("account", assoc.SourceEntity); + Assert.Equal("lead", assoc.TargetEntity); + Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), assoc.SourceId); + Assert.Equal(Guid.Parse("22222222-2222-2222-2222-222222222222"), assoc.TargetId); + } + + private string WriteFixture(string folderName) + { + var folder = Path.Combine(_tempRoot, folderName); + Directory.CreateDirectory(folder); + + File.WriteAllText(Path.Combine(folder, "data_schema.xml"), SchemaXml); + File.WriteAllText(Path.Combine(folder, "data.xml"), DataXml); + + return folder; + } + + private const string SchemaXml = """ + + + + account + contact + + + + + + + + + + + + + + + + + + +"""; + + private const string DataXml = """ + + + + + + + + + + + + + + + + + + + 22222222-2222-2222-2222-222222222222 + + + + + + + + + + + + + + +"""; +}