diff --git a/e2e/cluster/delete_edge_cases.go b/e2e/cluster/delete_edge_cases.go new file mode 100644 index 0000000..6c4f869 --- /dev/null +++ b/e2e/cluster/delete_edge_cases.go @@ -0,0 +1,353 @@ +package cluster + +import ( + "context" + "errors" + "net/http" + "sync" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels" +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] Re-DELETE Idempotency and API Boundary Tests", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should handle re-DELETE idempotently without changing deleted_time or generation", + ginkgo.Label(labels.Disruptive), + func(ctx context.Context) { + ginkgo.By("pausing sentinel to prevent hard-delete between DELETE calls") + sentinelDeployment, err := h.GetDeploymentName(ctx, h.Cfg.Namespace, helper.SentinelClustersRelease) + Expect(err).NotTo(HaveOccurred(), "failed to find sentinel-clusters deployment") + err = h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 0) + Expect(err).NotTo(HaveOccurred(), "failed to scale sentinel to 0") + ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.By("restoring sentinel-clusters to 1 replica") + if err := h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 1); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to restore sentinel: %v\n", err) + } + }) + + ginkgo.By("sending first DELETE request") + firstDelete, err := h.Client.DeleteCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred(), "first DELETE should succeed with 202") + Expect(firstDelete.DeletedTime).NotTo(BeNil(), "first DELETE should set deleted_time") + originalDeletedTime := *firstDelete.DeletedTime + originalGeneration := firstDelete.Generation + + ginkgo.By("sending second DELETE request") + secondDelete, err := h.Client.DeleteCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred(), "second DELETE should succeed with 202") + Expect(secondDelete.DeletedTime).NotTo(BeNil(), "second DELETE should still have deleted_time") + Expect(*secondDelete.DeletedTime).To(Equal(originalDeletedTime), "deleted_time should not change on re-DELETE") + Expect(secondDelete.Generation).To(Equal(originalGeneration), "generation should not increment on re-DELETE") + }) + + ginkgo.It("should return 409 Conflict when creating nodepool under soft-deleted cluster", + ginkgo.Label(labels.Negative), + func(ctx context.Context) { + ginkgo.By("soft-deleting the cluster") + deletedCluster, err := h.Client.DeleteCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred(), "DELETE should succeed with 202") + Expect(deletedCluster.DeletedTime).NotTo(BeNil()) + + ginkgo.By("attempting to create a nodepool under the soft-deleted cluster") + _, err = h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json")) + var httpErr *client.HTTPError + Expect(errors.As(err, &httpErr)).To(BeTrue(), "error should be HTTPError") + Expect(httpErr.StatusCode).To(Equal(http.StatusConflict), + "creating nodepool under soft-deleted cluster should return 409") + + ginkgo.By("verifying no nodepool was created") + npList, err := h.Client.ListNodePools(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + Expect(npList.Items).To(BeEmpty(), "no nodepools should exist under soft-deleted cluster") + }, + ) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] DELETE Non-Existent Cluster", + ginkgo.Label(labels.Tier1, labels.Negative), + func() { + var h *helper.Helper + + ginkgo.BeforeEach(func() { + h = helper.New() + }) + + ginkgo.It("should return 404 when deleting a non-existent cluster", func(ctx context.Context) { + ginkgo.By("sending DELETE for a non-existent cluster ID") + _, err := h.Client.DeleteCluster(ctx, "non-existent-cluster-id-12345") + var httpErr *client.HTTPError + Expect(errors.As(err, &httpErr)).To(BeTrue(), "error should be HTTPError") + Expect(httpErr.StatusCode).To(Equal(http.StatusNotFound), + "DELETE on non-existent cluster should return 404") + }) + }, +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] Concurrent Deletion", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should produce a single soft-delete record from simultaneous DELETE requests", func(ctx context.Context) { + ginkgo.By("capturing generation before deletion") + clusterBefore, err := h.Client.GetCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + genBefore := clusterBefore.Generation + + ginkgo.By("firing 5 concurrent DELETE requests") + const concurrency = 5 + type deleteResult struct { + cluster *openapi.Cluster + err error + } + results := make([]deleteResult, concurrency) + var wg sync.WaitGroup + wg.Add(concurrency) + for i := range concurrency { + go func(idx int) { + defer wg.Done() + defer ginkgo.GinkgoRecover() + c, e := h.Client.DeleteCluster(ctx, clusterID) + results[idx] = deleteResult{cluster: c, err: e} + }(i) + } + wg.Wait() + + ginkgo.By("verifying all requests succeeded with consistent state") + for i, r := range results { + Expect(r.err).NotTo(HaveOccurred(), "DELETE request %d should succeed", i) + Expect(r.cluster.DeletedTime).NotTo(BeNil(), "DELETE request %d should have deleted_time", i) + } + + // All responses should carry identical deleted_time and generation + referenceTime := *results[0].cluster.DeletedTime + referenceGen := results[0].cluster.Generation + for i := 1; i < concurrency; i++ { + Expect(*results[i].cluster.DeletedTime).To(Equal(referenceTime), + "all DELETE responses should have the same deleted_time") + Expect(results[i].cluster.Generation).To(Equal(referenceGen), + "all DELETE responses should have the same generation") + } + + ginkgo.By("verifying generation incremented exactly once") + Expect(referenceGen).To(Equal(genBefore+1), + "generation should increment by exactly 1, not by the number of concurrent requests") + + ginkgo.By("verifying cluster completes deletion lifecycle") + Eventually(h.PollClusterHTTPStatus(ctx, clusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(Equal(http.StatusNotFound)) + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] DELETE During Update Reconciliation", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled at generation 1") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil()) + clusterID = *cluster.Id + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should complete deletion when DELETE is sent during update reconciliation", func(ctx context.Context) { + ginkgo.By("sending PATCH to trigger generation 2 (do NOT wait for reconciliation)") + patchedCluster, err := h.Client.PatchCluster(ctx, clusterID, openapi.ClusterPatchRequest{ + Spec: &openapi.ClusterSpec{"trigger-update": "true"}, + }) + Expect(err).NotTo(HaveOccurred(), "PATCH should succeed") + Expect(patchedCluster.Generation).To(Equal(int32(2))) + + ginkgo.By("immediately sending DELETE before update reconciliation completes") + deletedCluster, err := h.Client.DeleteCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred(), "DELETE should succeed with 202") + Expect(deletedCluster.DeletedTime).NotTo(BeNil()) + Expect(deletedCluster.Generation).To(Equal(int32(3)), + "generation should be 3: create(1) + PATCH(2) + DELETE(3)") + + ginkgo.By("verifying cluster is hard-deleted") + Eventually(h.PollClusterHTTPStatus(ctx, clusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(Equal(http.StatusNotFound)) + + ginkgo.By("verifying downstream K8s namespace is cleaned up") + Eventually(h.PollNamespacesByPrefix(ctx, clusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(BeEmpty()) + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] Recreate Cluster After Hard-Delete", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var firstClusterID string + var secondClusterID string + var originalCluster *openapi.Cluster + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating first cluster and waiting for Reconciled") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil()) + firstClusterID = *cluster.Id + originalCluster = cluster + + Eventually(h.PollCluster(ctx, firstClusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should create a new cluster with the same name after the original is hard-deleted", func(ctx context.Context) { + ginkgo.By("deleting the first cluster and waiting for hard-delete") + _, err := h.Client.DeleteCluster(ctx, firstClusterID) + Expect(err).NotTo(HaveOccurred()) + + Eventually(h.PollClusterHTTPStatus(ctx, firstClusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(Equal(http.StatusNotFound)) + + ginkgo.By("waiting for namespace cleanup from first cluster") + Eventually(h.PollNamespacesByPrefix(ctx, firstClusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(BeEmpty()) + + ginkgo.By("creating a new cluster with the same name") + kind := "Cluster" + newCluster, err := h.Client.CreateCluster(ctx, openapi.ClusterCreateRequest{ + Kind: &kind, + Name: originalCluster.Name, + Labels: originalCluster.Labels, + Spec: originalCluster.Spec, + }) + Expect(err).NotTo(HaveOccurred(), "creating cluster with reused name should succeed") + Expect(newCluster.Id).NotTo(BeNil()) + secondClusterID = *newCluster.Id + + Expect(secondClusterID).NotTo(Equal(firstClusterID), + "new cluster should have a different ID than the deleted one") + Expect(newCluster.Generation).To(Equal(int32(1)), + "new cluster should start at generation 1") + + ginkgo.By("waiting for the new cluster to reach Reconciled") + Eventually(h.PollCluster(ctx, secondClusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + ginkgo.By("verifying the old cluster is still gone") + _, err = h.Client.GetCluster(ctx, firstClusterID) + var httpErr *client.HTTPError + Expect(errors.As(err, &httpErr)).To(BeTrue()) + Expect(httpErr.StatusCode).To(Equal(http.StatusNotFound), + "old cluster should remain 404 after recreate") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil { + return + } + for _, id := range []string{firstClusterID, secondClusterID} { + if id == "" { + continue + } + ginkgo.By("cleaning up cluster " + id) + if cluster, err := h.Client.GetCluster(ctx, id); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, id); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", id, err) + } + } + if err := h.CleanupTestCluster(ctx, id); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", id, err) + } + } + }) + }, +) diff --git a/e2e/cluster/delete_external.go b/e2e/cluster/delete_external.go new file mode 100644 index 0000000..926d23d --- /dev/null +++ b/e2e/cluster/delete_external.go @@ -0,0 +1,93 @@ +package cluster + +import ( + "context" + "errors" + "net/http" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels" +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] External K8s Resource Deletion", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + ginkgo.By("confirming managed K8s namespaces exist") + namespaces, err := h.K8sClient.FindNamespacesByPrefix(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + Expect(namespaces).NotTo(BeEmpty(), "managed namespaces should exist after Reconciled") + }) + + ginkgo.It("should treat externally-deleted K8s resources as finalized and complete hard-delete", func(ctx context.Context) { + ginkgo.By("externally deleting all managed K8s namespaces (bypass the API)") + namespaces, err := h.K8sClient.FindNamespacesByPrefix(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + + for _, ns := range namespaces { + err := h.K8sClient.DeleteNamespaceAndWait(ctx, ns) + Expect(err).NotTo(HaveOccurred(), "failed to delete namespace %s", ns) + } + + ginkgo.By("verifying all namespaces are gone") + Eventually(h.PollNamespacesByPrefix(ctx, clusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(BeEmpty()) + + ginkgo.By("sending DELETE through the API") + deletedCluster, err := h.Client.DeleteCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + Expect(deletedCluster.DeletedTime).NotTo(BeNil()) + + ginkgo.By("verifying adapters report Finalized=True with Health=True") + Eventually(func(g Gomega) { + var httpErr *client.HTTPError + statuses, err := h.Client.GetClusterStatuses(ctx, clusterID) + if errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound { + return + } + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(statuses).To(helper.HaveAllAdaptersWithCondition( + h.Cfg.Adapters.Cluster, client.ConditionTypeFinalized, openapi.AdapterConditionStatusTrue)) + g.Expect(statuses).To(helper.HaveAllAdaptersWithCondition( + h.Cfg.Adapters.Cluster, client.ConditionTypeHealth, openapi.AdapterConditionStatusTrue)) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("verifying cluster is hard-deleted") + Eventually(h.PollClusterHTTPStatus(ctx, clusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(Equal(http.StatusNotFound)) + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) diff --git a/e2e/cluster/delete_visibility.go b/e2e/cluster/delete_visibility.go new file mode 100644 index 0000000..f50917e --- /dev/null +++ b/e2e/cluster/delete_visibility.go @@ -0,0 +1,190 @@ +package cluster + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels" +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] Soft-Deleted Cluster Visibility", + ginkgo.Label(labels.Tier1, labels.Disruptive), + ginkgo.Serial, + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should remain visible via GET and LIST before hard-delete", func(ctx context.Context) { + ginkgo.By("pausing sentinel to freeze reconciliation before soft-delete") + sentinelDeployment, err := h.GetDeploymentName(ctx, h.Cfg.Namespace, helper.SentinelClustersRelease) + Expect(err).NotTo(HaveOccurred(), "failed to find sentinel-clusters deployment") + err = h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 0) + Expect(err).NotTo(HaveOccurred(), "failed to scale sentinel to 0") + ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.By("restoring sentinel-clusters to 1 replica") + if err := h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 1); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to restore sentinel: %v\n", err) + } + }) + + ginkgo.By("soft-deleting the cluster") + deletedCluster, err := h.Client.DeleteCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + Expect(deletedCluster.DeletedTime).NotTo(BeNil()) + + ginkgo.By("verifying GET returns the soft-deleted cluster with deleted_time") + Eventually(func(g Gomega) { + cluster, err := h.Client.GetCluster(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "GET should return 200, not 404") + g.Expect(cluster.DeletedTime).NotTo(BeNil(), "cluster should have deleted_time set") + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("verifying LIST includes the soft-deleted cluster") + Eventually(func(g Gomega) { + clusterList, err := h.Client.ListClusters(ctx) + g.Expect(err).NotTo(HaveOccurred()) + + found := false + for _, c := range clusterList.Items { + if c.Id != nil && *c.Id == clusterID { + g.Expect(c.DeletedTime).NotTo(BeNil(), "cluster in LIST should have deleted_time") + found = true + } + } + g.Expect(found).To(BeTrue(), "soft-deleted cluster should appear in LIST") + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: cluster][delete] LIST Shows Active and Soft-Deleted Clusters", + ginkgo.Label(labels.Tier1, labels.Disruptive), + ginkgo.Serial, + func() { + var h *helper.Helper + var activeClusterID string + var deletedClusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating two clusters and waiting for Reconciled") + var err error + activeClusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create active cluster") + + deletedClusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster to delete") + + Eventually(h.PollCluster(ctx, activeClusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + Eventually(h.PollCluster(ctx, deletedClusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should return both active and soft-deleted clusters in LIST", func(ctx context.Context) { + ginkgo.By("pausing sentinel to freeze reconciliation before soft-delete") + sentinelDeployment, err := h.GetDeploymentName(ctx, h.Cfg.Namespace, helper.SentinelClustersRelease) + Expect(err).NotTo(HaveOccurred(), "failed to find sentinel-clusters deployment") + err = h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 0) + Expect(err).NotTo(HaveOccurred(), "failed to scale sentinel to 0") + ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.By("restoring sentinel-clusters to 1 replica") + if err := h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 1); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to restore sentinel: %v\n", err) + } + }) + + ginkgo.By("soft-deleting one cluster") + _, err = h.Client.DeleteCluster(ctx, deletedClusterID) + Expect(err).NotTo(HaveOccurred()) + + ginkgo.By("verifying LIST returns both clusters simultaneously") + Eventually(func(g Gomega) { + clusterList, err := h.Client.ListClusters(ctx) + g.Expect(err).NotTo(HaveOccurred()) + + var foundActive, foundDeleted bool + for _, c := range clusterList.Items { + if c.Id == nil { + continue + } + if *c.Id == activeClusterID { + g.Expect(c.DeletedTime).To(BeNil(), "active cluster should not have deleted_time") + foundActive = true + } + if *c.Id == deletedClusterID { + g.Expect(c.DeletedTime).NotTo(BeNil(), "deleted cluster should have deleted_time") + foundDeleted = true + } + } + g.Expect(foundActive).To(BeTrue(), "active cluster should appear in LIST") + g.Expect(foundDeleted).To(BeTrue(), "soft-deleted cluster should appear in LIST") + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("verifying GET returns correct state for each cluster") + activeCluster, err := h.Client.GetCluster(ctx, activeClusterID) + Expect(err).NotTo(HaveOccurred()) + Expect(activeCluster.DeletedTime).To(BeNil()) + + Eventually(func(g Gomega) { + deletedCluster, err := h.Client.GetCluster(ctx, deletedClusterID) + g.Expect(err).NotTo(HaveOccurred(), "GET on soft-deleted cluster should return 200") + g.Expect(deletedCluster.DeletedTime).NotTo(BeNil()) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil { + return + } + for _, id := range []string{activeClusterID, deletedClusterID} { + if id == "" { + continue + } + ginkgo.By("cleaning up cluster " + id) + if cluster, err := h.Client.GetCluster(ctx, id); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, id); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", id, err) + } + } + if err := h.CleanupTestCluster(ctx, id); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", id, err) + } + } + }) + }, +) diff --git a/e2e/cluster/update_edge_cases.go b/e2e/cluster/update_edge_cases.go new file mode 100644 index 0000000..39c9403 --- /dev/null +++ b/e2e/cluster/update_edge_cases.go @@ -0,0 +1,206 @@ +package cluster + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels" +) + +var _ = ginkgo.Describe("[Suite: cluster][update] Rapid Update Coalescing", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled at generation 1") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil()) + clusterID = *cluster.Id + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should coalesce multiple rapid updates and reconcile to the latest generation", func(ctx context.Context) { + ginkgo.By("sending three PATCH requests in rapid succession") + patch1, err := h.Client.PatchCluster(ctx, clusterID, openapi.ClusterPatchRequest{ + Spec: &openapi.ClusterSpec{"update": "first"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(patch1.Generation).To(Equal(int32(2))) + + patch2, err := h.Client.PatchCluster(ctx, clusterID, openapi.ClusterPatchRequest{ + Spec: &openapi.ClusterSpec{"update": "second"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(patch2.Generation).To(Equal(int32(3))) + + patch3, err := h.Client.PatchCluster(ctx, clusterID, openapi.ClusterPatchRequest{ + Spec: &openapi.ClusterSpec{"update": "third"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(patch3.Generation).To(Equal(int32(4))) + + ginkgo.By("waiting for all adapters to reconcile at the final generation") + Eventually(h.PollClusterAdapterStatuses(ctx, clusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(helper.HaveAllAdaptersAtGeneration(h.Cfg.Adapters.Cluster, int32(4))) + + ginkgo.By("verifying cluster reaches Reconciled=True at final generation") + Eventually(func(g Gomega) { + finalCluster, err := h.Client.GetCluster(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(finalCluster.Generation).To(Equal(int32(4))) + + found := false + for _, cond := range finalCluster.Status.Conditions { + if cond.Type == client.ConditionTypeReconciled && cond.Status == openapi.ResourceConditionStatusTrue { + found = true + g.Expect(cond.ObservedGeneration).To(Equal(int32(4))) + } + } + g.Expect(found).To(BeTrue(), "cluster should have Reconciled=True") + }, h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval).Should(Succeed()) + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: cluster][update] Labels-Only PATCH", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled at generation 1") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil()) + clusterID = *cluster.Id + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should bump generation and trigger reconciliation from a labels-only PATCH", func(ctx context.Context) { + ginkgo.By("capturing spec before labels-only PATCH") + clusterBefore, err := h.Client.GetCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + specBefore := clusterBefore.Spec + + ginkgo.By("sending labels-only PATCH (preserving existing labels)") + newLabels := make(map[string]string) + if clusterBefore.Labels != nil { + for k, v := range *clusterBefore.Labels { + newLabels[k] = v + } + } + newLabels["env"] = "staging" + newLabels["team"] = "fleet-management" + patchedCluster, err := h.Client.PatchCluster(ctx, clusterID, openapi.ClusterPatchRequest{ + Labels: &newLabels, + }) + Expect(err).NotTo(HaveOccurred(), "labels-only PATCH should succeed") + Expect(patchedCluster.Generation).To(Equal(int32(2)), + "generation should increment after labels-only PATCH") + + ginkgo.By("waiting for all adapters to reconcile at generation 2") + Eventually(h.PollClusterAdapterStatuses(ctx, clusterID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(helper.HaveAllAdaptersAtGeneration(h.Cfg.Adapters.Cluster, int32(2))) + + ginkgo.By("verifying cluster reaches Reconciled=True and Available=True") + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeAvailable, openapi.ResourceConditionStatusTrue)) + + finalCluster, err := h.Client.GetCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + Expect(finalCluster.Labels).NotTo(BeNil(), "cluster should have labels") + Expect((*finalCluster.Labels)["env"]).To(Equal("staging")) + Expect((*finalCluster.Labels)["team"]).To(Equal("fleet-management")) + Expect(finalCluster.Spec).To(Equal(specBefore), + "spec should be unchanged after labels-only PATCH") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: cluster][update] No-Op PATCH", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled at generation 1") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil()) + clusterID = *cluster.Id + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should not increment generation when PATCHing with identical spec", func(ctx context.Context) { + ginkgo.By("PATCHing with a spec change to bump generation to 2") + spec := openapi.ClusterSpec{"update-trigger": "gen2"} + changed, err := h.Client.PatchCluster(ctx, clusterID, openapi.ClusterPatchRequest{ + Spec: &spec, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(changed.Generation).To(Equal(int32(2)), "generation should bump after spec change") + + ginkgo.By("replaying the same spec via PATCH") + replayed, err := h.Client.PatchCluster(ctx, clusterID, openapi.ClusterPatchRequest{ + Spec: &spec, + }) + Expect(err).NotTo(HaveOccurred(), "no-op PATCH should succeed") + Expect(replayed.Generation).To(Equal(int32(2)), + "generation should not increment for identical spec PATCH") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) diff --git a/e2e/nodepool/delete_edge_cases.go b/e2e/nodepool/delete_edge_cases.go new file mode 100644 index 0000000..eb64686 --- /dev/null +++ b/e2e/nodepool/delete_edge_cases.go @@ -0,0 +1,213 @@ +package nodepool + +import ( + "context" + "errors" + "net/http" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels" +) + +var _ = ginkgo.Describe("[Suite: nodepool][delete] Sibling Nodepool Isolation During Deletion", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + var nodepoolID1 string + var nodepoolID2 string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + ginkgo.By("creating two nodepools") + np1, err := h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create first nodepool") + Expect(np1.Id).NotTo(BeNil()) + nodepoolID1 = *np1.Id + + np2, err := h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create second nodepool") + Expect(np2.Id).NotTo(BeNil()) + nodepoolID2 = *np2.Id + + ginkgo.By("waiting for both nodepools to reach Reconciled") + Eventually(h.PollNodePool(ctx, clusterID, nodepoolID1), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + Eventually(h.PollNodePool(ctx, clusterID, nodepoolID2), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should not affect sibling nodepool when one is deleted", func(ctx context.Context) { + ginkgo.By("deleting the first nodepool") + deletedNP, err := h.Client.DeleteNodePool(ctx, clusterID, nodepoolID1) + Expect(err).NotTo(HaveOccurred(), "DELETE should succeed with 202") + Expect(deletedNP.DeletedTime).NotTo(BeNil()) + + ginkgo.By("waiting for the deleted nodepool to be hard-deleted") + Eventually(h.PollNodePoolHTTPStatus(ctx, clusterID, nodepoolID1), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(Equal(http.StatusNotFound)) + + ginkgo.By("verifying sibling nodepool is unaffected") + siblingNP, err := h.Client.GetNodePool(ctx, clusterID, nodepoolID2) + Expect(err).NotTo(HaveOccurred(), "sibling nodepool should still be accessible") + Expect(siblingNP.DeletedTime).To(BeNil(), "sibling nodepool should not have deleted_time") + + hasReconciled := h.HasResourceCondition(siblingNP.Status.Conditions, client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue) + Expect(hasReconciled).To(BeTrue(), "sibling nodepool should remain Reconciled=True") + + ginkgo.By("verifying sibling nodepool adapter statuses are intact") + Eventually(h.PollNodePoolAdapterStatuses(ctx, clusterID, nodepoolID2), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(helper.HaveAllAdaptersWithCondition( + h.Cfg.Adapters.NodePool, client.ConditionTypeApplied, openapi.AdapterConditionStatusTrue)) + + ginkgo.By("verifying parent cluster is unaffected") + parentCluster, err := h.Client.GetCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred(), "parent cluster should still exist") + Expect(parentCluster.DeletedTime).To(BeNil(), "parent cluster should not have deleted_time") + + hasParentReconciled := h.HasResourceCondition(parentCluster.Status.Conditions, client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue) + Expect(hasParentReconciled).To(BeTrue(), "parent cluster should remain Reconciled=True") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: nodepool][delete] Re-DELETE Idempotency", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + var nodepoolID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + ginkgo.By("creating nodepool and waiting for Reconciled") + np, err := h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create nodepool") + Expect(np.Id).NotTo(BeNil()) + nodepoolID = *np.Id + + Eventually(h.PollNodePool(ctx, clusterID, nodepoolID), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should handle re-DELETE on nodepool idempotently without changing deleted_time or generation", + ginkgo.Label(labels.Disruptive), + func(ctx context.Context) { + ginkgo.By("pausing sentinel to prevent hard-delete between DELETE calls") + sentinelDeployment, err := h.GetDeploymentName(ctx, h.Cfg.Namespace, helper.SentinelNodePoolsRelease) + Expect(err).NotTo(HaveOccurred(), "failed to find sentinel-nodepools deployment") + err = h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 0) + Expect(err).NotTo(HaveOccurred(), "failed to scale sentinel-nodepools to 0") + ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.By("restoring sentinel-nodepools to 1 replica") + if err := h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 1); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to restore sentinel-nodepools: %v\n", err) + } + }) + + ginkgo.By("sending first DELETE request") + firstDelete, err := h.Client.DeleteNodePool(ctx, clusterID, nodepoolID) + Expect(err).NotTo(HaveOccurred(), "first DELETE should succeed with 202") + Expect(firstDelete.DeletedTime).NotTo(BeNil()) + originalDeletedTime := *firstDelete.DeletedTime + originalGeneration := firstDelete.Generation + + ginkgo.By("sending second DELETE request") + secondDelete, err := h.Client.DeleteNodePool(ctx, clusterID, nodepoolID) + Expect(err).NotTo(HaveOccurred(), "second DELETE should succeed with 202") + Expect(secondDelete.DeletedTime).NotTo(BeNil()) + Expect(*secondDelete.DeletedTime).To(Equal(originalDeletedTime), "deleted_time should not change on re-DELETE") + Expect(secondDelete.Generation).To(Equal(originalGeneration), "generation should not increment on re-DELETE") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) + +var _ = ginkgo.Describe("[Suite: nodepool][delete] DELETE Non-Existent Nodepool", + ginkgo.Label(labels.Tier1, labels.Negative), + func() { + var h *helper.Helper + var clusterID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster for valid cluster_id path parameter") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + }) + + ginkgo.It("should return 404 when deleting a non-existent nodepool", func(ctx context.Context) { + ginkgo.By("sending DELETE for a non-existent nodepool ID") + _, err := h.Client.DeleteNodePool(ctx, clusterID, "non-existent-nodepool-id-12345") + var httpErr *client.HTTPError + Expect(errors.As(err, &httpErr)).To(BeTrue(), "error should be HTTPError") + Expect(httpErr.StatusCode).To(Equal(http.StatusNotFound), + "DELETE on non-existent nodepool should return 404") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) diff --git a/e2e/nodepool/delete_visibility.go b/e2e/nodepool/delete_visibility.go new file mode 100644 index 0000000..2cbf4b9 --- /dev/null +++ b/e2e/nodepool/delete_visibility.go @@ -0,0 +1,125 @@ +package nodepool + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels" +) + +var _ = ginkgo.Describe("[Suite: nodepool][delete] Soft-Deleted Nodepool Visibility", + ginkgo.Label(labels.Tier1, labels.Disruptive), + ginkgo.Serial, + func() { + var h *helper.Helper + var clusterID string + var activeNPID string + var deletedNPID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + ginkgo.By("creating two nodepools and waiting for Reconciled") + np1, err := h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(np1.Id).NotTo(BeNil()) + activeNPID = *np1.Id + + np2, err := h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(np2.Id).NotTo(BeNil()) + deletedNPID = *np2.Id + + Eventually(h.PollNodePool(ctx, clusterID, activeNPID), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + Eventually(h.PollNodePool(ctx, clusterID, deletedNPID), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should remain visible via GET and LIST before hard-delete", func(ctx context.Context) { + ginkgo.By("pausing sentinel-nodepools to freeze reconciliation before soft-delete") + sentinelDeployment, err := h.GetDeploymentName(ctx, h.Cfg.Namespace, helper.SentinelNodePoolsRelease) + Expect(err).NotTo(HaveOccurred(), "failed to find sentinel-nodepools deployment") + err = h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 0) + Expect(err).NotTo(HaveOccurred(), "failed to scale sentinel-nodepools to 0") + ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.By("restoring sentinel-nodepools to 1 replica") + if err := h.ScaleDeployment(ctx, h.Cfg.Namespace, sentinelDeployment, 1); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to restore sentinel-nodepools: %v\n", err) + } + }) + + ginkgo.By("soft-deleting one nodepool") + deletedNP, err := h.Client.DeleteNodePool(ctx, clusterID, deletedNPID) + Expect(err).NotTo(HaveOccurred()) + Expect(deletedNP.DeletedTime).NotTo(BeNil()) + + ginkgo.By("verifying GET returns the soft-deleted nodepool with deleted_time") + Eventually(func(g Gomega) { + np, err := h.Client.GetNodePool(ctx, clusterID, deletedNPID) + g.Expect(err).NotTo(HaveOccurred(), "GET should return 200, not 404") + g.Expect(np.DeletedTime).NotTo(BeNil(), "nodepool should have deleted_time set") + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("verifying LIST includes both active and soft-deleted nodepools") + Eventually(func(g Gomega) { + npList, err := h.Client.ListNodePools(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred()) + + var foundActive, foundDeleted bool + for _, np := range npList.Items { + if np.Id == nil { + continue + } + if *np.Id == activeNPID { + g.Expect(np.DeletedTime).To(BeNil(), "active nodepool should not have deleted_time") + foundActive = true + } + if *np.Id == deletedNPID { + g.Expect(np.DeletedTime).NotTo(BeNil(), "deleted nodepool should have deleted_time") + foundDeleted = true + } + } + g.Expect(foundActive).To(BeTrue(), "active nodepool should appear in LIST") + g.Expect(foundDeleted).To(BeTrue(), "soft-deleted nodepool should appear in LIST") + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("verifying active nodepool is unaffected") + activeNP, err := h.Client.GetNodePool(ctx, clusterID, activeNPID) + Expect(err).NotTo(HaveOccurred()) + Expect(activeNP.DeletedTime).To(BeNil()) + + hasReconciled := h.HasResourceCondition(activeNP.Status.Conditions, client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue) + Expect(hasReconciled).To(BeTrue(), "active nodepool should remain Reconciled=True") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if cluster, err := h.Client.GetCluster(ctx, clusterID); err == nil && cluster.DeletedTime == nil { + if _, err := h.Client.DeleteCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: API delete failed for cluster %s: %v\n", clusterID, err) + } + } + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) diff --git a/e2e/nodepool/update_edge_cases.go b/e2e/nodepool/update_edge_cases.go new file mode 100644 index 0000000..7ac0db3 --- /dev/null +++ b/e2e/nodepool/update_edge_cases.go @@ -0,0 +1,105 @@ +package nodepool + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels" +) + +var _ = ginkgo.Describe("[Suite: nodepool][update] Labels-Only PATCH", + ginkgo.Label(labels.Tier1), + func() { + var h *helper.Helper + var clusterID string + var nodepoolID string + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + ginkgo.By("creating cluster and waiting for Reconciled") + var err error + clusterID, err = h.GetTestCluster(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + + Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + + ginkgo.By("creating nodepool and waiting for Reconciled at generation 1") + np, err := h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create nodepool") + Expect(np.Id).NotTo(BeNil()) + nodepoolID = *np.Id + + Eventually(h.PollNodePool(ctx, clusterID, nodepoolID), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + }) + + ginkgo.It("should bump generation and trigger reconciliation from a labels-only PATCH", func(ctx context.Context) { + ginkgo.By("capturing state before labels-only PATCH") + npBefore, err := h.Client.GetNodePool(ctx, clusterID, nodepoolID) + Expect(err).NotTo(HaveOccurred()) + specBefore := npBefore.Spec + parentBefore, err := h.Client.GetCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + + ginkgo.By("sending labels-only PATCH to nodepool (preserving existing labels)") + newLabels := make(map[string]string) + if npBefore.Labels != nil { + for k, v := range *npBefore.Labels { + newLabels[k] = v + } + } + newLabels["env"] = "staging" + newLabels["pool-type"] = "gpu" + patchedNP, err := h.Client.PatchNodePool(ctx, clusterID, nodepoolID, openapi.NodePoolPatchRequest{ + Labels: &newLabels, + }) + Expect(err).NotTo(HaveOccurred(), "labels-only PATCH should succeed") + Expect(patchedNP.Generation).To(Equal(int32(2)), + "generation should increment after labels-only PATCH") + + ginkgo.By("waiting for all nodepool adapters to reconcile at generation 2") + Eventually(h.PollNodePoolAdapterStatuses(ctx, clusterID, nodepoolID), h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval). + Should(helper.HaveAllAdaptersAtGeneration(h.Cfg.Adapters.NodePool, int32(2))) + + ginkgo.By("verifying nodepool reaches Reconciled=True and Available=True") + Eventually(h.PollNodePool(ctx, clusterID, nodepoolID), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue)) + Eventually(h.PollNodePool(ctx, clusterID, nodepoolID), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval). + Should(helper.HaveResourceCondition(client.ConditionTypeAvailable, openapi.ResourceConditionStatusTrue)) + + finalNP, err := h.Client.GetNodePool(ctx, clusterID, nodepoolID) + Expect(err).NotTo(HaveOccurred()) + Expect(finalNP.Labels).NotTo(BeNil(), "nodepool should have labels") + Expect((*finalNP.Labels)["env"]).To(Equal("staging")) + Expect((*finalNP.Labels)["pool-type"]).To(Equal("gpu")) + Expect(finalNP.Spec).To(Equal(specBefore), + "spec should be unchanged after labels-only PATCH") + + ginkgo.By("verifying parent cluster generation is unchanged") + parentCluster, err := h.Client.GetCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + Expect(parentCluster.Generation).To(Equal(parentBefore.Generation), + "nodepool labels PATCH should not affect cluster generation") + + hasParentReconciled := h.HasResourceCondition(parentCluster.Status.Conditions, client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue) + Expect(hasParentReconciled).To(BeTrue(), "parent cluster should remain Reconciled=True") + }) + + ginkgo.AfterEach(func(ctx context.Context) { + if h == nil || clusterID == "" { + return + } + ginkgo.By("cleaning up cluster " + clusterID) + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: cleanup failed for cluster %s: %v\n", clusterID, err) + } + }) + }, +) diff --git a/pkg/helper/constants.go b/pkg/helper/constants.go index 3e18c8b..e6e3c08 100644 --- a/pkg/helper/constants.go +++ b/pkg/helper/constants.go @@ -11,5 +11,8 @@ const ( ResourceTypeClusters = "clusters" ResourceTypeNodepools = "nodepools" + SentinelClustersRelease = "sentinel-clusters" + SentinelNodePoolsRelease = "sentinel-nodepools" + defaultGCPProjectID = "hcm-hyperfleet" ) diff --git a/test-design/testcases/delete-cluster.md b/test-design/testcases/delete-cluster.md index 442d6c0..6a2faaf 100644 --- a/test-design/testcases/delete-cluster.md +++ b/test-design/testcases/delete-cluster.md @@ -285,7 +285,7 @@ This test validates that after a cluster is soft-deleted, it remains queryable v | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-15 | @@ -393,7 +393,7 @@ This test validates that calling DELETE on a cluster that has already been soft- | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-15 | @@ -474,7 +474,7 @@ This test validates that the API rejects mutation requests (PATCH) to clusters t | **Pos/Neg** | Negative | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-28 | @@ -566,7 +566,7 @@ This test validates that creating new subresources (nodepools) under a soft-dele | **Pos/Neg** | Negative | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-16 | @@ -654,7 +654,7 @@ This test validates that sending a DELETE request for a cluster ID that does not | **Pos/Neg** | Negative | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-15 | @@ -931,7 +931,7 @@ This test validates that when multiple DELETE requests for the same cluster are | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-16 | | **Updated** | 2026-04-16 | @@ -1036,7 +1036,7 @@ This test validates the adapter-side "NotFound as success" semantics. When the m | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-16 | | **Updated** | 2026-04-28 | @@ -1152,7 +1152,7 @@ This test validates the interaction between update and delete workflows. When a | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-16 | | **Updated** | 2026-04-16 | @@ -1263,7 +1263,7 @@ This test validates that after a cluster is fully deleted (hard-deleted from the | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-16 | | **Updated** | 2026-04-16 | @@ -1387,7 +1387,7 @@ This test validates that soft-deleted clusters (with `deleted_time` set) remain | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-17 | | **Updated** | 2026-04-28 | diff --git a/test-design/testcases/delete-nodepool.md b/test-design/testcases/delete-nodepool.md index 34f26d9..c88bf53 100644 --- a/test-design/testcases/delete-nodepool.md +++ b/test-design/testcases/delete-nodepool.md @@ -143,7 +143,7 @@ This test validates isolation between sibling nodepools during deletion. When on | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-15 | @@ -248,7 +248,7 @@ This test validates that calling DELETE on a nodepool that has already been soft | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-15 | @@ -332,7 +332,7 @@ This test validates that sending a DELETE request for a nodepool ID that does no | **Pos/Neg** | Negative | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-15 | @@ -400,7 +400,7 @@ This test validates that the API rejects mutation requests (PATCH) to nodepools | **Pos/Neg** | Negative | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-28 | @@ -501,7 +501,7 @@ This test validates that after a nodepool is soft-deleted, it remains queryable | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-17 | | **Updated** | 2026-04-28 | diff --git a/test-design/testcases/update-cluster.md b/test-design/testcases/update-cluster.md index 12c8d53..805cad0 100644 --- a/test-design/testcases/update-cluster.md +++ b/test-design/testcases/update-cluster.md @@ -123,6 +123,8 @@ curl -X DELETE ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} This test validates the intermediate status transitions during update reconciliation. When a cluster spec is updated, there is a window where adapters have not yet reconciled to the new generation. During this window, `Reconciled` should be `False` (indicating stale adapter statuses relative to the new generation). To guarantee this window is observable, a dedicated crash-adapter is deployed and scaled to 0 before the PATCH. With a stuck adapter, `Reconciled` remains `False` indefinitely, allowing reliable assertion via `Consistently`. After verification, the adapter is restored and full convergence is confirmed. +> **Automation note:** Deferred — the core mechanism (Reconciled=True requires all required adapters at current generation, ADR-0008) is already validated by the Tier 2 crash-recovery test (`e2e/cluster/crash_recovery.go`), which uses the same crash-adapter infrastructure. The only behavioral delta is _stale_ adapter (reported old generation) vs _absent_ adapter (never reported), which exercises the same aggregation code path. Automating this test would add ~180 lines of Disruptive/Serial infrastructure for minimal incremental coverage. + --- | **Field** | **Value** | @@ -130,10 +132,10 @@ This test validates the intermediate status transitions during update reconcilia | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Deferred | | **Version** | Post-MVP | | **Created** | 2026-04-15 | -| **Updated** | 2026-04-28 | +| **Updated** | 2026-05-11 | --- @@ -251,7 +253,7 @@ This test validates that when multiple PATCH requests are sent in rapid successi | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-15 | | **Updated** | 2026-04-15 | @@ -355,7 +357,7 @@ This test validates that a PATCH request that only modifies `labels` (without ch | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-17 | | **Updated** | 2026-04-20 | @@ -444,7 +446,7 @@ curl -X DELETE ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} ### Description -This test validates PATCH behavior at the no-op boundary for cluster updates. It covers four deterministic cases: canonical replay of the current spec, semantically identical replay with different raw JSON formatting, explicit empty-object replacement, and repeated identical PATCHes after the replacement. The objective is to verify generation changes only when the effective spec state changes. +This test validates that a PATCH request with an identical spec does not increment the cluster's generation. The test captures the current spec, replays it via PATCH, and verifies the generation remains unchanged. --- @@ -453,7 +455,7 @@ This test validates PATCH behavior at the no-op boundary for cluster updates. It | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-28 | | **Updated** | 2026-04-28 | @@ -473,119 +475,43 @@ This test validates PATCH behavior at the no-op boundary for cluster updates. It #### Step 1: Create a cluster and wait for Reconciled state at generation 1 **Action:** -- Create a cluster and wait for Reconciled: ```bash curl -X POST ${API_URL}/api/hyperfleet/v1/clusters \ -H "Content-Type: application/json" \ -d @testdata/payloads/clusters/cluster-request.json ``` -- Capture the cluster's canonical spec into a shell variable for replay in Step 3: -```bash -CANONICAL_SPEC=$(curl -s ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} | jq -c '.spec') -``` -- Record `generation` as `{G1}` (expected: 1) **Expected Result:** -- Cluster reaches `Reconciled: True` at `generation: {G1}` -- All adapters report `observed_generation: {G1}` +- Cluster reaches `Reconciled: True` at `generation: 1` -#### Step 2: Capture baseline adapter `last_report_time` values +#### Step 2: PATCH with a spec change and verify generation increments **Action:** ```bash -curl -X GET ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id}/statuses \ - | jq '[.items[] | {adapter, observed_generation, last_report_time}]' -``` - -**Expected Result:** -- Baseline captured for comparison after Case B and again after Case D - -#### Step 3: Exercise PATCH behavior at the no-op boundary - -**Case A: Byte-identical replay of the canonical spec** - -Send the captured `CANONICAL_SPEC` back as the PATCH payload: -```bash -curl -i -X PATCH ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} \ - -H "Content-Type: application/json" \ - -d "$(jq -n --argjson spec "$CANONICAL_SPEC" '{"spec": $spec}')" -``` - -**Expected Result:** -- Response returns HTTP 200 (OK) -- `generation` equals `{G1}` (unchanged) - -**Case B: Semantic replay with different raw JSON formatting or key order** - -Send the same key-value pairs as `CANONICAL_SPEC` but with reordered keys or pretty-printed formatting: -```bash -REORDERED_SPEC=$(echo "$CANONICAL_SPEC" | jq -S '.') curl -i -X PATCH ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} \ -H "Content-Type: application/json" \ - -d "$(jq -n --argjson spec "$REORDERED_SPEC" '{"spec": $spec}')" + -d '{"spec": {"update-trigger": "gen2"}}' ``` **Expected Result:** - Response returns HTTP 200 (OK) -- `generation` still equals `{G1}` because semantic equivalence alone does not change effective spec state -- No reconciliation is triggered; raw request formatting alone does not change effective state +- `generation` equals `2` -**Case C: Explicit empty-object replacement** +#### Step 3: Replay the same spec via PATCH -Send a PATCH with `spec` set to an empty object: +**Action:** +- Send the same PATCH request as Step 2: ```bash curl -i -X PATCH ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} \ -H "Content-Type: application/json" \ - -d '{"spec": {}}' + -d '{"spec": {"update-trigger": "gen2"}}' ``` **Expected Result:** - Response returns HTTP 200 (OK) -- `generation` equals `{G1} + 1` -- Cluster `spec` is now `{}` -- This is the only case in the test that should trigger reconciliation - -**Case D: Repeat the Case C payload three more times (stability check)** - -Send the same empty-object PATCH from Case C three more times in succession: -```bash -for i in 1 2 3; do - curl -i -X PATCH ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} \ - -H "Content-Type: application/json" \ - -d '{"spec": {}}' -done -``` - -**Expected Result:** -- All three return HTTP 200 (OK) -- `generation` remains `{G1} + 1` for all three calls because no additional state change occurs after Case C - -#### Step 4: Verify reconciliation only happened for the state-changing case +- `generation` remains `2` (unchanged) -**Action:** -- After Case B, capture the current cluster and adapter status timestamps: -```bash -curl -X GET ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} -curl -X GET ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id}/statuses \ - | jq '[.items[] | {adapter, observed_generation, last_report_time}]' -``` -- After Case D, capture them again: -```bash -curl -X GET ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id} -curl -X GET ${API_URL}/api/hyperfleet/v1/clusters/{cluster_id}/statuses \ - | jq '[.items[] | {adapter, observed_generation, last_report_time}]' -``` -- If Case C bumped `generation`, poll until all adapters report the current generation before comparing final state. - -**Expected Result:** -- After Case B, `generation` still equals `{G1}` and adapter `last_report_time` values still match the Step 2 baseline -- Case C causes the only additional generation bump in the test and may update adapter `last_report_time` values -- After Case D, there are no further generation increments or additional `last_report_time` changes beyond the post-Case-C state -- Final cluster `generation` equals `{G1} + 1` -- Final cluster `spec` equals `{}` -- `observed_generation` on all adapters matches the current cluster `generation` - -#### Step 5: Cleanup resources +#### Step 4: Cleanup resources **Action:** ```bash diff --git a/test-design/testcases/update-nodepool.md b/test-design/testcases/update-nodepool.md index 4539240..e21ef12 100644 --- a/test-design/testcases/update-nodepool.md +++ b/test-design/testcases/update-nodepool.md @@ -136,7 +136,7 @@ This test validates that a PATCH request that only modifies a nodepool's `labels | **Pos/Neg** | Positive | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | Post-MVP | | **Created** | 2026-04-17 | | **Updated** | 2026-04-20 |