diff --git a/openshift/tests-extension/Makefile b/openshift/tests-extension/Makefile index e7a126ae2b..6f275b371c 100644 --- a/openshift/tests-extension/Makefile +++ b/openshift/tests-extension/Makefile @@ -181,6 +181,11 @@ build: #HELP Build the extended tests binary @mkdir -p $(TOOLS_BIN_DIR) GO_COMPLIANCE_POLICY="exempt_all" go build -ldflags "$(LDFLAGS)" -mod=vendor -o $(TOOLS_BIN_DIR)/olmv1-tests-ext ./cmd/... +.PHONY: build-local-dev +build-local-dev: #HELP Build the extended tests binary with local dev commands (for local development only) + @mkdir -p $(TOOLS_BIN_DIR) + GO_COMPLIANCE_POLICY="exempt_all" go build -tags dev -ldflags "$(LDFLAGS)" -mod=vendor -o $(TOOLS_BIN_DIR)/olmv1-tests-ext ./cmd/... + .PHONY: update-metadata update-metadata: #HELP Build and run 'update-metadata' to generate test metadata $(TOOLS_BIN_DIR)/olmv1-tests-ext update --component openshift:payload:olmv1 @@ -245,3 +250,21 @@ verify-metadata: update-metadata .PHONY: verify-images-json #HELP Verify that 'images' command outputs valid JSON verify-images-json: @./hack/verify-images-json.sh $(TOOLS_BIN_DIR)/olmv1-tests-ext + +#SECTION Local Testing (Human-Readable Output) + +.PHONY: test-local +test-local: build-local-dev #HELP Run tests locally with clean, human-readable output (usage: make test-local SUITE=olmv1/all) + @if [ -z "$(SUITE)" ]; then \ + echo "ERROR: Please specify SUITE. Example: make test-local SUITE=olmv1/all"; \ + exit 1; \ + fi + @$(TOOLS_BIN_DIR)/olmv1-tests-ext run-suite-dev "$(SUITE)" + +.PHONY: test-local-single +test-local-single: build-local-dev #HELP Run a single test with clean output (usage: make test-local-single TEST="test name") + @if [ -z "$(TEST)" ]; then \ + echo "ERROR: Please specify TEST. Example: make test-local-single TEST=\"[sig-olmv1] OLMv1 should pass\""; \ + exit 1; \ + fi + @$(TOOLS_BIN_DIR)/olmv1-tests-ext run-test-dev -n "$(TEST)" diff --git a/openshift/tests-extension/README.md b/openshift/tests-extension/README.md index ca10ebd66c..ebf3b2a227 100644 --- a/openshift/tests-extension/README.md +++ b/openshift/tests-extension/README.md @@ -33,6 +33,10 @@ opts out with a skip label like `[Skipped:Disconnected]`. | Release jobs | [amd64.ocp.releases.ci.openshift.org](https://amd64.ocp.releases.ci.openshift.org/) | Click any build to see all validation jobs run against it | | Component Readiness | [Sippy](https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main) | Test results feed here. Failures trigger a red alert and a Slack notification to the team | | OpenShift CI docs | [docs.ci.openshift.org](https://docs.ci.openshift.org/) | General documentation on how OpenShift CI works | +| OTE Framework | [github.com/openshift-eng/openshift-tests-extension](https://github.com/openshift-eng/openshift-tests-extension) | OpenShift Tests Extension framework - wraps Ginkgo and exposes test commands | +| OTE Enhancement | [OTE Enhancement Proposal](https://github.com/openshift/enhancements/blob/master/enhancements/testing/openshift-tests-extension.md) | Official design doc for the OpenShift Tests Extension framework | +| Ginkgo v2 docs | [onsi.github.io/ginkgo](https://onsi.github.io/ginkgo/) | Official Ginkgo BDD testing framework documentation | +| Ginkgo CLI reference | [Ginkgo CLI flags](https://onsi.github.io/ginkgo/#the-ginkgo-cli) | Complete reference for Ginkgo command-line flags and options | | Help with alerts | `#forum-ocp-testplatform` on Slack | Managed by the TRT team | | Help with OTE | `#wg-openshift-tests-extension` on Slack | Questions about the OpenShift Tests Extension framework | @@ -106,55 +110,145 @@ Example ([source](https://github.com/openshift/release/blob/main/ci-operator/con ## How to Run the Tests Locally -| Command | Description | -|-------------------------------------------------|--------------------------------------------------------------------------| -| `make build` | Builds the OLMv1 test binary. | -| `./bin/olmv1-tests-ext info` | Shows info about the test binary and registered test suites. | -| `./bin/olmv1-tests-ext list` | Lists all available test cases. | -| `./bin/olmv1-tests-ext run-suite olmv1/all` | Runs the full OLMv1 test suite. | -| `./bin/olmv1-tests-ext run-test -n ` | Runs one specific test. Replace with the test's full name. | +You must run OTE tests (`./bin/olmv1-tests-ext`) against an OCP Cluster with TechPreview Features enabled. +### Setup: Get an OpenShift Cluster -## How to Run the Tests Locally +Use Cluster Bot to create an OpenShift cluster with OLMv1 installed: -The tests can be run locally using the `olmv1-tests-ext` binary against an OpenShift cluster. -These tests are specifically designed for OpenShift and require OpenShift-specific APIs and features. +```shell +launch 4.20 gcp,techpreview +``` -Use the environment variable `KUBECONFIG` to point to your cluster configuration file such as: +Set `KUBECONFIG`: ```shell -KUBECONFIG=path/to/kubeconfig ./bin/olmv1-tests-ext run-test -n +mv ~/Downloads/cluster-bot-2025-08-06-082741.kubeconfig ~/.kube/cluster-bot.kubeconfig +export KUBECONFIG=~/.kube/cluster-bot.kubeconfig ``` -To run tests that include tech preview features, -you need an OpenShift cluster with OLMv1 installed and those features enabled. +### Two Ways to Run Tests -### Local Test using OLMv1 on OpenShift +#### 1. **Developer-Friendly Output** (For local development) -1. Use the `Cluster Bot` to create an OpenShift cluster with OLMv1 installed. +Use the local dev commands (`run-suite-dev`, `run-test-dev`) that provide clean, human-readable output: -**Example:** +**Implementation:** Local dev commands in `localdevoutput/` are excluded from production builds using Go build tags. Only included with `make build-local-dev`. See [localdevoutput/README.md](localdevoutput/README.md). -```shell -launch 4.20 gcp,techpreview +| Command | Description | +|---------|-------------| +| `make build-local-dev` | Builds the test binary with local dev commands | +| `make test-local SUITE=olmv1/all` | Runs a test suite with clean, color-coded output | +| `make test-local-single TEST="test name"` | Runs a single test with clean output | +| `make list-test-names` | Lists all available test names | + +**Example** + +```bash +export KUBECONFIG=~/.kube/cluster-bot.kubeconfig +make build-local-dev +make test-local SUITE=olmv1/all ``` -2. Set the `KUBECONFIG` environment variable to point to your OpenShift cluster configuration file. +**Output:** Clean, color-coded summary with live progress: +```text +[46/46] ▶ Running: [sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA] OLMv1 operator with webhooks should have a working validating webhook + ✓ PASSED [194.1 seconds] (Total: ✓45 ✗0) -**Example:** -```shell -mv ~/Downloads/cluster-bot-2025-08-06-082741.kubeconfig ~/.kube/cluster-bot.kubeconfig -export KUBECONFIG=~/.kube/cluster-bot.kubeconfig +════════════════════════════════════════════════════════ + Final Summary +════════════════════════════════════════════════════════ +✓ Passed: 45 +✗ Failed: 0 +⊘ Skipped: 1 + +✓ ALL TESTS PASSED! ``` -3. Run the tests using the `olmv1-tests-ext` binary. +#### 2. **Raw OTE Framework Output** (For CI/CD integration) + +Run the binary directly for structured JSON reports: + +| Command | Description | +|---------|-------------| +| `./bin/olmv1-tests-ext info` | Shows info about the test binary and registered test suites | +| `./bin/olmv1-tests-ext list` | Lists all available test cases | +| `./bin/olmv1-tests-ext run-suite olmv1/all` | Runs the full OLMv1 test suite with JSON output | +| `./bin/olmv1-tests-ext run-test -n ` | Runs one specific test with JSON output | **Example:** -```shell + +```bash +export KUBECONFIG=~/.kube/cluster-bot.kubeconfig +make build ./bin/olmv1-tests-ext run-suite olmv1/all ``` +**Output:** Structured JSON report (as used by Component Readiness and other integrated solutions): +```text +Running Suite: - /Users/camilam/go/src/github/operator-framework-operator-controller/openshift/tests-extension +=============================================================================================================== +Random Seed: 1753508546 - will randomize all specs + +Will run 1 of 1 specs +------------------------------ +[sig-olmv1] OLMv1 should pass a trivial sanity check +/Users/camilam/go/src/github/operator-framework-operator-controller/openshift/tests-extension/test/olmv1.go:26 +• [0.000 seconds] +------------------------------ + +Ran 1 of 1 Specs in 0.000 seconds +SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped +[ + { + "name": "[sig-olmv1] OLMv1 should pass a trivial sanity check", + "lifecycle": "blocking", + "duration": 0, + "startTime": "2025-07-26 05:42:26.553852 UTC", + "endTime": "2025-07-26 05:42:26.580263 UTC", + "result": "passed", + "output": "" + } +] +``` + +This is the same output format used by: +- **Component Readiness** ([Sippy](https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main)) +- **OpenShift CI/CD** pipeline +- **Release validation jobs** +- Any automated test processing tools + +**When to use which:** +- Use **clean output** (`make test-local` or `make test-local-single`) for local development, debugging, and quick visual feedback +- Use **raw output** (direct binary execution with `./bin/olmv1-tests-ext`) when you need JSON reports, CI/CD integration, or programmatic processing + +### Discovering Available Flags + +The OTE framework wraps Ginkgo and exposes its own set of commands and flags. To see what's available: + +```bash +# See all available commands +./bin/olmv1-tests-ext --help + +# See flags for running test suites +./bin/olmv1-tests-ext run-suite --help + +# See flags for running individual tests +./bin/olmv1-tests-ext run-test --help +``` + +**Available OTE-specific flags:** +- `--component string` - Specify the component to enable (default "default") +- `--max-concurrency int` - Maximum number of tests to run in parallel (default 10) +- `--output string` - Output mode (default "json") +- `--junit-path string` - Write results to JUnit XML (for `run-suite`) +- `--names stringArray` - Specify test name, can be used multiple times (for `run-test`) + +**Note:** The OTE framework does not expose all Ginkgo CLI flags. +It provides a simplified interface focused on running tests in OpenShift environments. +For full Ginkgo flag reference, see the [Ginkgo CLI documentation](https://onsi.github.io/ginkgo/#the-ginkgo-cli). + ## Development Workflow - Add or update tests in: `openshift/tests-extension/tests/` @@ -283,14 +377,16 @@ that the metadata is up to date: ## Makefile Commands -| Target | Description | -|--------------------------|------------------------------------------------------------------------------| -| `make build` | Builds the test binary. | -| `make update-metadata` | Updates the metadata JSON file. | -| `make build-update` | Runs build + update-metadata + cleans codeLocations. | -| `make verify` | Runs formatting, vet, and linter. | -| `make list-test-names` | Shows all test names in the binary. | -| `make clean-metadata` | Removes machine-specific codeLocations from the JSON metadata. [More info](https://issues.redhat.com/browse/TRT-2186) | +| Target | Description | +|----------------------------------|------------------------------------------------------------------------------| +| `make build` | Builds the test binary. | +| `make test-local SUITE=` | Runs a test suite with clean, human-readable output for local development. | +| `make test-local-single TEST=""` | Runs a single test with clean, human-readable output. | +| `make list-test-names` | Shows all test names in the binary. | +| `make update-metadata` | Updates the metadata JSON file. | +| `make build-update` | Runs build + update-metadata + cleans codeLocations. | +| `make verify` | Runs formatting, vet, and linter. | +| `make clean-metadata` | Removes machine-specific codeLocations from the JSON metadata. [More info](https://issues.redhat.com/browse/TRT-2186) | **Note:** Metadata is stored in: `.openshift-tests-extension/openshift_payload_olmv1.json` diff --git a/openshift/tests-extension/cmd/main.go b/openshift/tests-extension/cmd/main.go index 9cc267a5db..88028f0209 100644 --- a/openshift/tests-extension/cmd/main.go +++ b/openshift/tests-extension/cmd/main.go @@ -14,12 +14,13 @@ import ( "os" "strings" - "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + otecmd "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" e "github.com/openshift-eng/openshift-tests-extension/pkg/extension" et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" "github.com/spf13/cobra" + localdevcmd "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/localdevoutput/cmd" "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/pkg/env" _ "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test" _ "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test/qe/specs" @@ -300,33 +301,38 @@ func main() { } // Get all default commands from the extension framework - allCommands := cmd.DefaultExtensionCommands(registry) + allCommands := otecmd.DefaultExtensionCommands(registry) - // Add KUBECONFIG check to run-suite and run-test commands only. + // Add local dev commands for local development (only included when built with -tags dev) + allCommands = append(allCommands, localdevcmd.RegisterLocalDevCommands(registry)...) + + // Add KUBECONFIG check to run-suite, run-test, and dev variants. // Other commands (list, info, images, update, completion, help) don't need KUBECONFIG. for _, command := range allCommands { - // Identify run-suite and run-test commands by their Use field - if command.Use == "run-suite NAME" || command.Use == "run-test [-n NAME...] [NAME]" { - // Save the original RunE function - originalRunE := command.RunE - - // Wrap it with KUBECONFIG check - command.RunE = func(cmd *cobra.Command, args []string) error { - // Check KUBECONFIG before running the test + switch command.Name() { + case "run-suite", "run-test", "run-suite-dev", "run-test-dev": + localCmd := command + originalRunE := localCmd.RunE + + localCmd.RunE = func(cmd *cobra.Command, args []string) error { if err := exutil.CheckKubeconfigSet(); err != nil { return err } - // Call the original RunE function - return originalRunE(cmd, args) + if originalRunE != nil { + return originalRunE(cmd, args) + } + if localCmd.Run != nil { + localCmd.Run(cmd, args) + return nil + } + return fmt.Errorf("command %s has no Run or RunE function", localCmd.Name()) } } } root.AddCommand(allCommands...) - if err := func() error { - return root.Execute() - }(); err != nil { + if err := root.Execute(); err != nil { os.Exit(1) } } diff --git a/openshift/tests-extension/go.mod b/openshift/tests-extension/go.mod index bee36a11d2..155b672f6d 100644 --- a/openshift/tests-extension/go.mod +++ b/openshift/tests-extension/go.mod @@ -13,6 +13,7 @@ require ( github.com/openshift/origin v1.5.0-alpha.3.0.20251010041851-79ff1dbbe815 github.com/operator-framework/operator-controller v1.8.1-0.20260319123036-8ccea5a0cf67 github.com/pborman/uuid v1.2.1 + github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.10.2 github.com/tidwall/gjson v1.18.0 github.com/tidwall/pretty v1.2.1 @@ -73,7 +74,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect diff --git a/openshift/tests-extension/localdevoutput/README.md b/openshift/tests-extension/localdevoutput/README.md new file mode 100644 index 0000000000..18b7cafe52 --- /dev/null +++ b/openshift/tests-extension/localdevoutput/README.md @@ -0,0 +1,54 @@ +# Local Dev Output Commands + +Developer-friendly test commands with clean, human-readable output for local execution. + +## Structure + +```text +localdevoutput/ +├── cmd/ # Cobra commands for local development +│ ├── register.go # Returns nil (production build) +│ ├── register_local_dev.go # Registers local dev commands (with -tags dev) +│ ├── run_suite_local_dev.go # run-suite-dev command +│ └── run_test_local_dev.go # run-test-dev command +└── pkg/output/ # Output formatting + ├── formatter.go # ANSI colors, progress, summaries + └── writer.go # OTE ResultWriter implementation +``` + +## Purpose + +These commands are **for local execution only**. They provide human-readable output when testing against OCP clusters locally. + +## Build Separation + +- **Production build**: `make build` → Excludes local dev commands (ships to OCP payload) +- **Local dev build**: `make build-local-dev` → Includes local dev commands (for developers) + +Build tags (`//go:build dev`) ensure dev commands are only compiled when explicitly requested with `-tags dev`. + +### Output Flow + +```text +Test Run → ResultWriter.Write(result) → Formatter → Colored Terminal Output +``` + +The `CleanResultWriter` implements OTE's `ResultWriter` interface to intercept test results and format them with colors and progress indicators. + +## Usage + +```bash +# Run test suite with clean output +make test-local SUITE=olmv1/all + +# Run single test +make test-local-single TEST="[sig-olmv1] test name" + +# Direct binary usage +./bin/olmv1-tests-ext run-suite-dev olmv1/all +./bin/olmv1-tests-ext run-test-dev -n "test name" +``` + +## Why Separate Directory? + +Isolated in `localdevoutput/` to keep local development tools separate from the core test framework that ships to production. diff --git a/openshift/tests-extension/localdevoutput/cmd/register.go b/openshift/tests-extension/localdevoutput/cmd/register.go new file mode 100644 index 0000000000..e4f3b462e3 --- /dev/null +++ b/openshift/tests-extension/localdevoutput/cmd/register.go @@ -0,0 +1,12 @@ +//go:build !dev + +package cmd + +import ( + "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + "github.com/spf13/cobra" +) + +func RegisterLocalDevCommands(registry *extension.Registry) []*cobra.Command { + return nil +} diff --git a/openshift/tests-extension/localdevoutput/cmd/register_local_dev.go b/openshift/tests-extension/localdevoutput/cmd/register_local_dev.go new file mode 100644 index 0000000000..cd17844b1e --- /dev/null +++ b/openshift/tests-extension/localdevoutput/cmd/register_local_dev.go @@ -0,0 +1,15 @@ +//go:build dev + +package cmd + +import ( + "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + "github.com/spf13/cobra" +) + +func RegisterLocalDevCommands(registry *extension.Registry) []*cobra.Command { + return []*cobra.Command{ + NewRunSuiteDevCommand(registry), + NewRunTestDevCommand(registry), + } +} diff --git a/openshift/tests-extension/localdevoutput/cmd/run_suite_local_dev.go b/openshift/tests-extension/localdevoutput/cmd/run_suite_local_dev.go new file mode 100644 index 0000000000..efee0e9b08 --- /dev/null +++ b/openshift/tests-extension/localdevoutput/cmd/run_suite_local_dev.go @@ -0,0 +1,120 @@ +//go:build dev + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + pkgerrors "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + "github.com/openshift-eng/openshift-tests-extension/pkg/flags" + + "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/localdevoutput/pkg/output" +) + +// ErrTestsFailed is returned when one or more tests fail. +var ErrTestsFailed = errors.New("one or more tests failed") + +func NewRunSuiteDevCommand(registry *extension.Registry) *cobra.Command { + opts := struct { + componentFlags *flags.ComponentFlags + concurrencyFlags *flags.ConcurrencyFlags + }{ + componentFlags: flags.NewComponentFlags(), + concurrencyFlags: flags.NewConcurrencyFlags(), + } + + cmd := &cobra.Command{ + Use: "run-suite-dev NAME", + Short: "Run a test suite with clean, human-readable output for local development", + Long: `Run a test suite with clean, human-readable output. + +This command provides a developer-friendly alternative to run-suite with: + - Live test progress indicators + - Color-coded pass/fail status + - Running totals after each test + - Clean summary with detailed failure information + +Perfect for local development and debugging. For CI/CD integration, use run-suite instead.`, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancelCause := context.WithCancelCause(context.Background()) + defer cancelCause(errors.New("exiting")) + + abortCh := make(chan os.Signal, 2) + go func() { + <-abortCh + fmt.Fprintf(os.Stderr, "Interrupted, terminating tests") + cancelCause(errors.New("interrupt received")) + + select { + case sig := <-abortCh: + fmt.Fprintf(os.Stderr, "Interrupted twice, exiting (%s)", sig) + switch sig { + case syscall.SIGINT: + os.Exit(130) + default: + os.Exit(130) + } + + case <-time.After(30 * time.Minute): + fmt.Fprintf(os.Stderr, "Timed out during cleanup, exiting") + os.Exit(130) + } + }() + signal.Notify(abortCh, syscall.SIGINT, syscall.SIGTERM) + + ext := registry.Get(opts.componentFlags.Component) + if ext == nil { + return fmt.Errorf("component not found: %s", opts.componentFlags.Component) + } + if len(args) != 1 { + return fmt.Errorf("must specify one suite name") + } + suite, err := ext.GetSuite(args[0]) + if err != nil { + return pkgerrors.Wrapf(err, "couldn't find suite: %s", args[0]) + } + + cleanWriter := output.NewCleanResultWriter(os.Stdout) + + specs, err := ext.GetSpecs().Filter(suite.Qualifiers) + if err != nil { + return pkgerrors.Wrap(err, "couldn't filter specs") + } + + cleanWriter.SetTotalTests(len(specs)) + + if err := specs.Run(ctx, cleanWriter, opts.concurrencyFlags.MaxConcurency); err != nil { + if flushErr := cleanWriter.Flush(); flushErr != nil { + fmt.Fprintf(os.Stderr, "failed to write results: %v\n", flushErr) + } + return err + } + + if err := cleanWriter.Flush(); err != nil { + fmt.Fprintf(os.Stderr, "failed to write results: %v\n", err) + return err + } + + if cleanWriter.HasFailures() { + return ErrTestsFailed + } + + return nil + }, + } + + opts.componentFlags.BindFlags(cmd.Flags()) + opts.concurrencyFlags.BindFlags(cmd.Flags()) + + return cmd +} diff --git a/openshift/tests-extension/localdevoutput/cmd/run_test_local_dev.go b/openshift/tests-extension/localdevoutput/cmd/run_test_local_dev.go new file mode 100644 index 0000000000..99a5c6ad74 --- /dev/null +++ b/openshift/tests-extension/localdevoutput/cmd/run_test_local_dev.go @@ -0,0 +1,124 @@ +//go:build dev + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" + + "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + "github.com/openshift-eng/openshift-tests-extension/pkg/flags" + + "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/localdevoutput/pkg/output" +) + +func NewRunTestDevCommand(registry *extension.Registry) *cobra.Command { + opts := struct { + componentFlags *flags.ComponentFlags + nameFlags *flags.NamesFlags + }{ + componentFlags: flags.NewComponentFlags(), + nameFlags: flags.NewNamesFlags(), + } + + cmd := &cobra.Command{ + Use: "run-test-dev [-n NAME...] [NAME]", + Short: "Run individual tests with clean, human-readable output for local development", + Long: `Run one or more tests with clean, human-readable output. + +This command provides a developer-friendly alternative to run-test with: + - Live test progress indicators + - Color-coded pass/fail status + - Running totals after each test + - Clean summary with detailed failure information + +Perfect for local development and debugging. For CI/CD integration, use run-test instead. + +Examples: + # Run a single test + run-test-dev -n "[sig-olmv1] OLMv1 should pass a trivial sanity check" + + # Run multiple tests + run-test-dev -n "test 1" -n "test 2"`, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancelCause := context.WithCancelCause(context.Background()) + defer cancelCause(errors.New("exiting")) + + abortCh := make(chan os.Signal, 2) + go func() { + <-abortCh + fmt.Fprintf(os.Stderr, "Interrupted, terminating tests") + cancelCause(errors.New("interrupt received")) + + select { + case sig := <-abortCh: + fmt.Fprintf(os.Stderr, "Interrupted twice, exiting (%s)", sig) + switch sig { + case syscall.SIGINT: + os.Exit(130) + default: + os.Exit(130) + } + + case <-time.After(30 * time.Minute): + fmt.Fprintf(os.Stderr, "Timed out during cleanup, exiting") + os.Exit(130) + } + }() + signal.Notify(abortCh, syscall.SIGINT, syscall.SIGTERM) + + ext := registry.Get(opts.componentFlags.Component) + if ext == nil { + return fmt.Errorf("component not found: %s", opts.componentFlags.Component) + } + + names := opts.nameFlags.Names + if len(args) > 0 { + names = append(names, args...) + } + if len(names) == 0 { + return fmt.Errorf("must specify at least one test name via -n flag or argument") + } + + cleanWriter := output.NewCleanResultWriter(os.Stdout) + + specs, err := ext.FindSpecsByName(names...) + if err != nil { + return err + } + + cleanWriter.SetTotalTests(len(specs)) + + if err := specs.Run(ctx, cleanWriter, 1); err != nil { + if flushErr := cleanWriter.Flush(); flushErr != nil { + fmt.Fprintf(os.Stderr, "failed to write results: %v\n", flushErr) + } + return err + } + + if err := cleanWriter.Flush(); err != nil { + fmt.Fprintf(os.Stderr, "failed to write results: %v\n", err) + return err + } + + if cleanWriter.HasFailures() { + return ErrTestsFailed + } + + return nil + }, + } + + opts.componentFlags.BindFlags(cmd.Flags()) + opts.nameFlags.BindFlags(cmd.Flags()) + + return cmd +} diff --git a/openshift/tests-extension/localdevoutput/pkg/output/formatter.go b/openshift/tests-extension/localdevoutput/pkg/output/formatter.go new file mode 100644 index 0000000000..3e86e9e5c1 --- /dev/null +++ b/openshift/tests-extension/localdevoutput/pkg/output/formatter.go @@ -0,0 +1,187 @@ +//go:build dev + +// Package output provides human-readable formatting utilities for local development. +// The Formatter type provides color-coded progress indicators and summaries that are +// used by CleanResultWriter (defined in writer.go), which implements the OTE ResultWriter interface. +package output + +import ( + "fmt" + "io" + "strings" + "time" + + et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" +) + +// ANSI color codes +const ( + ColorReset = "\033[0m" + ColorRed = "\033[0;31m" + ColorGreen = "\033[0;32m" + ColorYellow = "\033[1;33m" + ColorBlue = "\033[0;34m" + ColorCyan = "\033[0;36m" + ColorGray = "\033[0;90m" + ColorBold = "\033[1m" +) + +// Formatter provides human-readable output for test results +type Formatter struct { + writer io.Writer + totalTests int + currentTest int + passedCount int + failedCount int + skippedCount int + pendingCount int + failedTests []FailedTest +} + +// FailedTest holds information about a failed test +type FailedTest struct { + Name string + Error string +} + +// NewFormatter creates a new formatter instance +func NewFormatter(w io.Writer) *Formatter { + return &Formatter{ + writer: w, + failedTests: make([]FailedTest, 0), + } +} + +// PrintHeader prints the test suite header +func (f *Formatter) PrintHeader() { + fmt.Fprintf(f.writer, "%s%s════════════════════════════════════════════════════════%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintf(f.writer, "%s%s OLMv1 Test Suite - Starting%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintf(f.writer, "%s%s════════════════════════════════════════════════════════%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintln(f.writer) +} + +func (f *Formatter) SetTotalTests(total int) { + f.totalTests = total + if total > 0 { + fmt.Fprintf(f.writer, "%sTotal tests: %d%s\n", ColorBold, total, ColorReset) + fmt.Fprintf(f.writer, "%s━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━%s\n", ColorGray, ColorReset) + fmt.Fprintln(f.writer) + } +} + +func (f *Formatter) PrintTestStart(name string) { + f.currentTest++ + if f.totalTests > 0 { + fmt.Fprintf(f.writer, "%s[%d/%d]%s %s▶ Running:%s %s\n", + ColorBold, f.currentTest, f.totalTests, ColorReset, + ColorBlue, ColorReset, name) + } else { + fmt.Fprintf(f.writer, "%s▶ Running:%s %s\n", ColorBlue, ColorReset, name) + } +} + +func (f *Formatter) PrintTestResult(result *et.ExtensionTestResult) { + duration := formatDuration(result.Duration) + + switch result.Result { + case et.ResultPassed: + f.passedCount++ + fmt.Fprintf(f.writer, "%s ✓ PASSED%s [%s] %s(Total: ✓%d ✗%d)%s\n", + ColorGreen, ColorReset, duration, + ColorGray, f.passedCount, f.failedCount, ColorReset) + case et.ResultFailed: + f.failedCount++ + f.failedTests = append(f.failedTests, FailedTest{ + Name: result.Name, + Error: result.Error, + }) + fmt.Fprintf(f.writer, "%s ✗ FAILED%s [%s] %s(Total: ✓%d ✗%d)%s\n", + ColorRed, ColorReset, duration, + ColorGray, f.passedCount, f.failedCount, ColorReset) + case et.ResultSkipped: + f.skippedCount++ + fmt.Fprintf(f.writer, "%s ⊘ SKIPPED%s [%s] %s(Total: ✓%d ✗%d ⊘%d)%s\n", + ColorYellow, ColorReset, duration, + ColorGray, f.passedCount, f.failedCount, f.skippedCount, ColorReset) + default: + f.failedCount++ + errorMsg := fmt.Sprintf("unexpected result type: %s", result.Result) + if result.Error != "" { + errorMsg = fmt.Sprintf("%s; %s", errorMsg, result.Error) + } + f.failedTests = append(f.failedTests, FailedTest{ + Name: result.Name, + Error: errorMsg, + }) + fmt.Fprintf(f.writer, "%s ✗ UNKNOWN/FAILED%s [%s] %s(Total: ✓%d ✗%d)%s\n", + ColorRed, ColorReset, duration, + ColorGray, f.passedCount, f.failedCount, ColorReset) + } + fmt.Fprintln(f.writer) +} + +func (f *Formatter) PrintSummary() { + fmt.Fprintln(f.writer) + fmt.Fprintf(f.writer, "%s%s════════════════════════════════════════════════════════%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintf(f.writer, "%s%s Final Summary%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintf(f.writer, "%s%s════════════════════════════════════════════════════════%s\n", ColorCyan, ColorBold, ColorReset) + + fmt.Fprintf(f.writer, "%s✓ Passed: %d%s\n", ColorGreen, f.passedCount, ColorReset) + fmt.Fprintf(f.writer, "%s✗ Failed: %d%s\n", ColorRed, f.failedCount, ColorReset) + fmt.Fprintf(f.writer, "%s⊘ Skipped: %d%s\n", ColorYellow, f.skippedCount, ColorReset) + + fmt.Fprintln(f.writer) + + if f.failedCount == 0 { + fmt.Fprintf(f.writer, "%s%s✓ ALL TESTS PASSED!%s\n", ColorGreen, ColorBold, ColorReset) + } else { + fmt.Fprintf(f.writer, "%s%s✗ %d TEST(S) FAILED%s\n", ColorRed, ColorBold, f.failedCount, ColorReset) + f.printFailedTestDetails() + } +} + +func (f *Formatter) printFailedTestDetails() { + if len(f.failedTests) == 0 { + return + } + + fmt.Fprintln(f.writer) + fmt.Fprintf(f.writer, "%s%s════════════════════════════════════════════════════════%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintf(f.writer, "%s%s Failed Test Details%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintf(f.writer, "%s%s════════════════════════════════════════════════════════%s\n", ColorCyan, ColorBold, ColorReset) + fmt.Fprintln(f.writer) + + for _, ft := range f.failedTests { + fmt.Fprintf(f.writer, "%s%sTest: %s%s\n", ColorRed, ColorBold, ft.Name, ColorReset) + if ft.Error != "" { + // Clean up the error message + errorMsg := cleanErrorMessage(ft.Error) + fmt.Fprintf(f.writer, "%s%s%s\n", ColorGray, errorMsg, ColorReset) + } + fmt.Fprintln(f.writer) + } +} + +func (f *Formatter) HasFailures() bool { + return f.failedCount > 0 +} + +func formatDuration(durationMs int64) string { + d := time.Duration(durationMs) * time.Millisecond + if d < time.Second { + return fmt.Sprintf("%.3f seconds", d.Seconds()) + } + return fmt.Sprintf("%.1f seconds", d.Seconds()) +} + +func cleanErrorMessage(msg string) string { + lines := strings.Split(msg, "\n") + var cleaned []string + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + cleaned = append(cleaned, trimmed) + } + } + return strings.Join(cleaned, "\n") +} diff --git a/openshift/tests-extension/localdevoutput/pkg/output/writer.go b/openshift/tests-extension/localdevoutput/pkg/output/writer.go new file mode 100644 index 0000000000..fc83c1c900 --- /dev/null +++ b/openshift/tests-extension/localdevoutput/pkg/output/writer.go @@ -0,0 +1,66 @@ +//go:build dev + +package output + +import ( + "io" + "os" + "sync" + + et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" +) + +type CleanResultWriter struct { + lock sync.Mutex + formatter *Formatter + results []*et.ExtensionTestResult + writer io.Writer +} + +func NewCleanResultWriter(w io.Writer) *CleanResultWriter { + if w == nil { + w = os.Stdout + } + + formatter := NewFormatter(w) + formatter.PrintHeader() + + return &CleanResultWriter{ + formatter: formatter, + results: make([]*et.ExtensionTestResult, 0), + writer: w, + } +} + +func (w *CleanResultWriter) SetTotalTests(total int) { + w.lock.Lock() + defer w.lock.Unlock() + w.formatter.SetTotalTests(total) +} + +func (w *CleanResultWriter) Write(result *et.ExtensionTestResult) { + w.lock.Lock() + defer w.lock.Unlock() + + if result == nil { + return + } + + w.results = append(w.results, result) + w.formatter.PrintTestStart(result.Name) + w.formatter.PrintTestResult(result) +} + +func (w *CleanResultWriter) Flush() error { + w.lock.Lock() + defer w.lock.Unlock() + + w.formatter.PrintSummary() + return nil +} + +func (w *CleanResultWriter) HasFailures() bool { + w.lock.Lock() + defer w.lock.Unlock() + return w.formatter.HasFailures() +}