diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97497db..02073c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,12 @@ jobs: contents: read steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version-file: go.mod cache-dependency-path: go.sum + - run: go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... - run: go test ./... - run: go build -o bin/secretsync ./cmd/secretsync diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2197fa8..60d545f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,10 +37,12 @@ jobs: with: ref: ${{ needs.release-please.outputs.tag_name }} fetch-depth: 0 + persist-credentials: false - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version-file: go.mod cache-dependency-path: go.sum + - run: go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... - name: Run GoReleaser uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 with: diff --git a/.goreleaser.yml b/.goreleaser.yml index 453b941..5c5ee8e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -94,27 +94,22 @@ release: prerelease: auto mode: replace header: | - ## secretsync {{ .Tag }} + ## SecretSync {{ .Tag }} ### Installation - **Docker:** - ```bash - docker pull jbcom/secretssync:{{ .Version }} - ``` + **Binary:** + Download from the assets below for your platform. - **Helm (OCI):** + **Go install:** ```bash - helm install secretsync oci://registry-1.docker.io/jbcom/secretssync --version {{ .Version }} + go install github.com/jbcom/secrets-sync/cmd/secretsync@{{ .Tag }} ``` - **Binary:** - Download from the assets below for your platform. - footer: | --- **Full Changelog**: https://github.com/jbcom/secrets-sync/compare/{{ .PreviousTag }}...{{ .Tag }} -# Docker images and Helm charts are built/pushed separately -# via dedicated workflow jobs for better control +# Container and chart artifacts are outside this binary release workflow. +# Keep release notes limited to artifacts built here. dockers: [] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a09cddb..1577563 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,49 +206,47 @@ secretsync/ ├── cmd/secretsync/ # CLI application │ ├── cmd/ # Cobra commands │ └── main.go # Entry point -├── pkg/ # Public packages -│ ├── pipeline/ # Pipeline orchestration -│ ├── diff/ # Diff computation -│ └── ... -├── stores/ # Secret store implementations -│ ├── vault/ # Vault store -│ ├── aws/ # AWS Secrets Manager -│ └── ... -├── internal/ # Private packages +├── pkg/ +│ ├── client/ # Vault, AWS, and provider clients +│ ├── discovery/ # AWS Organizations and Identity Center discovery +│ ├── driver/ # Supported driver names and validation helpers +│ ├── pipeline/ # Merge, sync, graph, and execution orchestration +│ ├── diff/ # Diff computation and masking +│ └── observability/ # Metrics and request tracking +├── python/ # Optional gopy binding sources ├── docs/ # Documentation ├── examples/ # Example configurations └── deploy/ # Deployment manifests ``` -## Adding a New Secret Store +## Adding a New Secret Backend -To add support for a new secret store: +To add support for a new backend: -1. **Create store package** +1. **Create a client package** ```bash - mkdir -p stores/newstore + mkdir -p pkg/client/newbackend ``` -2. **Implement Store interface** +2. **Implement the current client shape** ```go - package newstore + package newbackend - import "github.com/jbcom/secrets-sync/pkg/store" + import "github.com/jbcom/secrets-sync/pkg/driver" - type Store struct { - // configuration fields + type Client struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` } - func (s *Store) Get(ctx context.Context, key string) ([]byte, error) { - // implementation + func (c *Client) Validate() error { + if c.Name == "" { + return driver.ErrPathRequired + } + return nil } - func (s *Store) Set(ctx context.Context, key string, value []byte) error { - // implementation - } - - func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { - // implementation + func (c *Client) Driver() driver.DriverName { + return driver.DriverName("newbackend") } ``` @@ -261,9 +259,10 @@ To add support for a new secret store: } ``` -4. **Register store** - - Update pipeline config to include new store - - Add store initialization logic +4. **Register the backend** + - Add the driver name in `pkg/driver` + - Update pipeline config types and validation + - Add client initialization logic in the pipeline layer - Update documentation 5. **Add examples** diff --git a/Dockerfile b/Dockerfile index 89b2fa3..534623b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ # Tests now run in CI (outside Docker), so this Dockerfile focuses purely # on compiling and packaging the runtime image. ### -FROM golang:1.25-trixie AS builder +FROM golang:1.26.4-trixie AS builder ARG TARGETOS=linux ARG TARGETARCH=amd64 @@ -59,8 +59,6 @@ WORKDIR /app RUN mkdir -p /etc/ssl/certs COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /out/secretsync /usr/local/bin/secretsync -# Keep vss as a symlink for backwards compatibility -RUN ln -s /usr/local/bin/secretsync /usr/local/bin/vss # Default command - Viper reads SECRETSYNC_* env vars directly ENTRYPOINT ["/usr/local/bin/secretsync"] diff --git a/Makefile b/Makefile index 293e99c..97b6f0f 100644 --- a/Makefile +++ b/Makefile @@ -40,8 +40,8 @@ python-bindings: @mkdir -p $(PYTHON_OUTPUT) $(GOPY) pkg -output=$(PYTHON_OUTPUT) -vm=$(PYTHON) -name=$(PYTHON_PKG) \ -version=$(VERSION) \ - -author="Extended Data Library" \ - -email="support@extended-data.dev" \ + -author="jbcom" \ + -email="jon@jonbogaty.com" \ -url="https://github.com/jbcom/secrets-sync" \ -desc="Enterprise-grade secret synchronization pipeline with Python bindings" \ ./python/secretssync diff --git a/README.md b/README.md index 890b449..48bff9a 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![GitHub release](https://img.shields.io/github/release/jbcom/secrets-sync.svg)](https://github.com/jbcom/secrets-sync/releases) [![Go Report Card](https://goreportcard.com/badge/github.com/jbcom/secrets-sync)](https://goreportcard.com/report/github.com/jbcom/secrets-sync) -[![Python Bindings](https://img.shields.io/badge/python-bindings-blue.svg)](./python/) +[![Python Integration](https://img.shields.io/badge/python-integration-blue.svg)](./docs/PYTHON_BINDINGS.md) -[Quick Start](#quick-start) • [Package Docs](https://extended-data.dev/packages/secretssync/) • [Repo Docs](./docs/) • [Python Bindings](#python-bindings) • [Examples](./examples/) • [GitHub Action](./docs/GITHUB_ACTIONS.md) +[Quick Start](#quick-start) • [Repo Docs](./docs/) • [Python Integration](#python-integration) • [Examples](./examples/) • [GitHub Action](./docs/GITHUB_ACTIONS.md) @@ -18,11 +18,15 @@ SecretSync provides **fully automated, enterprise-grade secret synchronization** across multiple cloud providers and secret stores. Built for scale with a **two-phase pipeline architecture** (merge → sync), it supports inheritance, dynamic target discovery, and CI/CD-friendly diff reporting. -## 🏢 Part of Extended Data Library +## 🏢 Independent Repository, Extended Data Integration -SecretSync is part of the [Extended Data Library](https://github.com/jbcom/secrets-sync) ecosystem - a collection of high-performance, enterprise-grade tools for data management, secret handling, and infrastructure automation. +SecretSync is an independent [jbcom/secrets-sync](https://github.com/jbcom/secrets-sync) repository and MIT-licensed release artifact for secret synchronization workflows. -**🐍 Python Integration**: SecretSync provides Python bindings via [gopy](https://github.com/go-python/gopy), enabling seamless integration with the [extended-data](https://github.com/jbcom/extended-data) library and Python-based AI agents. +**🐍 Python Integration**: SecretSync is available to Python through the +[extended-data](https://github.com/jbcom/extended-data) `extended-data[secrets]` +connector, which executes the supported `secretsync` CLI contract. This repo +also keeps direct [gopy](https://github.com/go-python/gopy) binding sources for +local experiments. **🚀 Perfect for:** Multi-account AWS environments, Kubernetes deployments, CI/CD pipelines, and enterprise secret management at scale. @@ -39,24 +43,24 @@ SecretSync is part of the [Extended Data Library](https://github.com/jbcom/secre ## ✨ Key Features -### 🔍 **Advanced Discovery** (v1.2.0) +### 🔍 **Advanced Discovery** - **AWS Organizations Integration**: Discover accounts with tag filtering, wildcards, and OU-based selection - **AWS Identity Center**: Permission set discovery and account assignment mapping - **Smart Caching**: Multi-level caching for optimal performance at scale -### 📚 **Secret Versioning** (v1.2.0) +### 📚 **Secret Versioning** - **Complete Audit Trail**: Track every secret change with metadata - **S3-Based Storage**: Reliable, scalable version history - **Rollback Capability**: CLI support for version rollback - **Retention Policies**: Configurable cleanup of old versions -### 🎨 **Enhanced Diff Output** (v1.2.0) +### 🎨 **Enhanced Diff Output** - **Side-by-Side Comparison**: Visual diff with aligned columns and color coding - **Intelligent Masking**: Automatic detection and masking of sensitive values - **Multiple Formats**: Human, JSON, GitHub Actions, and compact outputs - **Rich Statistics**: Detailed change counts, sizes, and timing -### 🛡️ **Enterprise Reliability** (v1.1.0) +### 🛡️ **Enterprise Reliability** - **Circuit Breakers**: Automatic failure detection and recovery - **Prometheus Metrics**: Production-ready observability with `/metrics` endpoint - **Request Tracking**: Unique request IDs and duration tracking @@ -78,15 +82,16 @@ SecretSync originated as a fork of [robertlestak/vault-secret-sync](https://gith - Dynamic target discovery (AWS Organizations, Identity Center) - Comprehensive diff/dry-run system with CI/CD integration - DeepMerge semantics for secret aggregation -- Kubernetes operator with CRD support +- Kubernetes CronJob and Helm pipeline-runner deployment ## Supported Secret Stores -| Store | Source | Target | Merge Store | -|-------|--------|--------|-------------| -| HashiCorp Vault (KV2) | ✅ | ✅ | ✅ | +| Store | Source | Sync Target | Merge Store | +|-------|--------|-------------|-------------| +| HashiCorp Vault (KV2) | ✅ | ❌ | ✅ | | AWS Secrets Manager | ✅ | ✅ | ❌ | | AWS S3 | ❌ | ❌ | ✅ | +| AWS Organizations | Discovery | ❌ | ❌ | | AWS Identity Center | Discovery | ❌ | ❌ | ## Two-Phase Pipeline Architecture @@ -104,8 +109,7 @@ SecretSync originated as a fork of [robertlestak/vault-secret-sync](https://gith │ SYNC PHASE │ │ Merge Store ──┬──▶ AWS Account 1 (via STS AssumeRole) │ │ (or Source) ├──▶ AWS Account 2 │ -│ ├──▶ Vault Cluster │ -│ └──▶ GCP Project │ +│ └──▶ AWS Account 3 │ └─────────────────────────────────────────────────────────────────┘ ``` @@ -125,9 +129,15 @@ cd secrets-sync make build ``` -## Python Bindings +## Python Integration -SecretSync provides Python bindings via [gopy](https://github.com/go-python/gopy), enabling integration with Python applications and AI agent frameworks. +The recommended Python surface is the `extended-data[secrets]` connector. It +uses the `secretsync` CLI and returns mapping-style `ExtendedDict` payloads from +configuration, dry-run, merge, sync, and full pipeline operations. + +The repository also includes direct gopy binding sources under `python/` for +local binding experiments. Those bindings are not the runtime contract used by +`extended-data`. ### Building Python Bindings @@ -153,8 +163,7 @@ pip install extended-data[secrets] ``` This installs the Python connector surface. To execute the full pipeline from -Python, make sure the `secretsync` CLI is installed or the native bindings have -been built in the current environment. +Python, make sure the `secretsync` CLI is installed and available on `PATH`. ```python from extended_data.secrets import SecretsConnector @@ -163,19 +172,21 @@ from extended_data.secrets import SecretsConnector connector = SecretsConnector() # Validate configuration -is_valid, message = connector.validate_config("pipeline.yaml") +validation = connector.validate_config("pipeline.yaml") +if not validation["valid"]: + raise SystemExit(validation["message"]) # Dry run to see what would change result = connector.dry_run("pipeline.yaml") -print(f"Would sync {result.secrets_processed} secrets") -print(f" Add: {result.secrets_added}") -print(f" Modify: {result.secrets_modified}") -print(f" Remove: {result.secrets_removed}") +print(f"Would sync {result['secrets_processed']} secrets") +print(f" Add: {result['secrets_added']}") +print(f" Modify: {result['secrets_modified']}") +print(f" Remove: {result['secrets_removed']}") # Execute the full pipeline result = connector.run_pipeline("pipeline.yaml") -if result.success: - print(f"Successfully synced {result.secrets_added} secrets") +if result["success"]: + print(f"Successfully synced {result['secrets_added']} secrets") ``` ### AI Agent Integration @@ -199,87 +210,80 @@ crewai_tools = get_tools("crewai") # Validate configuration secretsync validate --config pipeline.yaml -# Dry run with enhanced diff output (v1.2.0) -secretsync pipeline --config pipeline.yaml --dry-run --format side-by-side +# Dry run with enhanced diff output +secretsync pipeline --config pipeline.yaml --dry-run --output side-by-side -# Full pipeline execution with metrics (v1.1.0) +# Full pipeline execution with metrics secretsync pipeline --config pipeline.yaml --metrics-port 9090 # CI/CD mode (exit codes: 0=no changes, 1=changes, 2=errors) -secretsync pipeline --config pipeline.yaml --dry-run --exit-code +secretsync pipeline --config pipeline.yaml --dry-run --diff --output json --exit-code -# Version management (v1.2.0) -secretsync versions --secret-path "app/database/password" -secretsync sync --version 5 --target production +# Inspect dependency order +secretsync graph --config pipeline.yaml ``` ### Example Configuration ```yaml -# pipeline.yaml - v1.2.0 with advanced features vault: - address: "https://vault.example.com" - namespace: "admin" + address: https://vault.example.com/ + namespace: admin + auth: + approle: + role_id: ${VAULT_ROLE_ID} + secret_id: ${VAULT_SECRET_ID} aws: - region: "us-east-1" - execution_role_pattern: "arn:aws:iam::{account_id}:role/SecretsSync" - -# Advanced discovery (v1.2.0) -discovery: - aws_organizations: - enabled: true - tag_filters: - - key: "Environment" - values: ["production", "staging"] - operator: "equals" - - key: "Team" - values: ["platform*"] - operator: "contains" - organizational_units: - - "ou-production-12345" - tag_logic: "AND" - cache_ttl: "1h" - - identity_center: - enabled: true - region: "us-east-1" - cache_ttl: "30m" - -# Secret versioning (v1.2.0) -versioning: - enabled: true - s3_bucket: "company-secretsync-versions" - retention_days: 90 - -# Observability (v1.1.0) -observability: - metrics: + region: us-east-1 + execution_context: + type: delegated_admin + account_id: "123456789012" + control_tower: enabled: true - port: 9090 - address: "0.0.0.0" + execution_role: + name: AWSControlTowerExecution merge_store: - vault: - mount: "secret/merged" + s3: + bucket: company-secretsync-merge-store + prefix: merged/ + versioning: + enabled: true + retain_versions: 90 sources: api-keys: vault: - path: "secret/api-keys" + mount: secret + paths: [api-keys] database: vault: - path: "secret/database" + mount: secret + paths: [database] targets: - Staging: - imports: [api-keys, database] + staging: account_id: "111111111111" - - Production: - inherits: Staging - imports: [production-overrides] + imports: [api-keys, database] + + production: account_id: "222222222222" + imports: [staging, production-overrides] + +dynamic_targets: + production-accounts: + discovery: + organizations: + ous: ["ou-production-12345"] + tag_filters: + - key: Environment + values: ["production"] + operator: equals + recursive: true + imports: [production] + region: us-east-1 + secret_prefix: platform/ ``` ## GitHub Actions @@ -288,7 +292,7 @@ SecretSync is available as a GitHub Action for seamless CI/CD integration: ```yaml - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml dry-run: 'false' @@ -327,28 +331,28 @@ See [GitHub Actions documentation](./docs/GITHUB_ACTIONS.md) for complete usage secretsync pipeline --config pipeline.yaml ``` -### Output Formats (Enhanced in v1.2.0) +### Output Formats | Format | Use Case | Features | |--------|----------|----------| | `human` | Interactive terminal output | Color coding, readable layout | -| `side-by-side` | **NEW** Visual comparison | Aligned columns, intelligent masking | +| `side-by-side` | Visual comparison | Aligned columns, intelligent masking | | `json` | Machine parsing, logging | Structured data with metadata | | `github` | GitHub Actions annotations | PR comments, file annotations | | `compact` | One-line CI status | Minimal output for scripts | -**Value Masking (v1.2.0)**: Sensitive values are automatically masked by default. Use `--show-values` flag to display actual values (use with caution in CI/CD). +**Value Masking**: Sensitive values are automatically masked by default. Use `--show-values` flag to display actual values (use with caution in CI/CD). ## 📚 Documentation ### Getting Started -- [🌐 Published Package Docs](https://extended-data.dev/packages/secretssync/) - Public package overview, installation paths, and Python integration guidance - [🚀 Getting Started Guide](./docs/GETTING_STARTED.md) - Step-by-step setup tutorial - [❓ FAQ](./docs/FAQ.md) - Frequently asked questions - [📋 Examples](./examples/) - Complete configuration examples ### Core Documentation - [🏗️ Architecture Overview](./docs/ARCHITECTURE.md) - System design and components +- [🔎 Architecture Audit](./docs/ARCHITECTURE_AUDIT.md) - Current implementation and release-contract status - [🔄 Two-Phase Pipeline](./docs/TWO_PHASE_ARCHITECTURE.md) - Merge → Sync architecture - [⚙️ Pipeline Configuration](./docs/PIPELINE.md) - Configuration reference - [🚀 Deployment Guide](./docs/DEPLOYMENT.md) - Production deployment patterns @@ -365,15 +369,27 @@ See [GitHub Actions documentation](./docs/GITHUB_ACTIONS.md) for complete usage - [🛡️ Security Policy](./SECURITY.md) - Security reporting - [📜 Code of Conduct](./CODE_OF_CONDUCT.md) - Community guidelines -## Helm Deployment +## Kubernetes -```bash -# Add Helm repo -helm repo add secretsync https://jbcom.github.io/secrets-sync +Run SecretSync as a scheduled pipeline runner. See +[docs/DEPLOYMENT.md](./docs/DEPLOYMENT.md) for a complete CronJob example. -# Install -helm install secretsync secretsync/secretsync \ - --set vault.address=https://vault.example.com +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: secretsync +spec: + schedule: "*/30 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: secretsync + image: jbcom/secretssync:v1 + args: ["pipeline", "--config", "/config/config.yaml", "--diff", "--output", "json"] ``` ## Docker @@ -381,7 +397,7 @@ helm install secretsync secretsync/secretsync \ ```bash # Run with config file docker run -v $(pwd)/config.yaml:/config.yaml \ - jbcom/secrets-sync-secretssync pipeline --config /config.yaml + jbcom/secretssync:v1 pipeline --config /config.yaml # Multi-arch images available: linux/amd64, linux/arm64 ``` @@ -455,6 +471,9 @@ cd secrets-sync # Build go build ./... +# Vulnerability scan +go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... + # Unit tests go test ./... @@ -505,7 +524,7 @@ For detailed documentation, see [tests/integration/README.md](./tests/integratio ## 🌟 Community & Support ### Getting Help -- **📚 Documentation**: Start with the [published package docs](https://extended-data.dev/packages/secretssync/) and the repo-local [docs folder](./docs/) +- **📚 Documentation**: Start with the repo-local [docs folder](./docs/) - **🐛 GitHub Issues**: Questions, bug reports, and feature requests - **🔒 Security**: Private security vulnerability reporting diff --git a/SECURITY.md b/SECURITY.md index 219f64b..bf899d8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,10 +6,8 @@ We actively support the following versions of SecretSync with security updates: | Version | Supported | | ------- | ------------------ | -| 1.2.x | ✅ Yes | -| 1.1.x | ✅ Yes | -| 1.0.x | ⚠️ Critical fixes only | -| < 1.0 | ❌ No | +| 2.x | ✅ Yes | +| < 2.0 | ❌ No | ## Reporting a Vulnerability @@ -113,7 +111,7 @@ We follow responsible disclosure practices: Security updates are released as: -- **Patch releases** for supported versions (e.g., 1.2.1 → 1.2.2) +- **Patch releases** for supported versions (e.g., 2.0.1 → 2.0.2) - **Security advisories** published on GitHub - **Release notes** highlighting security fixes diff --git a/action.yml b/action.yml index 2211101..f994cca 100644 --- a/action.yml +++ b/action.yml @@ -2,6 +2,80 @@ name: "SecretSync" author: "jbcom" description: "Sync secrets from HashiCorp Vault to AWS Secrets Manager across multiple accounts" +inputs: + config: + description: "Path to SecretSync configuration file" + required: false + default: "config.yaml" + targets: + description: "Comma-separated list of targets to process" + required: false + default: "" + dry-run: + description: "Run without making changes" + required: false + default: "false" + merge-only: + description: "Only run the merge phase" + required: false + default: "false" + sync-only: + description: "Only run the sync phase" + required: false + default: "false" + discover: + description: "Enable dynamic target discovery from AWS Organizations/Identity Center" + required: false + default: "false" + output-format: + description: "Output format: human, json, github, compact, side-by-side" + required: false + default: "github" + compute-diff: + description: "Compute and show diff even when not in dry-run mode" + required: false + default: "false" + exit-code: + description: "Use exit codes: 0=no changes, 1=changes, 2=errors" + required: false + default: "false" + continue-on-error: + description: "Continue processing remaining targets after an error" + required: false + default: "true" + parallelism: + description: "Maximum concurrent target operations; 0 uses config/default" + required: false + default: "0" + metrics-addr: + description: "Metrics server bind address" + required: false + default: "0.0.0.0" + metrics-port: + description: "Metrics server port; 0 disables metrics" + required: false + default: "0" + log-level: + description: "Logging level: debug, info, warn, error" + required: false + default: "info" + log-format: + description: "Log format: text or json" + required: false + default: "text" +outputs: + changes: + description: "Total changed secrets when diff output is computed" + added: + description: "Secrets that would be added or were added" + removed: + description: "Secrets that would be removed or were removed" + modified: + description: "Secrets that would be modified or were modified" + unchanged: + description: "Secrets with no detected changes" + zero_sum: + description: "true when the computed diff has no changes" runs: using: "docker" # Prefer a digest-pinned image once the release workflow can refresh the @@ -11,12 +85,38 @@ runs: # 3. image: "docker://jbcom/secretssync:v1@sha256:" # Until then, keep the release-please-managed tag reference below. image: "docker://jbcom/secretssync:v1" # x-release-please-version + args: + - "pipeline" + - "--config" + - "${{ inputs.config }}" + - "--targets" + - "${{ inputs.targets }}" + - "--dry-run=${{ inputs.dry-run }}" + - "--merge-only=${{ inputs.merge-only }}" + - "--sync-only=${{ inputs.sync-only }}" + - "--discover=${{ inputs.discover }}" + - "--output" + - "${{ inputs.output-format }}" + - "--diff=${{ inputs.compute-diff }}" + - "--exit-code=${{ inputs.exit-code }}" + - "--continue-on-error=${{ inputs.continue-on-error }}" + - "--parallelism" + - "${{ inputs.parallelism }}" + - "--metrics-addr" + - "${{ inputs.metrics-addr }}" + - "--metrics-port" + - "${{ inputs.metrics-port }}" + - "--log-level" + - "${{ inputs.log-level }}" + - "--log-format" + - "${{ inputs.log-format }}" branding: icon: "lock" color: "blue" -# Configuration via environment variables (GitHub Actions inputs map to env vars) -# See https://github.com/jbcom/secrets-sync#configuration for full documentation +# Action inputs are passed to the Docker action as explicit CLI flags. +# The CLI also supports SECRETSYNC_* environment variables for direct container +# use; see https://github.com/jbcom/secrets-sync#configuration for full docs. # # Required: # SECRETSYNC_CONFIG - Path to config file (default: config.yaml) @@ -27,7 +127,7 @@ branding: # SECRETSYNC_MERGE_ONLY - Only run merge phase (default: false) # SECRETSYNC_SYNC_ONLY - Only run sync phase (default: false) # SECRETSYNC_DISCOVER - Enable dynamic target discovery (default: false) -# SECRETSYNC_OUTPUT - Output format: human, json, github (default: github) +# SECRETSYNC_OUTPUT - Output format: human, json, github, compact, side-by-side (default: github) # SECRETSYNC_DIFF - Show diff even without dry-run (default: false) # SECRETSYNC_EXIT_CODE - Use exit codes for CI (default: false) # SECRETSYNC_LOG_LEVEL - Log level: debug, info, warn, error (default: info) @@ -40,4 +140,4 @@ branding: # VAULT_SECRET_ID - AppRole secret ID # # AWS Authentication: -# Use aws-actions/configure-aws-credentials@v4 before this action for OIDC +# Use aws-actions/configure-aws-credentials pinned to a reviewed commit before this action for OIDC diff --git a/action_docs_test.go b/action_docs_test.go new file mode 100644 index 0000000..67bf0b6 --- /dev/null +++ b/action_docs_test.go @@ -0,0 +1,205 @@ +package secretsync_test + +import ( + "os" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +type actionMetadata struct { + Inputs map[string]actionInput `yaml:"inputs"` + Outputs map[string]actionOutput `yaml:"outputs"` +} + +type actionInput struct { + Default string `yaml:"default"` +} + +type actionOutput struct { + Description string `yaml:"description"` +} + +func TestActionInputDocsMatchMetadata(t *testing.T) { + actionInputs := readActionInputDefaults(t) + + for _, doc := range []struct { + path string + heading string + }{ + {"docs/GITHUB_ACTIONS.md", "## Input Parameters"}, + {"docs/ACTION_QUICK_REFERENCE.md", "## All Inputs"}, + } { + docInputs := readDocumentedInputDefaults(t, doc.path, doc.heading) + if diff := compareInputDefaults(actionInputs, docInputs); len(diff) > 0 { + t.Fatalf("%s action input table must match action.yml:\n%s", doc.path, strings.Join(diff, "\n")) + } + } +} + +func TestActionOutputDocsMatchMetadata(t *testing.T) { + metadata := readActionMetadata(t) + if len(metadata.Outputs) == 0 { + t.Fatal("action.yml should declare outputs") + } + + for _, path := range []string{"docs/GITHUB_ACTIONS.md", "docs/ACTION_QUICK_REFERENCE.md"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + text := string(content) + for name := range metadata.Outputs { + if !strings.Contains(text, "`"+name+"`") { + t.Fatalf("%s should document action output %q", path, name) + } + } + } +} + +func readActionInputDefaults(t *testing.T) map[string]string { + t.Helper() + + metadata := readActionMetadata(t) + inputs := make(map[string]string, len(metadata.Inputs)) + for name, input := range metadata.Inputs { + inputs[name] = input.Default + } + return inputs +} + +func readActionMetadata(t *testing.T) actionMetadata { + t.Helper() + + content, err := os.ReadFile("action.yml") + if err != nil { + t.Fatalf("read action.yml: %v", err) + } + + var metadata actionMetadata + if err := yaml.Unmarshal(content, &metadata); err != nil { + t.Fatalf("parse action.yml: %v", err) + } + return metadata +} + +func readDocumentedInputDefaults(t *testing.T, path string, heading string) map[string]string { + t.Helper() + + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + lines := strings.Split(string(content), "\n") + inSection := false + inputColumn := -1 + defaultColumn := -1 + inputs := map[string]string{} + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == heading { + inSection = true + continue + } + if !inSection { + continue + } + if strings.HasPrefix(trimmed, "## ") { + break + } + if !strings.HasPrefix(trimmed, "|") { + continue + } + + cells := splitMarkdownTableRow(trimmed) + if len(cells) == 0 || isMarkdownSeparatorRow(cells) { + continue + } + if inputColumn == -1 || defaultColumn == -1 { + for index, cell := range cells { + switch strings.ToLower(strings.TrimSpace(cell)) { + case "input": + inputColumn = index + case "default": + defaultColumn = index + } + } + continue + } + if len(cells) <= inputColumn || len(cells) <= defaultColumn { + continue + } + + inputName := trimMarkdownCode(cells[inputColumn]) + if inputName == "" { + continue + } + inputs[inputName] = normalizeDefaultCell(cells[defaultColumn]) + } + + if len(inputs) == 0 { + t.Fatalf("%s has no action input table under %q", path, heading) + } + return inputs +} + +func splitMarkdownTableRow(row string) []string { + trimmed := strings.Trim(row, "|") + parts := strings.Split(trimmed, "|") + for index, part := range parts { + parts[index] = strings.TrimSpace(part) + } + return parts +} + +func isMarkdownSeparatorRow(cells []string) bool { + for _, cell := range cells { + if strings.Trim(cell, "-: ") != "" { + return false + } + } + return true +} + +func trimMarkdownCode(value string) string { + return strings.Trim(strings.TrimSpace(value), "`") +} + +func normalizeDefaultCell(value string) string { + defaultValue := strings.TrimSpace(value) + if strings.HasPrefix(defaultValue, "`") { + withoutOpeningTick := strings.TrimPrefix(defaultValue, "`") + if codeSpan, _, ok := strings.Cut(withoutOpeningTick, "`"); ok { + defaultValue = codeSpan + } + } else { + defaultValue = trimMarkdownCode(defaultValue) + } + if before, _, ok := strings.Cut(defaultValue, " "); ok { + defaultValue = before + } + return strings.Trim(defaultValue, `"`) +} + +func compareInputDefaults(want map[string]string, got map[string]string) []string { + var diff []string + for name, defaultValue := range want { + documented, ok := got[name] + if !ok { + diff = append(diff, "missing input: "+name) + continue + } + if documented != defaultValue { + diff = append(diff, name+": documented default "+documented+" != action default "+defaultValue) + } + } + for name := range got { + if _, ok := want[name]; !ok { + diff = append(diff, "extra input: "+name) + } + } + return diff +} diff --git a/api/v1alpha1/register.go b/api/v1alpha1/register.go deleted file mode 100644 index 69ffff5..0000000 --- a/api/v1alpha1/register.go +++ /dev/null @@ -1,37 +0,0 @@ -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: "secretsync.extendeddata.dev", Version: "v1alpha1"} - -// Kind takes an unqualified kind and returns back a Group qualified GroupKind -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -var ( - // SchemeBuilder initializes a scheme builder - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - // AddToScheme is a global function that registers this API group & version to a scheme - AddToScheme = SchemeBuilder.AddToScheme -) - -// Adds the list of known types to Scheme. -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &SecretSync{}, - &SecretSyncList{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil -} diff --git a/api/v1alpha1/secretsync_types.go b/api/v1alpha1/secretsync_types.go deleted file mode 100644 index 48cd477..0000000 --- a/api/v1alpha1/secretsync_types.go +++ /dev/null @@ -1,148 +0,0 @@ -// +k8s:deepcopy-gen=package -// +groupName=secretsync.extendeddata.dev -package v1alpha1 - -import ( - "github.com/jbcom/secrets-sync/pkg/client/aws" - "github.com/jbcom/secrets-sync/pkg/client/vault" - "github.com/jbcom/secrets-sync/pkg/discovery/identitycenter" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// +genclient - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// +kubebuilder:resource:path=secretsyncs,scope=Namespaced,shortName=ss -// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.status`,description="Current status of the SecretSync" -// +kubebuilder:printcolumn:name="SyncDestinations",type=integer,JSONPath=`.status.syncDestinations`,description="Number of destinations synced" - -// SecretSync is the Schema for the secretsyncs API -type SecretSync struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec SecretSyncSpec `json:"spec,omitempty"` - Status SecretSyncStatus `json:"status,omitempty"` -} - -type NotificationEvent string - -const ( - NotificationEventSyncSuccess NotificationEvent = "success" - NotificationEventSyncFailure NotificationEvent = "failure" -) - -type StoreConfig struct { - AWS *aws.AwsClient `json:"aws,omitempty" yaml:"aws,omitempty"` - IdentityCenter *identitycenter.IdentityCenterClient `json:"awsIdentityCenter,omitempty" yaml:"awsIdentityCenter,omitempty"` - Vault *vault.VaultClient `json:"vault,omitempty" yaml:"vault,omitempty"` -} - -type RegexpFilterConfig struct { - Include []string `json:"include,omitempty" yaml:"include,omitempty"` - Exclude []string `json:"exclude,omitempty" yaml:"exclude,omitempty"` -} - -type PathFilterConfig struct { - Include []string `json:"include,omitempty" yaml:"include,omitempty"` - Exclude []string `json:"exclude,omitempty" yaml:"exclude,omitempty"` -} - -type FilterConfig struct { - Regex *RegexpFilterConfig `json:"regex,omitempty" yaml:"regex,omitempty"` - Path *PathFilterConfig `json:"path,omitempty" yaml:"path,omitempty"` -} - -type RenameTransform struct { - From string `json:"from"` - To string `json:"to"` -} - -type TransformSpec struct { - Include []string `yaml:"include,omitempty" json:"include,omitempty"` - Exclude []string `yaml:"exclude,omitempty" json:"exclude,omitempty"` - Rename []RenameTransform `json:"rename,omitempty"` - Template *string `json:"template,omitempty"` -} - -// Webhook represents the configuration for a webhook. -type WebhookNotification struct { - Events []NotificationEvent `json:"events"` - URL string `yaml:"url,omitempty" json:"url,omitempty"` - Method string `yaml:"method,omitempty" json:"method,omitempty"` - Headers map[string]string `yaml:"headers,omitempty" json:"headers,omitempty"` - HeaderSecret *string `yaml:"headerSecret,omitempty" json:"headerSecret,omitempty"` - Body string `yaml:"body,omitempty" json:"body,omitempty"` - ExcludeBody bool `yaml:"excludeBody,omitempty" json:"excludeBody,omitempty"` -} - -type EmailNotification struct { - Events []NotificationEvent `json:"events"` - To string `yaml:"to,omitempty" json:"to,omitempty"` - From string `yaml:"from,omitempty" json:"from,omitempty"` - Subject string `yaml:"subject,omitempty" json:"subject,omitempty"` - Body string `yaml:"body,omitempty" json:"body,omitempty"` - - Host string `yaml:"host,omitempty" json:"host,omitempty"` - Port int `yaml:"port,omitempty" json:"port,omitempty"` - Username string `yaml:"username,omitempty" json:"username,omitempty"` - Password string `yaml:"password,omitempty" json:"password,omitempty"` - InsecureSkipVerify bool `yaml:"insecureSkipVerify,omitempty" json:"insecureSkipVerify,omitempty"` -} - -type SlackNotification struct { - Events []NotificationEvent `json:"events"` - URL *string `yaml:"url,omitempty" json:"url,omitempty"` - URLSecret *string `yaml:"urlSecret,omitempty" json:"urlSecret,omitempty"` - URLSecretKey *string `yaml:"urlSecretKey,omitempty" json:"urlSecretKey,omitempty"` - Body string `yaml:"body,omitempty" json:"body,omitempty"` -} - -type NotificationMessage struct { - Event NotificationEvent `json:"event"` - Message string `json:"message"` - SecretSync SecretSync `json:"secretSync"` -} - -type NotificationSpec struct { - Webhook *WebhookNotification `json:"webhook,omitempty"` - Email *EmailNotification `json:"email,omitempty"` - Slack *SlackNotification `json:"slack,omitempty"` -} - -// +kubebuilder:object:generate=true - -// SecretSyncSpec defines the desired state of SecretSync -type SecretSyncSpec struct { - Source *vault.VaultClient `yaml:"source" json:"source"` - Dest []*StoreConfig `yaml:"dest" json:"dest"` - SyncDelete *bool `yaml:"syncDelete,omitempty" json:"syncDelete,omitempty"` - DryRun *bool `yaml:"dryRun,omitempty" json:"dryRun,omitempty"` - Suspend *bool `yaml:"suspend,omitempty" json:"suspend,omitempty"` - Filters *FilterConfig `yaml:"filters,omitempty" json:"filters,omitempty"` - Transforms *TransformSpec `json:"transforms,omitempty"` - Notifications []*NotificationSpec `json:"notifications,omitempty"` - NotificationsTemplate *string `json:"notificationsTemplate,omitempty"` -} - -// +kubebuilder:object:generate=true - -// SecretSyncStatus defines the observed state of SecretSync -type SecretSyncStatus struct { - Status string `json:"status,omitempty"` - LastSyncTime metav1.Time `json:"lastSyncTime,omitempty"` - SyncDestinations int `json:"syncDestinations,omitempty"` - Hash string `json:"hash,omitempty"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// +kubebuilder:object:root=true - -// SecretSyncList contains a list of SecretSync -type SecretSyncList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []SecretSync `json:"items"` -} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index 58d7596..0000000 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,440 +0,0 @@ -//go:build !ignore_autogenerated - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EmailNotification) DeepCopyInto(out *EmailNotification) { - *out = *in - if in.Events != nil { - in, out := &in.Events, &out.Events - *out = make([]NotificationEvent, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmailNotification. -func (in *EmailNotification) DeepCopy() *EmailNotification { - if in == nil { - return nil - } - out := new(EmailNotification) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FilterConfig) DeepCopyInto(out *FilterConfig) { - *out = *in - if in.Regex != nil { - in, out := &in.Regex, &out.Regex - *out = new(RegexpFilterConfig) - (*in).DeepCopyInto(*out) - } - if in.Path != nil { - in, out := &in.Path, &out.Path - *out = new(PathFilterConfig) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FilterConfig. -func (in *FilterConfig) DeepCopy() *FilterConfig { - if in == nil { - return nil - } - out := new(FilterConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NotificationMessage) DeepCopyInto(out *NotificationMessage) { - *out = *in - in.SecretSync.DeepCopyInto(&out.SecretSync) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotificationMessage. -func (in *NotificationMessage) DeepCopy() *NotificationMessage { - if in == nil { - return nil - } - out := new(NotificationMessage) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NotificationSpec) DeepCopyInto(out *NotificationSpec) { - *out = *in - if in.Webhook != nil { - in, out := &in.Webhook, &out.Webhook - *out = new(WebhookNotification) - (*in).DeepCopyInto(*out) - } - if in.Email != nil { - in, out := &in.Email, &out.Email - *out = new(EmailNotification) - (*in).DeepCopyInto(*out) - } - if in.Slack != nil { - in, out := &in.Slack, &out.Slack - *out = new(SlackNotification) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotificationSpec. -func (in *NotificationSpec) DeepCopy() *NotificationSpec { - if in == nil { - return nil - } - out := new(NotificationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PathFilterConfig) DeepCopyInto(out *PathFilterConfig) { - *out = *in - if in.Include != nil { - in, out := &in.Include, &out.Include - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Exclude != nil { - in, out := &in.Exclude, &out.Exclude - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PathFilterConfig. -func (in *PathFilterConfig) DeepCopy() *PathFilterConfig { - if in == nil { - return nil - } - out := new(PathFilterConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RegexpFilterConfig) DeepCopyInto(out *RegexpFilterConfig) { - *out = *in - if in.Include != nil { - in, out := &in.Include, &out.Include - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Exclude != nil { - in, out := &in.Exclude, &out.Exclude - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegexpFilterConfig. -func (in *RegexpFilterConfig) DeepCopy() *RegexpFilterConfig { - if in == nil { - return nil - } - out := new(RegexpFilterConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RenameTransform) DeepCopyInto(out *RenameTransform) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RenameTransform. -func (in *RenameTransform) DeepCopy() *RenameTransform { - if in == nil { - return nil - } - out := new(RenameTransform) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SlackNotification) DeepCopyInto(out *SlackNotification) { - *out = *in - if in.Events != nil { - in, out := &in.Events, &out.Events - *out = make([]NotificationEvent, len(*in)) - copy(*out, *in) - } - if in.URL != nil { - in, out := &in.URL, &out.URL - *out = new(string) - **out = **in - } - if in.URLSecret != nil { - in, out := &in.URLSecret, &out.URLSecret - *out = new(string) - **out = **in - } - if in.URLSecretKey != nil { - in, out := &in.URLSecretKey, &out.URLSecretKey - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SlackNotification. -func (in *SlackNotification) DeepCopy() *SlackNotification { - if in == nil { - return nil - } - out := new(SlackNotification) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoreConfig) DeepCopyInto(out *StoreConfig) { - *out = *in - if in.AWS != nil { - in, out := &in.AWS, &out.AWS - *out = (*in).DeepCopy() - } - if in.IdentityCenter != nil { - in, out := &in.IdentityCenter, &out.IdentityCenter - *out = (*in).DeepCopy() - } - if in.Vault != nil { - in, out := &in.Vault, &out.Vault - *out = (*in).DeepCopy() - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoreConfig. -func (in *StoreConfig) DeepCopy() *StoreConfig { - if in == nil { - return nil - } - out := new(StoreConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TransformSpec) DeepCopyInto(out *TransformSpec) { - *out = *in - if in.Include != nil { - in, out := &in.Include, &out.Include - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Exclude != nil { - in, out := &in.Exclude, &out.Exclude - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Rename != nil { - in, out := &in.Rename, &out.Rename - *out = make([]RenameTransform, len(*in)) - copy(*out, *in) - } - if in.Template != nil { - in, out := &in.Template, &out.Template - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TransformSpec. -func (in *TransformSpec) DeepCopy() *TransformSpec { - if in == nil { - return nil - } - out := new(TransformSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SecretSync) DeepCopyInto(out *SecretSync) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSync. -func (in *SecretSync) DeepCopy() *SecretSync { - if in == nil { - return nil - } - out := new(SecretSync) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *SecretSync) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SecretSyncList) DeepCopyInto(out *SecretSyncList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]SecretSync, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSyncList. -func (in *SecretSyncList) DeepCopy() *SecretSyncList { - if in == nil { - return nil - } - out := new(SecretSyncList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *SecretSyncList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SecretSyncSpec) DeepCopyInto(out *SecretSyncSpec) { - *out = *in - if in.Source != nil { - in, out := &in.Source, &out.Source - *out = (*in).DeepCopy() - } - if in.Dest != nil { - in, out := &in.Dest, &out.Dest - *out = make([]*StoreConfig, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = new(StoreConfig) - (*in).DeepCopyInto(*out) - } - } - } - if in.SyncDelete != nil { - in, out := &in.SyncDelete, &out.SyncDelete - *out = new(bool) - **out = **in - } - if in.DryRun != nil { - in, out := &in.DryRun, &out.DryRun - *out = new(bool) - **out = **in - } - if in.Suspend != nil { - in, out := &in.Suspend, &out.Suspend - *out = new(bool) - **out = **in - } - if in.Filters != nil { - in, out := &in.Filters, &out.Filters - *out = new(FilterConfig) - (*in).DeepCopyInto(*out) - } - if in.Transforms != nil { - in, out := &in.Transforms, &out.Transforms - *out = new(TransformSpec) - (*in).DeepCopyInto(*out) - } - if in.Notifications != nil { - in, out := &in.Notifications, &out.Notifications - *out = make([]*NotificationSpec, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = new(NotificationSpec) - (*in).DeepCopyInto(*out) - } - } - } - if in.NotificationsTemplate != nil { - in, out := &in.NotificationsTemplate, &out.NotificationsTemplate - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSyncSpec. -func (in *SecretSyncSpec) DeepCopy() *SecretSyncSpec { - if in == nil { - return nil - } - out := new(SecretSyncSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SecretSyncStatus) DeepCopyInto(out *SecretSyncStatus) { - *out = *in - in.LastSyncTime.DeepCopyInto(&out.LastSyncTime) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSyncStatus. -func (in *SecretSyncStatus) DeepCopy() *SecretSyncStatus { - if in == nil { - return nil - } - out := new(SecretSyncStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WebhookNotification) DeepCopyInto(out *WebhookNotification) { - *out = *in - if in.Events != nil { - in, out := &in.Events, &out.Events - *out = make([]NotificationEvent, len(*in)) - copy(*out, *in) - } - if in.Headers != nil { - in, out := &in.Headers, &out.Headers - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.HeaderSecret != nil { - in, out := &in.HeaderSecret, &out.HeaderSecret - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookNotification. -func (in *WebhookNotification) DeepCopy() *WebhookNotification { - if in == nil { - return nil - } - out := new(WebhookNotification) - in.DeepCopyInto(out) - return out -} diff --git a/cmd/secretsync/cmd/context.go b/cmd/secretsync/cmd/context.go index 66a20e5..6d1ae90 100644 --- a/cmd/secretsync/cmd/context.go +++ b/cmd/secretsync/cmd/context.go @@ -25,8 +25,8 @@ This shows: Understanding your execution context is critical for multi-account operations. Examples: - vss context - vss context --config config.yaml`, + secretsync context + secretsync context --config config.yaml`, RunE: runContext, } diff --git a/cmd/secretsync/cmd/graph.go b/cmd/secretsync/cmd/graph.go index d132f32..6e7c9dd 100644 --- a/cmd/secretsync/cmd/graph.go +++ b/cmd/secretsync/cmd/graph.go @@ -21,8 +21,8 @@ The graph shows: - Execution order (by dependency level) Examples: - vss graph --config config.yaml - vss graph --config config.yaml --format dot`, + secretsync graph --config config.yaml + secretsync graph --config config.yaml --format dot`, RunE: runGraph, } diff --git a/cmd/secretsync/cmd/help_text_test.go b/cmd/secretsync/cmd/help_text_test.go new file mode 100644 index 0000000..416db18 --- /dev/null +++ b/cmd/secretsync/cmd/help_text_test.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "strings" + "testing" +) + +func TestCommandHelpDoesNotAdvertiseVSSAlias(t *testing.T) { + commands := []*cobraCommandText{ + {name: "context", long: contextCmd.Long}, + {name: "graph", long: graphCmd.Long}, + {name: "migrate", long: migrateCmd.Long}, + {name: "validate", long: validateCmd.Long}, + } + + for _, command := range commands { + t.Run(command.name, func(t *testing.T) { + if strings.Contains(command.long, "vss ") || strings.Contains(command.long, "./vss") { + t.Fatalf("%s help should advertise secretsync, not vss:\n%s", command.name, command.long) + } + if !strings.Contains(command.long, "secretsync ") { + t.Fatalf("%s help should include secretsync examples:\n%s", command.name, command.long) + } + }) + } +} + +type cobraCommandText struct { + name string + long string +} diff --git a/cmd/secretsync/cmd/migrate.go b/cmd/secretsync/cmd/migrate.go index edd9fc5..5167bce 100644 --- a/cmd/secretsync/cmd/migrate.go +++ b/cmd/secretsync/cmd/migrate.go @@ -24,17 +24,17 @@ var ( var migrateCmd = &cobra.Command{ Use: "migrate", Short: "Migrate from other secret management tools", - Long: `Migrate configuration from other secret management tools to vss pipeline format. + Long: `Migrate configuration from other secret management tools to SecretSync pipeline format. Supported sources: - terraform-secretsmanager: Terraform-based AWS Secrets Manager pipeline Example: - vss migrate --from terraform-secretsmanager \ - --targets config/targets.yaml \ - --secrets config/secrets.yaml \ - --accounts config/accounts.yaml \ - --output config.yaml`, + secretsync migrate --from terraform-secretsmanager \ + --targets config/targets.yaml \ + --secrets config/secrets.yaml \ + --accounts config/accounts.yaml \ + --output config.yaml`, RunE: runMigrate, } @@ -214,7 +214,7 @@ func migrateTerraformSecretManager() error { // Add header comment header := `# Pipeline configuration migrated from terraform-aws-secretsmanager -# Generated by: vss migrate --from terraform-secretsmanager +# Generated by: secretsync migrate --from terraform-secretsmanager # # Review and adjust as needed: # - Verify Vault address and authentication @@ -236,8 +236,8 @@ func migrateTerraformSecretManager() error { fmt.Println("Next steps:") fmt.Printf(" 1. Review the generated config: %s\n", outputFile) fmt.Println(" 2. Add Vault authentication (token, approle, etc.)") - fmt.Println(" 3. Validate: vss validate --config " + outputFile) - fmt.Println(" 4. Dry run: vss pipeline --config " + outputFile + " --dry-run") + fmt.Println(" 3. Validate: secretsync validate --config " + outputFile) + fmt.Println(" 4. Dry run: secretsync pipeline --config " + outputFile + " --dry-run") return nil } diff --git a/cmd/secretsync/cmd/pipeline.go b/cmd/secretsync/cmd/pipeline.go index cbd61ae..c00a2fb 100644 --- a/cmd/secretsync/cmd/pipeline.go +++ b/cmd/secretsync/cmd/pipeline.go @@ -2,12 +2,15 @@ package cmd import ( "context" + "encoding/json" "fmt" "os" "os/signal" + "regexp" "sort" "strings" "syscall" + "time" "github.com/jbcom/secrets-sync/pkg/diff" "github.com/jbcom/secrets-sync/pkg/pipeline" @@ -24,6 +27,14 @@ var ( outputFormat string computeDiff bool exitCodeMode bool + continueOnError bool + parallelism int +) + +var ( + bearerTokenPattern = regexp.MustCompile(`(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+`) + sensitiveAssignmentPattern = regexp.MustCompile(`(?i)\b(password|passwd|secret|secret_id|client_secret|api[_-]?key|access[_-]?key|token|authorization)(\s*[:=]\s*)(Bearer\s+\[REDACTED\]|\[[^\]]+\]|"[^"]*"|'[^']*'|[^\s,;}\]]+)`) + sensitiveURLParameterPattern = regexp.MustCompile(`(?i)([?&](?:password|passwd|secret|secret_id|client_secret|api[_-]?key|access[_-]?key|token|authorization)=)(\[[^\]]+\]|[^&#\s]+)`) ) // pipelineCmd runs the full merge-then-sync pipeline @@ -39,18 +50,18 @@ var pipelineCmd = &cobra.Command{ 2. SYNC PHASE: Sync merged secrets to target AWS accounts - Assumes Control Tower execution role in each account - - Runs in parallel (respects --parallel setting) + - Runs in parallel (respects --parallelism or config settings) 3. DIFF REPORTING: Track and report all changes - Zero-sum validation for migration verification - - Multiple output formats (human, JSON, GitHub Actions) + - Multiple output formats (human, JSON, GitHub Actions, compact, side-by-side) - CI/CD-friendly exit codes (0=no changes, 1=changes, 2=errors) Examples: # Full pipeline secretsync pipeline --config config.yaml - # Dry run with diff output (validates zero-sum) + # Dry run with machine-readable result and nested diff output secretsync pipeline --config config.yaml --dry-run --output json # CI/CD mode with exit codes @@ -60,6 +71,9 @@ Examples: # GitHub Actions compatible output secretsync pipeline --config config.yaml --dry-run --output github + # Visual side-by-side diff output + secretsync pipeline --config config.yaml --dry-run --output side-by-side + # Specific targets only secretsync pipeline --config config.yaml --targets "Serverless_Stg,Serverless_Prod" @@ -79,9 +93,11 @@ func init() { pipelineCmd.Flags().BoolVar(&syncOnly, "sync-only", false, "only run sync phase") pipelineCmd.Flags().BoolVar(&dryRun, "dry-run", false, "dry run mode (no changes)") pipelineCmd.Flags().BoolVar(&discoverTargets, "discover", false, "enable dynamic target discovery from AWS Organizations/Identity Center") + pipelineCmd.Flags().BoolVar(&continueOnError, "continue-on-error", true, "continue processing remaining targets after an error") + pipelineCmd.Flags().IntVar(¶llelism, "parallelism", 0, "max concurrent target operations (default: pipeline.merge.parallel config or 4)") // Diff and output options - pipelineCmd.Flags().StringVarP(&outputFormat, "output", "o", "human", "output format: human, json, github, compact") + pipelineCmd.Flags().StringVarP(&outputFormat, "output", "o", "human", "output format: human, json, github, compact, side-by-side") pipelineCmd.Flags().BoolVar(&computeDiff, "diff", false, "compute and show diff even when not in dry-run mode") pipelineCmd.Flags().BoolVar(&exitCodeMode, "exit-code", false, "use exit codes: 0=no changes, 1=changes, 2=errors (useful for CI/CD)") } @@ -144,7 +160,8 @@ func runPipeline(cmd *cobra.Command, args []string) error { Operation: op, Targets: targetList, DryRun: dryRun, - ContinueOnError: true, + ContinueOnError: continueOnError, + Parallelism: parallelism, OutputFormat: format, ComputeDiff: computeDiff || dryRun, } @@ -158,11 +175,26 @@ func runPipeline(cmd *cobra.Command, args []string) error { }).Info("Starting pipeline") // Run pipeline + start := time.Now() results, err := p.Run(ctx, opts) + duration := time.Since(start) - // Print diff output if computed - if d := p.Diff(); d != nil { - diffOutput := p.FormatDiff(format) + // Print machine JSON as a stable result envelope for both diff and non-diff runs. + pipelineDiff := p.Diff() + diffOutput := "" + if pipelineDiff != nil { + diffOutput = p.FormatDiff(format) + } + if format == diff.OutputFormatGitHub { + if outputErr := writeGitHubDiffOutputs(os.Getenv("GITHUB_OUTPUT"), pipelineDiff); outputErr != nil { + return outputErr + } + } + if format == diff.OutputFormatJSON { + if jsonErr := printPipelineJSONSummary(results, err, duration, diffOutput, pipelineDiff); jsonErr != nil { + return jsonErr + } + } else if pipelineDiff != nil { if diffOutput != "" { fmt.Println(diffOutput) } @@ -172,8 +204,13 @@ func runPipeline(cmd *cobra.Command, args []string) error { } // Determine exit behavior + hasErrors := pipelineHadErrors(err, results) if exitCodeMode { - exitCode := p.ExitCode() + diffExitCode := 0 + if !hasErrors { + diffExitCode = p.ExitCode() + } + exitCode := pipelineExitCode(hasErrors, diffExitCode) if exitCode != 0 { os.Exit(exitCode) } @@ -185,13 +222,65 @@ func runPipeline(cmd *cobra.Command, args []string) error { } // Check for any failures + if hasErrors { + return fmt.Errorf("pipeline completed with errors") + } + + l.Info("Pipeline completed successfully") + return nil +} + +func pipelineHadErrors(err error, results []pipeline.Result) bool { + if err != nil { + return true + } + for _, r := range results { if !r.Success { - return fmt.Errorf("pipeline completed with errors") + return true + } + } + + return false +} + +func pipelineExitCode(hasErrors bool, diffExitCode int) int { + if hasErrors { + return 2 + } + return diffExitCode +} + +func writeGitHubDiffOutputs(outputPath string, pipelineDiff *diff.PipelineDiff) error { + if outputPath == "" || pipelineDiff == nil { + return nil + } + + outputFile, err := os.OpenFile(outputPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("failed to open GitHub output file: %w", err) + } + defer outputFile.Close() + + changed := pipelineDiff.Summary.Added + pipelineDiff.Summary.Removed + pipelineDiff.Summary.Modified + outputs := []struct { + name string + value string + }{ + {name: "changes", value: fmt.Sprint(changed)}, + {name: "added", value: fmt.Sprint(pipelineDiff.Summary.Added)}, + {name: "removed", value: fmt.Sprint(pipelineDiff.Summary.Removed)}, + {name: "modified", value: fmt.Sprint(pipelineDiff.Summary.Modified)}, + {name: "unchanged", value: fmt.Sprint(pipelineDiff.Summary.Unchanged)}, + {name: "zero_sum", value: fmt.Sprintf("%t", pipelineDiff.IsZeroSum())}, + } + + for _, output := range outputs { + if _, err := fmt.Fprintf(outputFile, "%s=%s\n", output.name, output.value); err != nil { + return fmt.Errorf("failed to write GitHub output %q: %w", output.name, err) } } - l.Info("Pipeline completed successfully") return nil } @@ -204,6 +293,8 @@ func parseOutputFormat(s string) diff.OutputFormat { return diff.OutputFormatGitHub case "compact": return diff.OutputFormatCompact + case "side-by-side", "sidebyside", "side_by_side": + return diff.OutputFormatSideBySide default: return diff.OutputFormatHuman } @@ -266,3 +357,124 @@ func printResults(results []pipeline.Result) { fmt.Printf("\nTotal: %d/%d succeeded\n", successCount, len(results)) fmt.Println(strings.Repeat("=", 60)) } + +type pipelineJSONSummary struct { + Success bool `json:"success"` + TargetCount int `json:"target_count"` + SecretsProcessed int `json:"secrets_processed"` + SecretsAdded int `json:"secrets_added"` + SecretsModified int `json:"secrets_modified"` + SecretsRemoved int `json:"secrets_removed"` + SecretsUnchanged int `json:"secrets_unchanged"` + DurationMs int64 `json:"duration_ms"` + ErrorMessage string `json:"error_message,omitempty"` + Results []pipelineJSONItem `json:"results"` + DiffOutput string `json:"diff_output,omitempty"` + Diff *diff.PipelineDiff `json:"diff,omitempty"` +} + +type pipelineJSONItem struct { + Target string `json:"target"` + Phase string `json:"phase"` + Operation string `json:"operation"` + Success bool `json:"success"` + Error string `json:"error,omitempty"` + DurationMs int64 `json:"duration_ms"` + Details pipeline.ResultDetails `json:"details,omitempty"` + Diff *diff.TargetDiff `json:"diff,omitempty"` +} + +func printPipelineJSONSummary( + results []pipeline.Result, + runErr error, + duration time.Duration, + diffOutput string, + pipelineDiff *diff.PipelineDiff, +) error { + payload := newPipelineJSONSummary(results, runErr, duration, diffOutput, pipelineDiff) + encoded, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return fmt.Errorf("failed to encode pipeline JSON output: %w", err) + } + fmt.Println(string(encoded)) + return nil +} + +func newPipelineJSONSummary( + results []pipeline.Result, + runErr error, + duration time.Duration, + diffOutput string, + pipelineDiff *diff.PipelineDiff, +) pipelineJSONSummary { + summary := pipelineJSONSummary{ + Success: runErr == nil, + DurationMs: duration.Milliseconds(), + Results: make([]pipelineJSONItem, 0, len(results)), + DiffOutput: diffOutput, + Diff: pipelineDiff, + } + if runErr != nil { + summary.ErrorMessage = redactPipelineDiagnostic(runErr.Error()) + } + + targetsSeen := make(map[string]struct{}) + for _, result := range results { + if result.Target != "" { + targetsSeen[result.Target] = struct{}{} + } + + summary.SecretsProcessed += result.Details.SecretsProcessed + summary.SecretsAdded += result.Details.SecretsAdded + summary.SecretsModified += result.Details.SecretsModified + summary.SecretsRemoved += result.Details.SecretsRemoved + summary.SecretsUnchanged += result.Details.SecretsUnchanged + + item := pipelineJSONItem{ + Target: result.Target, + Phase: result.Phase, + Operation: result.Operation, + Success: result.Success, + DurationMs: result.Duration.Milliseconds(), + Details: result.Details, + Diff: result.Diff, + } + if result.Error != nil { + item.Error = redactPipelineDiagnostic(result.Error.Error()) + } + summary.Results = append(summary.Results, item) + + if !result.Success { + summary.Success = false + if summary.ErrorMessage == "" { + if item.Error != "" { + summary.ErrorMessage = item.Error + } else { + summary.ErrorMessage = "pipeline completed with errors" + } + } + } + } + + summary.TargetCount = len(targetsSeen) + return summary +} + +func redactPipelineDiagnostic(value string) string { + if value == "" { + return "" + } + + redacted := bearerTokenPattern.ReplaceAllString(value, "Bearer [REDACTED]") + redacted = sensitiveURLParameterPattern.ReplaceAllString(redacted, "${1}[REDACTED]") + return sensitiveAssignmentPattern.ReplaceAllStringFunc(redacted, func(match string) string { + if strings.Contains(match, "[REDACTED]") { + return match + } + parts := sensitiveAssignmentPattern.FindStringSubmatch(match) + if len(parts) != 4 { + return "[REDACTED]" + } + return parts[1] + parts[2] + "[REDACTED]" + }) +} diff --git a/cmd/secretsync/cmd/pipeline_test.go b/cmd/secretsync/cmd/pipeline_test.go new file mode 100644 index 0000000..6e106c0 --- /dev/null +++ b/cmd/secretsync/cmd/pipeline_test.go @@ -0,0 +1,307 @@ +package cmd + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jbcom/secrets-sync/pkg/diff" + "github.com/jbcom/secrets-sync/pkg/pipeline" +) + +func TestParseOutputFormat(t *testing.T) { + tests := map[string]diff.OutputFormat{ + "human": diff.OutputFormatHuman, + "json": diff.OutputFormatJSON, + "github": diff.OutputFormatGitHub, + "compact": diff.OutputFormatCompact, + "side-by-side": diff.OutputFormatSideBySide, + "sidebyside": diff.OutputFormatSideBySide, + "side_by_side": diff.OutputFormatSideBySide, + "unknown": diff.OutputFormatHuman, + } + + for input, expected := range tests { + t.Run(input, func(t *testing.T) { + if actual := parseOutputFormat(input); actual != expected { + t.Fatalf("parseOutputFormat(%q) = %q, want %q", input, actual, expected) + } + }) + } +} + +func TestNewPipelineJSONSummaryAggregatesResults(t *testing.T) { + results := []pipeline.Result{ + { + Target: "prod", + Phase: "merge", + Operation: "merge", + Success: true, + Duration: 1500 * time.Millisecond, + Details: pipeline.ResultDetails{ + SecretsProcessed: 2, + SecretsAdded: 1, + SecretsUnchanged: 1, + }, + }, + { + Target: "prod", + Phase: "sync", + Operation: "sync", + Success: true, + Duration: 2500 * time.Millisecond, + Details: pipeline.ResultDetails{ + SecretsProcessed: 2, + SecretsModified: 1, + SecretsRemoved: 1, + }, + }, + { + Target: "staging", + Phase: "sync", + Operation: "sync", + Success: true, + Duration: time.Second, + Details: pipeline.ResultDetails{ + SecretsProcessed: 1, + SecretsUnchanged: 1, + }, + }, + } + pipelineDiff := &diff.PipelineDiff{ + Summary: diff.ChangeSummary{Added: 1, Modified: 1, Total: 2}, + DryRun: true, + } + + summary := newPipelineJSONSummary(results, nil, 4200*time.Millisecond, `{"summary":{"added":1}}`, pipelineDiff) + + if !summary.Success { + t.Fatal("Success = false, want true") + } + if summary.TargetCount != 2 { + t.Fatalf("TargetCount = %d, want 2", summary.TargetCount) + } + if summary.SecretsProcessed != 5 { + t.Fatalf("SecretsProcessed = %d, want 5", summary.SecretsProcessed) + } + if summary.SecretsAdded != 1 || summary.SecretsModified != 1 || summary.SecretsRemoved != 1 || summary.SecretsUnchanged != 2 { + t.Fatalf("unexpected secret counts: %+v", summary) + } + if summary.DurationMs != 4200 { + t.Fatalf("DurationMs = %d, want 4200", summary.DurationMs) + } + if len(summary.Results) != len(results) { + t.Fatalf("len(Results) = %d, want %d", len(summary.Results), len(results)) + } + if summary.Results[0].DurationMs != 1500 { + t.Fatalf("Results[0].DurationMs = %d, want 1500", summary.Results[0].DurationMs) + } + if summary.Diff == nil { + t.Fatal("Diff = nil, want structured diff") + } + + encoded, err := json.Marshal(summary) + if err != nil { + t.Fatalf("json.Marshal(summary) failed: %v", err) + } + if !strings.Contains(string(encoded), `"target_count":2`) { + t.Fatalf("encoded summary missing target_count: %s", encoded) + } +} + +func TestNewPipelineJSONSummaryReportsFailures(t *testing.T) { + results := []pipeline.Result{ + { + Target: "prod", + Phase: "sync", + Success: false, + Error: errors.New("assume role failed"), + }, + } + + summary := newPipelineJSONSummary(results, nil, time.Second, "", nil) + + if summary.Success { + t.Fatal("Success = true, want false") + } + if summary.ErrorMessage != "assume role failed" { + t.Fatalf("ErrorMessage = %q, want %q", summary.ErrorMessage, "assume role failed") + } + if summary.Results[0].Error != "assume role failed" { + t.Fatalf("Results[0].Error = %q, want %q", summary.Results[0].Error, "assume role failed") + } +} + +func TestNewPipelineJSONSummaryRedactsDiagnosticSecrets(t *testing.T) { + results := []pipeline.Result{ + { + Target: "prod", + Phase: "sync", + Success: false, + Error: errors.New( + "write failed api_key=key_123 Authorization: Bearer raw_token callback=https://example.test/hook?token=tok_456", + ), + }, + } + + summary := newPipelineJSONSummary( + results, + errors.New("pipeline failed password=hunter2 client_secret=secret_123"), + time.Second, + "", + nil, + ) + + if summary.Success { + t.Fatal("Success = true, want false") + } + for _, raw := range []string{"hunter2", "secret_123", "key_123", "raw_token", "tok_456"} { + if strings.Contains(summary.ErrorMessage, raw) { + t.Fatalf("ErrorMessage leaked %q: %s", raw, summary.ErrorMessage) + } + if strings.Contains(summary.Results[0].Error, raw) { + t.Fatalf("Results[0].Error leaked %q: %s", raw, summary.Results[0].Error) + } + } + if !strings.Contains(summary.ErrorMessage, "[REDACTED]") { + t.Fatalf("ErrorMessage missing redaction marker: %s", summary.ErrorMessage) + } + if !strings.Contains(summary.Results[0].Error, "[REDACTED]") { + t.Fatalf("Results[0].Error missing redaction marker: %s", summary.Results[0].Error) + } + if strings.Contains(summary.Results[0].Error, "[REDACTED] [REDACTED]") || strings.Contains(summary.Results[0].Error, "[REDACTED]]") { + t.Fatalf("Results[0].Error should not double-redact already redacted segments: %s", summary.Results[0].Error) + } + + encoded, err := json.Marshal(summary) + if err != nil { + t.Fatalf("json.Marshal(summary) failed: %v", err) + } + for _, raw := range []string{"hunter2", "secret_123", "key_123", "raw_token", "tok_456"} { + if strings.Contains(string(encoded), raw) { + t.Fatalf("encoded summary leaked %q: %s", raw, encoded) + } + } +} + +func TestPipelineHadErrors(t *testing.T) { + tests := map[string]struct { + err error + results []pipeline.Result + want bool + }{ + "run error": { + err: errors.New("connection failed"), + want: true, + }, + "target failure": { + results: []pipeline.Result{{Target: "prod", Success: false}}, + want: true, + }, + "all targets successful": { + results: []pipeline.Result{{Target: "prod", Success: true}}, + want: false, + }, + "no results": { + want: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + if got := pipelineHadErrors(tc.err, tc.results); got != tc.want { + t.Fatalf("pipelineHadErrors() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestPipelineExitCode(t *testing.T) { + tests := map[string]struct { + hasErrors bool + diffExitCode int + want int + }{ + "pipeline errors exit as execution errors": { + hasErrors: true, + diffExitCode: 0, + want: 2, + }, + "execution errors win over changed diff": { + hasErrors: true, + diffExitCode: 1, + want: 2, + }, + "changed diff preserves diff exit code": { + diffExitCode: 1, + want: 1, + }, + "diff errors preserve diff error exit code": { + diffExitCode: 2, + want: 2, + }, + "clean run exits zero": { + want: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + if got := pipelineExitCode(tc.hasErrors, tc.diffExitCode); got != tc.want { + t.Fatalf("pipelineExitCode() = %d, want %d", got, tc.want) + } + }) + } +} + +func TestWriteGitHubDiffOutputs(t *testing.T) { + pipelineDiff := &diff.PipelineDiff{ + Summary: diff.ChangeSummary{ + Added: 2, + Removed: 1, + Modified: 3, + Unchanged: 5, + Total: 11, + }, + } + outputPath := filepath.Join(t.TempDir(), "github_output") + + if err := writeGitHubDiffOutputs(outputPath, pipelineDiff); err != nil { + t.Fatalf("writeGitHubDiffOutputs() failed: %v", err) + } + + content, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read GitHub output file: %v", err) + } + + text := string(content) + for _, expected := range []string{ + "changes=6\n", + "added=2\n", + "removed=1\n", + "modified=3\n", + "unchanged=5\n", + "zero_sum=false\n", + } { + if !strings.Contains(text, expected) { + t.Fatalf("GitHub output missing %q:\n%s", expected, text) + } + } + if strings.Contains(text, "::set-output") { + t.Fatalf("GitHub output file should not contain deprecated commands:\n%s", text) + } +} + +func TestWriteGitHubDiffOutputsNoopsWithoutOutputFile(t *testing.T) { + if err := writeGitHubDiffOutputs("", &diff.PipelineDiff{}); err != nil { + t.Fatalf("writeGitHubDiffOutputs() with empty path failed: %v", err) + } + if err := writeGitHubDiffOutputs(filepath.Join(t.TempDir(), "github_output"), nil); err != nil { + t.Fatalf("writeGitHubDiffOutputs() with nil diff failed: %v", err) + } +} diff --git a/cmd/secretsync/cmd/validate.go b/cmd/secretsync/cmd/validate.go index 519e7c3..b23378b 100644 --- a/cmd/secretsync/cmd/validate.go +++ b/cmd/secretsync/cmd/validate.go @@ -22,8 +22,8 @@ Checks: - AWS execution context (optional) Examples: - vss validate --config config.yaml - vss validate --config config.yaml --check-aws`, + secretsync validate --config config.yaml + secretsync validate --config config.yaml --check-aws`, RunE: runValidate, } diff --git a/deploy/charts/secretsync/Chart.yaml b/deploy/charts/secretsync/Chart.yaml index 6584c4c..4de39c1 100644 --- a/deploy/charts/secretsync/Chart.yaml +++ b/deploy/charts/secretsync/Chart.yaml @@ -4,11 +4,3 @@ description: Universal secrets synchronization pipeline for multi-cloud secret m type: application version: 0.1.0 appVersion: "0.1.0" - -dependencies: - - name: secretsync-events - version: 0.1.0 - condition: secretsync-events.enabled - - name: secretsync-operator - version: 0.1.0 - condition: secretsync-operator.enabled \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-events/.helmignore b/deploy/charts/secretsync/charts/secretsync-events/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deploy/charts/secretsync/charts/secretsync-events/Chart.yaml b/deploy/charts/secretsync/charts/secretsync-events/Chart.yaml deleted file mode 100644 index 5b6eed3..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: secretsync-events -description: SecretSync events server for webhook-based secret synchronization - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/deploy/charts/secretsync/charts/secretsync-events/templates/_helpers.tpl b/deploy/charts/secretsync/charts/secretsync-events/templates/_helpers.tpl deleted file mode 100644 index 2ffbf31..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/templates/_helpers.tpl +++ /dev/null @@ -1,88 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "secretsync-events.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "secretsync-events.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "secretsync-events.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "secretsync-events.labels" -}} -helm.sh/chart: {{ include "secretsync-events.chart" . }} -{{ include "secretsync-events.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "secretsync-events.selectorLabels" -}} -app.kubernetes.io/name: {{ include "secretsync-events.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "secretsync-events.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "secretsync-events.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -Simplify the definition of the event port -*/}} -{{- define "secretsync-events.containerPort" -}} -{{- default "8080" .Values.containerPort }} -{{- end }} - -{{/* -Simplify the definition of the event port -*/}} -{{- define "secretsync-events.metricsPort" -}} -{{- default "9090" .Values.metricsPort }} -{{- end }} - - -{{/* -Create the name of the configMap to use -*/}} -{{- define "secretsync-events.configMapName" -}} -{{- if .Values.existingConfigMap }} -{{- .Values.existingConfigMap -}} -{{- else }} -{{- include "secretsync-events.fullname" . -}} -{{- end }} -{{- end -}} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-events/templates/configmap.yaml b/deploy/charts/secretsync/charts/secretsync-events/templates/configmap.yaml deleted file mode 100644 index 47b9c93..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/templates/configmap.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if (eq .Values.existingConfigMap "") }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "secretsync-events.configMapName" . }} - labels: - {{- include "secretsync-events.labels" . | nindent 4 }} -data: - config.yaml: | - {{- .Values.config | toYaml | nindent 4 }} -{{- end }} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-events/templates/deployment.yaml b/deploy/charts/secretsync/charts/secretsync-events/templates/deployment.yaml deleted file mode 100644 index 84317fa..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/templates/deployment.yaml +++ /dev/null @@ -1,91 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "secretsync-events.fullname" . }} - labels: - {{- include "secretsync-events.labels" . | nindent 4 }} - {{- with .Values.deploymentAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "secretsync-events.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "secretsync-events.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "secretsync-events.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - args: ["-config", "/config/config.yaml", "-events"] - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: {{ include "secretsync-events.containerPort" . }} - name: http - - containerPort: {{ include "secretsync-events.metricsPort" . }} - name: metrics - livenessProbe: - httpGet: - path: /healthz - port: metrics - readinessProbe: - httpGet: - path: /healthz - port: metrics - {{- with .Values.env }} - env: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.envFrom }} - envFrom: - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: config - mountPath: /config - readOnly: true - {{- with .Values.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: config - configMap: - name: {{ include "secretsync-events.configMapName" . }} - defaultMode: 420 - optional: true - {{- with .Values.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-events/templates/hpa.yaml b/deploy/charts/secretsync/charts/secretsync-events/templates/hpa.yaml deleted file mode 100644 index e515a68..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "secretsync-events.fullname" . }} - labels: - {{- include "secretsync-events.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "secretsync-events.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deploy/charts/secretsync/charts/secretsync-events/templates/service.yaml b/deploy/charts/secretsync/charts/secretsync-events/templates/service.yaml deleted file mode 100644 index 5ca498c..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "secretsync-events.fullname" . }} - labels: - {{- include "secretsync-events.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "secretsync-events.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-events/templates/serviceaccount.yaml b/deploy/charts/secretsync/charts/secretsync-events/templates/serviceaccount.yaml deleted file mode 100644 index e3f5900..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "secretsync-events.serviceAccountName" . }} - labels: - {{- include "secretsync-events.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-events/values.yaml b/deploy/charts/secretsync/charts/secretsync-events/values.yaml deleted file mode 100644 index a70d986..0000000 --- a/deploy/charts/secretsync/charts/secretsync-events/values.yaml +++ /dev/null @@ -1,190 +0,0 @@ -# Default values for secretsync-events. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -existingConfigMap: "" -config: {} -# # The log level for the application. Can be one of "debug", "info", "warn", "error", "fatal", or "panic". -# log: -# level: "debug" # The log level for the application. Can be one of "debug", "info", "warn", "error", "fatal", or "panic". -# format: "json" # The format of the log output. Can be one of "json" or "text" -# events: true # Whether to log events. - -# # Configuration for the event server. -# events: -# # Whether the event server is enabled. -# enabled: true -# # The port the event server listens on. -# port: 8080 -# # Security settings for the event server. -# security: -# # Whether security is enabled for the event server. -# enabled: true -# # The token used for authentication. -# token: "your-token" -# # TLS configuration for the event server. -# tls: -# certFile: "/path/to/certfile" -# keyFile: "/path/to/keyfile" -# # Whether to deduplicate events. -# dedupe: true - -# # Configuration for the operator. -# operator: -# # Whether the operator is enabled. -# enabled: true -# # Backend configuration for the operator. -# workerPoolSize: 10 -# # The number of subscriptions to use. -# numSubscriptions: 10 -# backend: -# # The type of backend to use. -# type: "your-backend-type" -# # Parameters for the backend. -# params: -# param1: "value1" -# param2: "value2" - -# # Configuration for the stores. -# stores: - # aws: - # region: "us-west-2" - - # github: - # installId: 12345 - # appId: 67890 - # privateKeyPath: "/path/to/private/key" - -# # Configuration for the queue. -# queue: -# # The type of queue to use. -# type: "your-queue-type" -# # Parameters for the queue. -# params: -# param1: "value1" -# param2: "value2" - -# # Configuration for the metrics server. -# metrics: -# # The port the metrics server listens on. -# port: 9090 -# # Security settings for the metrics server. -# security: -# # Whether security is enabled for the metrics server. -# enabled: true -# # The token used for authentication. -# token: "your-token" -# # TLS configuration for the metrics server. -# tls: -# certFile: "/path/to/certfile" -# keyFile: "/path/to/keyfile" - -# notifications: -# email: -# enabled: true -# host: "smtp.example.com" -# port: 587 -# username: "your-email@example.com" -# password: "your-email-password" -# from: "your-email@example.com" -# to: "recipient@example.com" -# subject: "Notification Subject" -# body: "This is the notification body." -# slack: -# enabled: true -# url: "https://hooks.slack.example.com/services/xxx/xxx/xxx" -# message: "This is the notification message." -# webhook: -# enabled: true -# url: "https://example.com/webhook" -# method: "POST" -# headers: -# Content-Type: "application/json" -# body: | -# { -# "status": "{{ .Status }}", -# "message": "{{ .Message }}" -# } - - - -replicaCount: 1 - -image: - repository: docker.io/jbcom/secretssync - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -deploymentAnnotations: {} -podAnnotations: {} - -podSecurityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - fsGroup: 65534 - -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65534 - -containerPort: 8080 - -service: - type: ClusterIP - port: 8080 - -resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -env: [] -envFrom: [] -extraVolumeMounts: [] -extraVolumes: [] \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/.helmignore b/deploy/charts/secretsync/charts/secretsync-operator/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deploy/charts/secretsync/charts/secretsync-operator/Chart.yaml b/deploy/charts/secretsync/charts/secretsync-operator/Chart.yaml deleted file mode 100644 index 80b6078..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: secretsync-operator -description: A Helm chart for managing the SecretSync Operator - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "0.1.0" diff --git a/deploy/charts/secretsync/charts/secretsync-operator/crds/vaultsecretsync.lestak.sh_vaultsecretsyncs.yaml b/deploy/charts/secretsync/charts/secretsync-operator/crds/vaultsecretsync.lestak.sh_vaultsecretsyncs.yaml deleted file mode 100644 index 2134ed6..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/crds/vaultsecretsync.lestak.sh_vaultsecretsyncs.yaml +++ /dev/null @@ -1,328 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - name: vaultsecretsyncs.vaultsecretsync.lestak.sh -spec: - group: vaultsecretsync.lestak.sh - names: - kind: VaultSecretSync - listKind: VaultSecretSyncList - plural: vaultsecretsyncs - shortNames: - - vss - singular: vaultsecretsync - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Current status of the VaultSecretSync - jsonPath: .status.status - name: Status - type: string - - description: Number of destinations synced - jsonPath: .status.syncDestinations - name: SyncDestinations - type: integer - name: v1alpha1 - schema: - openAPIV3Schema: - description: VaultSecretSync is the Schema for the vaultsecretsyncs API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: VaultSecretSyncSpec defines the desired state of VaultSecretSync - properties: - dest: - items: - properties: - aws: - properties: - encryptionKey: - type: string - name: - type: string - region: - type: string - replicaRegions: - items: - type: string - type: array - roleArn: - type: string - tags: - additionalProperties: - type: string - type: object - type: object - gcp: - properties: - labels: - additionalProperties: - type: string - type: object - name: - type: string - project: - type: string - replicationLocations: - items: - type: string - type: array - type: object - github: - properties: - appId: - type: integer - env: - type: string - installId: - type: integer - merge: - type: boolean - org: - type: boolean - orgInstallIds: - additionalProperties: - type: integer - type: object - owner: - type: string - privateKey: - type: string - privateKeyPath: - type: string - repo: - type: string - type: object - http: - properties: - headerSecret: - type: string - headers: - additionalProperties: - type: string - type: object - method: - type: string - successCodes: - items: - type: integer - type: array - template: - type: string - url: - type: string - type: object - vault: - description: VaultClient is a single self-contained vault client - properties: - address: - type: string - authMethod: - type: string - cidr: - type: string - merge: - type: boolean - namespace: - type: string - path: - type: string - role: - type: string - ttl: - type: string - type: object - type: object - type: array - dryRun: - type: boolean - filters: - properties: - path: - properties: - exclude: - items: - type: string - type: array - include: - items: - type: string - type: array - type: object - regex: - properties: - exclude: - items: - type: string - type: array - include: - items: - type: string - type: array - type: object - type: object - notifications: - items: - properties: - email: - properties: - body: - type: string - events: - items: - type: string - type: array - from: - type: string - host: - type: string - insecureSkipVerify: - type: boolean - password: - type: string - port: - type: integer - subject: - type: string - to: - type: string - username: - type: string - required: - - events - type: object - slack: - properties: - body: - type: string - events: - items: - type: string - type: array - url: - type: string - urlSecret: - type: string - urlSecretKey: - type: string - required: - - events - type: object - webhook: - description: Webhook represents the configuration for a webhook. - properties: - body: - type: string - events: - items: - type: string - type: array - excludeBody: - type: boolean - headerSecret: - type: string - headers: - additionalProperties: - type: string - type: object - method: - type: string - url: - type: string - required: - - events - type: object - type: object - type: array - notificationsTemplate: - type: string - source: - description: VaultClient is a single self-contained vault client - properties: - address: - type: string - authMethod: - type: string - cidr: - type: string - merge: - type: boolean - namespace: - type: string - path: - type: string - role: - type: string - ttl: - type: string - type: object - suspend: - type: boolean - syncDelete: - type: boolean - transforms: - properties: - exclude: - items: - type: string - type: array - include: - items: - type: string - type: array - rename: - items: - properties: - from: - type: string - to: - type: string - required: - - from - - to - type: object - type: array - template: - type: string - type: object - required: - - dest - - source - type: object - status: - description: VaultSecretSyncStatus defines the observed state of VaultSecretSync - properties: - hash: - type: string - lastSyncTime: - format: date-time - type: string - status: - type: string - syncDestinations: - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/NOTES.txt b/deploy/charts/secretsync/charts/secretsync-operator/templates/NOTES.txt deleted file mode 100644 index 48029b1..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/NOTES.txt +++ /dev/null @@ -1,5 +0,0 @@ -The SecretSync Operator has been installed. - -To verify that the operator is running, run: - - kubectl get pods -n {{ .Release.Namespace }} -l "app.kubernetes.io/name=secretsync-operator" \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/_helpers.tpl b/deploy/charts/secretsync/charts/secretsync-operator/templates/_helpers.tpl deleted file mode 100644 index a170261..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/_helpers.tpl +++ /dev/null @@ -1,100 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "secretsync-operator.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "secretsync-operator.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "secretsync-operator.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "secretsync-operator.labels" -}} -helm.sh/chart: {{ include "secretsync-operator.chart" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Common Selector labels -*/}} -{{- define "secretsync-operator.selectorLabels" -}} -app.kubernetes.io/name: {{ include "secretsync-operator.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Operator Selector labels -*/}} -{{- define "secretsync-operator.operatorLabels" -}} -{{ include "secretsync-operator.labels" . }} -{{ include "secretsync-operator.selectorLabels" . }} -app.kubernetes.io/component: operator -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "secretsync-operator.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "secretsync-operator.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - - -{{/* -Simplify the definition of the kube metrics port -*/}} -{{- define "secretsync-operator.kubeMetricsPort" -}} -{{- if .Values.config.backend }} -{{- default "9080" .Values.config.backend.metricsAddr }} -{{- else }} -{{- "9080" }} -{{- end }} -{{- end }} - -{{/* -Simplify the definition of the event port -*/}} -{{- define "secretsync-operator.metricsPort" -}} -{{- default "9090" .Values.metricsPort }} -{{- end }} - -{{/* -Create the name of the configMap to use -*/}} -{{- define "secretsync-operator.configMapName" -}} -{{- if .Values.existingConfigMap }} -{{- .Values.existingConfigMap -}} -{{- else }} -{{- include "secretsync-operator.fullname" . -}} -{{- end }} -{{- end -}} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/clusterrole.yaml b/deploy/charts/secretsync/charts/secretsync-operator/templates/clusterrole.yaml deleted file mode 100644 index 4b28b8c..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/clusterrole.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.rbac.create -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "secretsync-operator.fullname" . }} - labels: - {{- include "secretsync-operator.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["events", "secrets", "configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: ["vaultsecretsync.lestak.sh"] - resources: ["vaultsecretsyncs"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: ["vaultsecretsync.lestak.sh"] - resources: ["vaultsecretsyncs/status"] - verbs: ["get", "update", "patch"] -{{- end -}} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/clusterrolebinding.yaml b/deploy/charts/secretsync/charts/secretsync-operator/templates/clusterrolebinding.yaml deleted file mode 100644 index 7f10b16..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if .Values.rbac.create -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "secretsync-operator.fullname" . }} - labels: - {{- include "secretsync-operator.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "secretsync-operator.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "secretsync-operator.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end -}} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/configmap.yaml b/deploy/charts/secretsync/charts/secretsync-operator/templates/configmap.yaml deleted file mode 100644 index afbb3bd..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/configmap.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if (eq .Values.existingConfigMap "") }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "secretsync-operator.configMapName" . }} - labels: - {{- include "secretsync-operator.labels" . | nindent 4 }} -data: - config.yaml: | - {{- .Values.config | toYaml | nindent 4 }} -{{- end }} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/deployment.yaml b/deploy/charts/secretsync/charts/secretsync-operator/templates/deployment.yaml deleted file mode 100644 index 090f311..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/deployment.yaml +++ /dev/null @@ -1,95 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "secretsync-operator.fullname" . }} - labels: - {{- include "secretsync-operator.labels" . | nindent 4 }} - {{- with .Values.deploymentAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "secretsync-operator.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "secretsync-operator.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "secretsync-operator.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - args: - - "-config" - - "/config/config.yaml" - - "-operator" - {{- if .Values.leaderElection.enabled }} - - "-enable-leader-election" - {{- end }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: {{ include "secretsync-operator.metricsPort" . }} - name: metrics - livenessProbe: - httpGet: - path: /healthz - port: metrics - readinessProbe: - httpGet: - path: /healthz - port: metrics - {{- with .Values.env }} - env: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.envFrom }} - envFrom: - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: config - mountPath: /config - readOnly: true - {{- with .Values.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: config - configMap: - name: {{ include "secretsync-operator.configMapName" . }} - defaultMode: 420 - optional: true - {{- with .Values.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/hpa.yaml b/deploy/charts/secretsync/charts/secretsync-operator/templates/hpa.yaml deleted file mode 100644 index dad71b8..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "secretsync-operator.fullname" . }} - labels: - {{- include "secretsync-operator.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "secretsync-operator.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deploy/charts/secretsync/charts/secretsync-operator/templates/serviceaccount.yaml b/deploy/charts/secretsync/charts/secretsync-operator/templates/serviceaccount.yaml deleted file mode 100644 index f93ec67..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "secretsync-operator.serviceAccountName" . }} - labels: - {{- include "secretsync-operator.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/charts/secretsync/charts/secretsync-operator/values.yaml b/deploy/charts/secretsync/charts/secretsync-operator/values.yaml deleted file mode 100644 index d59da4b..0000000 --- a/deploy/charts/secretsync/charts/secretsync-operator/values.yaml +++ /dev/null @@ -1,183 +0,0 @@ -# Default values for secretsync-operator. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# the configuration for the operator -existingConfigMap: "" -config: {} - -# Leader election configuration for HA deployments -leaderElection: - enabled: true -# # The log level for the application. Can be one of "debug", "info", "warn", "error", "fatal", or "panic". -# log: -# level: "debug" # The log level for the application. Can be one of "debug", "info", "warn", "error", "fatal", or "panic". -# format: "json" # The format of the log output. Can be one of "json" or "text" -# events: true # Whether to log events. - -# # Configuration for the event server. -# events: -# # Whether the event server is enabled. -# enabled: true -# # The port the event server listens on. -# port: 8080 -# # Security settings for the event server. -# security: -# # Whether security is enabled for the event server. -# enabled: true -# # The token used for authentication. -# token: "your-token" -# # TLS configuration for the event server. -# tls: -# certFile: "/path/to/certfile" -# keyFile: "/path/to/keyfile" -# # Whether to deduplicate events. -# dedupe: true - -# # Configuration for the operator. -# operator: -# # Whether the operator is enabled. -# enabled: true -# workerPoolSize: 10 -# # The number of subscriptions to use. -# numSubscriptions: 10 -# # Backend configuration for the operator. -# backend: -# # The type of backend to use. -# type: "your-backend-type" -# # Parameters for the backend. -# params: -# param1: "value1" -# param2: "value2" - -# # Configuration for the stores. -# stores: - # aws: - # region: "us-west-2" - - # github: - # installId: 12345 - # appId: 67890 - # privateKeyPath: "/path/to/private/key" - -# # Configuration for the queue. -# queue: -# # The type of queue to use. -# type: "your-queue-type" -# # Parameters for the queue. -# params: -# param1: "value1" -# param2: "value2" - -# # Configuration for the metrics server. -# metrics: -# # The port the metrics server listens on. -# port: 9090 -# # Security settings for the metrics server. -# security: -# # Whether security is enabled for the metrics server. -# enabled: true -# # The token used for authentication. -# token: "your-token" -# # TLS configuration for the metrics server. -# tls: -# certFile: "/path/to/certfile" -# keyFile: "/path/to/keyfile" - -# notifications: -# email: -# enabled: true -# host: "smtp.example.com" -# port: 587 -# username: "your-email@example.com" -# password: "your-email-password" -# from: "your-email@example.com" -# to: "recipient@example.com" -# subject: "Notification Subject" -# body: "This is the notification body." -# slack: -# enabled: true -# url: "https://hooks.slack.example.com/services/xxx/xxx/xxx" -# message: "This is the notification message." -# webhook: -# enabled: true -# url: "https://example.com/webhook" -# method: "POST" -# headers: -# Content-Type: "application/json" -# body: | -# { -# "status": "{{ .Status }}", -# "message": "{{ .Message }}" -# } - - - -replicaCount: 1 - -image: - repository: docker.io/jbcom/secretssync - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -rbac: - # Specifies whether RBAC resources should be created - create: true - -deploymentAnnotations: {} -podAnnotations: {} - -podSecurityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - fsGroup: 65534 - -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65534 - -resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -env: [] -envFrom: [] -extraVolumeMounts: [] -extraVolumes: [] \ No newline at end of file diff --git a/deploy/charts/secretsync/templates/_helpers.tpl b/deploy/charts/secretsync/templates/_helpers.tpl index a0b3c3e..7f29b11 100644 --- a/deploy/charts/secretsync/templates/_helpers.tpl +++ b/deploy/charts/secretsync/templates/_helpers.tpl @@ -5,17 +5,30 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} +{{/* +Create a default fully qualified app name. +*/}} +{{- define "secretsync.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} {{/* Define the name of the configMap. */}} {{- define "secretsync.configMapName" -}} -{{- if .Values.existingConfigMap }} -{{- .Values.existingConfigMap -}} -{{- else if .Values.configMapName }} -{{- .Values.configMapName -}} +{{- if .Values.pipeline.existingConfigMap }} +{{- .Values.pipeline.existingConfigMap -}} {{- else }} -{{- printf "%s-%s" .Chart.Name "config" | trunc 63 | trimSuffix "-" -}} +{{- printf "%s-config" (include "secretsync.fullname" .) | trunc 63 | trimSuffix "-" -}} {{- end }} {{- end -}} @@ -36,4 +49,23 @@ helm.sh/chart: {{ include "secretsync.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} \ No newline at end of file +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "secretsync.selectorLabels" -}} +app.kubernetes.io/name: {{ include "secretsync.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use. +*/}} +{{- define "secretsync.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "secretsync.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/charts/secretsync/templates/configmap.yaml b/deploy/charts/secretsync/templates/configmap.yaml index f639efb..75d872c 100644 --- a/deploy/charts/secretsync/templates/configmap.yaml +++ b/deploy/charts/secretsync/templates/configmap.yaml @@ -1,10 +1,12 @@ ---- +{{- if and .Values.pipeline.enabled (not .Values.pipeline.existingConfigMap) }} apiVersion: v1 kind: ConfigMap metadata: name: {{ include "secretsync.configMapName" . }} labels: {{- include "secretsync.labels" . | nindent 4 }} + {{- include "secretsync.selectorLabels" . | nindent 4 }} data: config.yaml: | - {{- .Values.config | toYaml | nindent 4 }} + {{- .Values.pipeline.config | toYaml | nindent 4 }} +{{- end }} diff --git a/deploy/charts/secretsync/templates/cronjob.yaml b/deploy/charts/secretsync/templates/cronjob.yaml new file mode 100644 index 0000000..269dc1d --- /dev/null +++ b/deploy/charts/secretsync/templates/cronjob.yaml @@ -0,0 +1,118 @@ +{{- if .Values.pipeline.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "secretsync.fullname" . }} + labels: + {{- include "secretsync.labels" . | nindent 4 }} + {{- include "secretsync.selectorLabels" . | nindent 4 }} +spec: + schedule: {{ required "pipeline.schedule is required when pipeline.enabled=true" .Values.pipeline.schedule | quote }} + concurrencyPolicy: {{ .Values.pipeline.concurrencyPolicy }} + successfulJobsHistoryLimit: {{ .Values.pipeline.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.pipeline.failedJobsHistoryLimit }} + jobTemplate: + spec: + backoffLimit: {{ .Values.pipeline.backoffLimit }} + template: + metadata: + labels: + {{- include "secretsync.selectorLabels" . | nindent 12 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} + spec: + serviceAccountName: {{ include "secretsync.serviceAccountName" . }} + restartPolicy: {{ .Values.pipeline.restartPolicy }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + containers: + - name: secretsync + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 16 }} + args: + - pipeline + - --config + - /config/config.yaml + - --continue-on-error={{ .Values.pipeline.continueOnError }} + - --diff={{ .Values.pipeline.diff }} + - --dry-run={{ .Values.pipeline.dryRun }} + - --exit-code={{ .Values.pipeline.exitCode }} + - --output + - {{ .Values.pipeline.output | quote }} + - --parallelism + - {{ .Values.pipeline.parallelism | quote }} + {{- if .Values.pipeline.targets }} + - --targets + - {{ .Values.pipeline.targets | quote }} + {{- end }} + {{- if .Values.pipeline.mergeOnly }} + - --merge-only + {{- end }} + {{- if .Values.pipeline.syncOnly }} + - --sync-only + {{- end }} + {{- if .Values.pipeline.discover }} + - --discover + {{- end }} + {{- if .Values.logLevel }} + - --log-level + - {{ .Values.logLevel | quote }} + {{- end }} + {{- if .Values.logFormat }} + - --log-format + - {{ .Values.logFormat | quote }} + {{- end }} + {{- if .Values.metrics.enabled }} + - --metrics-addr + - {{ .Values.metrics.addr | quote }} + - --metrics-port + - {{ .Values.metrics.port | quote }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 16 }} + {{- end }} + {{- with .Values.envFrom }} + envFrom: + {{- toYaml . | nindent 16 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 16 }} + volumeMounts: + - name: config + mountPath: /config + readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 16 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "secretsync.configMapName" . }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end }} diff --git a/deploy/charts/secretsync/templates/serviceaccount.yaml b/deploy/charts/secretsync/templates/serviceaccount.yaml new file mode 100644 index 0000000..6ba878a --- /dev/null +++ b/deploy/charts/secretsync/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.pipeline.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "secretsync.serviceAccountName" . }} + labels: + {{- include "secretsync.labels" . | nindent 4 }} + {{- include "secretsync.selectorLabels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/charts/secretsync/values.yaml b/deploy/charts/secretsync/values.yaml index 42cafd1..4e164d6 100644 --- a/deploy/charts/secretsync/values.yaml +++ b/deploy/charts/secretsync/values.yaml @@ -1,329 +1,78 @@ nameOverride: "" - -existingConfigMap: "" -configMapName: secretsync-config - -# Legacy config format (operator mode) -config: {} - -# ============================================================================= -# Pipeline Configuration (NEW) -# ============================================================================= -# Unified pipeline configuration for multi-account secrets management. -# Works identically via CLI, Kubernetes operator, or Helm. -# -# Operations: -# - merge: Source stores → Merge store (with inheritance) -# - sync: Merge store → AWS Secrets Manager -# - pipeline: merge + sync in dependency order +fullnameOverride: "" + +image: + repository: docker.io/jbcom/secretssync + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +env: [] +envFrom: [] +extraVolumeMounts: [] +extraVolumes: [] + +nodeSelector: {} +tolerations: [] +affinity: {} + +logLevel: info +logFormat: json + +metrics: + enabled: false + addr: "0.0.0.0" + port: 9090 pipeline: - # Enable pipeline mode (uses new unified config format) enabled: false - - # Pipeline configuration (inline or reference existing ConfigMap) - # existingConfigMap: my-pipeline-config - - config: {} - # Example pipeline configuration: - # config: - # vault: - # address: https://vault.example.com/ - # namespace: eng/data-platform - # auth: - # approle: - # role_id: ${VAULT_ROLE_ID} - # secret_id: ${VAULT_SECRET_ID} - # - # aws: - # region: us-east-1 - # execution_context: - # type: delegated_admin - # account_id: "123456789012" - # control_tower: - # enabled: true - # execution_role: - # name: AWSControlTowerExecution - # - # sources: - # analytics: - # vault: - # mount: analytics - # analytics-engineers: - # vault: - # mount: analytics-engineers - # - # merge_store: - # vault: - # mount: merged-secrets - # - # targets: - # Serverless_Stg: - # account_id: "111111111111" - # imports: - # - analytics - # - analytics-engineers - # Serverless_Prod: - # account_id: "222222222222" - # imports: - # - Serverless_Stg # Inherits from Stg - # - # pipeline: - # merge: - # parallel: 4 - # sync: - # parallel: 4 - # delete_orphans: false - - # Schedule for automatic pipeline runs (cron format) - # Leave empty to disable scheduled runs schedule: "" - # schedule: "0,30 * * * *" # Every 30 minutes - - # Default operation when triggered - operation: pipeline # merge, sync, or pipeline - - # Continue processing even if some targets fail + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + backoffLimit: 1 + restartPolicy: Never + existingConfigMap: "" + config: {} + targets: "" + mergeOnly: false + syncOnly: false + dryRun: false + discover: false continueOnError: true -# # The log level for the application. Can be one of "debug", "info", "warn", "error", "fatal", or "panic". -# log: -# level: "debug" # The log level for the application. Can be one of "debug", "info", "warn", "error", "fatal", or "panic". -# format: "json" # The format of the log output. Can be one of "json" or "text" -# events: true # Whether to log events. - -# # Configuration for the event server. -# events: -# # Whether the event server is enabled. -# enabled: true -# # The port the event server listens on. -# port: 8080 -# # Security settings for the event server. -# security: -# # Whether security is enabled for the event server. -# enabled: true -# # The token used for authentication. -# token: "your-token" -# # TLS configuration for the event server. -# tls: -# certFile: "/path/to/certfile" -# keyFile: "/path/to/keyfile" -# # Whether to deduplicate events. -# dedupe: true - -# # Configuration for the operator. -# operator: -# # Whether the operator is enabled. -# enabled: true -# # Backend configuration for the operator. -# backend: -# # The type of backend to use. -# type: "your-backend-type" -# # Parameters for the backend. -# params: -# param1: "value1" -# param2: "value2" - -# # Configuration for the stores. -# stores: - # aws: - # region: "us-west-2" - - # github: - # installId: 12345 - # appId: 67890 - # privateKeyPath: "/path/to/private/key" - -# # Configuration for the queue. -# queue: -# # The type of queue to use. -# type: "your-queue-type" -# # Parameters for the queue. -# params: -# param1: "value1" -# param2: "value2" - -# # Configuration for the metrics server. -# metrics: -# # The port the metrics server listens on. -# port: 9090 -# # Security settings for the metrics server. -# security: -# # Whether security is enabled for the metrics server. -# enabled: true -# # The token used for authentication. -# token: "your-token" -# # TLS configuration for the metrics server. -# tls: -# certFile: "/path/to/certfile" -# keyFile: "/path/to/keyfile" - -# notifications: -# email: -# enabled: true -# host: "smtp.example.com" -# port: 587 -# username: "your-email@example.com" -# password: "your-email-password" -# from: "your-email@example.com" -# to: "recipient@example.com" -# subject: "Notification Subject" -# body: "This is the notification body." -# slack: -# enabled: true -# url: "https://hooks.slack.example.com/services/xxx/xxx/xxx" -# message: "This is the notification message." -# webhook: -# enabled: true -# url: "https://example.com/webhook" -# method: "POST" -# headers: -# Content-Type: "application/json" -# body: | -# { -# "status": "{{ .Status }}", -# "message": "{{ .Message }}" -# } - - -secretsync-operator: - enabled: true - existingConfigMap: secretsync-config - replicaCount: 1 - - image: - repository: docker.io/jbcom/secretssync - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - - imagePullSecrets: [] - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - rbac: - # Specifies whether RBAC resources should be created - create: true - - deploymentAnnotations: {} - podAnnotations: {} - - podSecurityContext: - runAsNonRoot: true - runAsUser: 65534 # nobody user - runAsGroup: 65534 - fsGroup: 65534 - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65534 - - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - - nodeSelector: {} - - tolerations: [] - - affinity: {} - - env: [] - envFrom: [] - extraVolumeMounts: [] - extraVolumes: [] - - -secretsync-events: - enabled: true - existingConfigMap: secretsync-config - replicaCount: 1 - - image: - repository: docker.io/jbcom/secretssync - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - - imagePullSecrets: [] - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - deploymentAnnotations: {} - podAnnotations: {} - - podSecurityContext: {} - # fsGroup: 2000 - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - containerPort: 8080 - - service: - type: ClusterIP - port: 8080 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - - nodeSelector: {} - - tolerations: [] - - affinity: {} - - env: [] - envFrom: [] - extraVolumeMounts: [] - extraVolumes: [] \ No newline at end of file + parallelism: 0 + output: json + diff: true + exitCode: false diff --git a/dockerfile_test.go b/dockerfile_test.go new file mode 100644 index 0000000..09410f3 --- /dev/null +++ b/dockerfile_test.go @@ -0,0 +1,47 @@ +package secretsync_test + +import ( + "os" + "regexp" + "strings" + "testing" +) + +func TestDockerfileDoesNotShipVSSAlias(t *testing.T) { + content, err := os.ReadFile("Dockerfile") + if err != nil { + t.Fatalf("read Dockerfile: %v", err) + } + + text := string(content) + for _, forbidden := range []string{"/usr/local/bin/vss", "backwards compatibility", "backward compatibility"} { + if strings.Contains(text, forbidden) { + t.Fatalf("Dockerfile should not ship vss compatibility alias %q", forbidden) + } + } +} + +func TestOrganizationsTestingDocsDoNotAdvertiseVSSAlias(t *testing.T) { + path := "docs/testing/organizations-discovery-integration-tests.md" + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := strings.ToLower(strings.Join(strings.Fields(string(content)), " ")) + vssToken := regexp.MustCompile(`\bvss\b`) + if strings.Contains(text, "./vss") || vssToken.MatchString(text) { + t.Fatalf("%s should advertise secretsync, not vss", path) + } +} + +func TestForkBreakScriptIsNotShipped(t *testing.T) { + path := "scripts/break-fork.sh" + _, err := os.Stat(path) + if err == nil { + t.Fatalf("%s should not ship in the independent repository", path) + } + if !os.IsNotExist(err) { + t.Fatalf("stat %s: %v", path, err) + } +} diff --git a/docs/ACTION_QUICK_REFERENCE.md b/docs/ACTION_QUICK_REFERENCE.md index 1c1ca20..ec2bb00 100644 --- a/docs/ACTION_QUICK_REFERENCE.md +++ b/docs/ACTION_QUICK_REFERENCE.md @@ -3,14 +3,14 @@ ## Installation ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z ``` ## Minimal Example ```yaml - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml env: @@ -28,9 +28,13 @@ | `merge-only` | `false` | Only run merge phase | | `sync-only` | `false` | Only run sync phase | | `discover` | `false` | Enable dynamic discovery | -| `output-format` | `github` | Output format (human, json, github, compact) | +| `output-format` | `github` | Output format (human, json, github, compact, side-by-side) | | `compute-diff` | `false` | Show diff even without dry-run | | `exit-code` | `false` | Use exit codes (0=no changes, 1=changes, 2=errors) | +| `continue-on-error` | `true` | Continue processing remaining targets after an error | +| `parallelism` | `0` | Maximum concurrent target operations (`0` uses config/default) | +| `metrics-addr` | `0.0.0.0` | Metrics server bind address | +| `metrics-port` | `0` | Metrics server port (`0` disables metrics) | | `log-level` | `info` | Log level (debug, info, warn, error) | | `log-format` | `text` | Log format (text, json) | @@ -39,7 +43,7 @@ ### Dry Run (PR Validation) ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml dry-run: 'true' @@ -49,7 +53,7 @@ ### Specific Targets ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml targets: 'Staging,Production' @@ -58,7 +62,7 @@ ### Merge Only ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml merge-only: 'true' @@ -67,7 +71,7 @@ ### With Exit Codes ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml dry-run: 'true' @@ -78,7 +82,7 @@ ### Debug Mode ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml log-level: 'debug' @@ -103,16 +107,16 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml env: @@ -139,7 +143,7 @@ SecretSync supports all environment variables from the CLI. Common ones: ```yaml - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 @@ -189,11 +193,22 @@ env: ### `github` (Default for Action) -Shows GitHub Actions annotations in workflow logs. +Shows GitHub Actions annotations in workflow logs and writes these action +outputs when diff computation is enabled: + +| Output | Description | +| --- | --- | +| `changes` | Total added, removed, and modified secrets | +| `added` | Secrets that would be added or were added | +| `removed` | Secrets that would be removed or were removed | +| `modified` | Secrets that would be modified or were modified | +| `unchanged` | Secrets with no detected changes | +| `zero_sum` | `true` when no changes are detected | ### `json` -Machine-readable JSON output. +Machine-readable pipeline result envelope. Diff details are nested under +`diff` and `diff_output` when diff computation is enabled. ### `compact` @@ -216,7 +231,7 @@ Use with `continue-on-error: true` to handle: ```yaml - name: Check Changes id: check - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: dry-run: 'true' exit-code: 'true' @@ -232,8 +247,8 @@ Use with `continue-on-error: true` to handle: ### Config File Not Found ```yaml -- uses: actions/checkout@v4 # Must checkout first! -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: path/to/config.yaml # Relative to repo root ``` @@ -258,7 +273,7 @@ Ensure OIDC is configured correctly and trust policy allows your repository. ```yaml # Recommended: Pin to a package release tag -uses: jbcom/secrets-sync@secretssync-v2.0.2 +uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z # Not recommended: Track the branch tip uses: jbcom/secrets-sync@main diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a9611a0..145a8a9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,23 +1,101 @@ # Architecture -## High Level Architecture +SecretSync is a pipeline runner. The supported runtime is the `secretsync` CLI +executing a configured merge, sync, or full pipeline operation. Kubernetes and +GitHub Actions deployments wrap that same CLI contract instead of introducing a +separate controller API. -![High Level Architecture](./architecture/HLA.drawio.png) +See [Architecture Audit](./ARCHITECTURE_AUDIT.md) for the current +implementation-status checklist and release-contract notes. -Before we discuss how to technically deploy the solution, it is important to understand the high-level architecture of the service. The above reference architecture can be viewed as a "logical" architecture, as the service can be deployed in a variety of ways. Conceptually, the operator exposes a REST webhook endpoint at `/events` that listens for HashiCorp Vault audit log events. When an event is received, the operator will evaluate it against the configured `SecretSync` resources and sync the secret to the respective destination secret store(s). +## Runtime Shape -In the most basic deployment model, the operator can be deployed as a single process / docker container / pod. In this mode, the container both exposes the webhook server as well as performs the actual sync operation. This may not be ideal for a more security-conscious environment, as the exposed webhook service would need to be granted access to the backend secret stores and therefore could be a potential attack vector. For this reason, it is generally recommended to deploy the solution as a set of decoupled microservices, where the webhook service is deployed in a separate container / pod from the sync operator. This then allows you to enforce more strict network policies and security controls around the webhook service which then queues the work to be consumed by the operator service deeper in the network. +```text +pipeline.yaml + | + v +secretsync pipeline --config pipeline.yaml + | + +--> merge phase: source secrets -> merge store + | + +--> sync phase: merged/source secrets -> target stores + | + +--> result envelope: success, counts, per-target results, optional diff +``` + +The pipeline reads one YAML configuration file, resolves source and target +inheritance, optionally writes a merge store, then syncs destination stores. The +same command can run a dry-run with diff output, a merge-only operation, a +sync-only operation, or the full merge-plus-sync pipeline. + +## Core Components + +- **CLI entrypoint**: `cmd/secretsync` exposes `validate`, `pipeline`, and + graph-related commands for local, CI, and scheduled execution. +- **Pipeline package**: `pkg/pipeline` owns config loading, validation, + inheritance resolution, discovery, merge, sync, diff integration, and result + envelopes. +- **Diff package**: `pkg/diff` builds masked human, JSON, GitHub Actions, + compact, and side-by-side diff output. +- **Observability package**: `pkg/observability` exposes metrics for pipeline + runs that opt into the metrics endpoint. +- **GitHub Action**: `action.yml` packages the CLI contract for CI/CD workflows. +- **Helm chart**: the chart renders a Kubernetes `CronJob` plus ConfigMap or + existing config mount for scheduled pipeline execution. +- **Python integration**: `extended-data[secrets]` calls the supported CLI and + consumes the JSON result envelope as mapping-style Python data. ## Deployment Models -While this documentation generally focuses on Kubernetes-based deployments, do know that the service is not coupled to Kubernetes and therefore the same models discussed below can be followed to deploy the service in other environments, either as a set of docker containers, standalone processes, or a single binary. +### Local Or CI Execution + +Run the CLI directly when an operator or engineer controls the execution +environment: + +```bash +secretsync validate --config pipeline.yaml +secretsync pipeline --config pipeline.yaml --dry-run --diff --output json +secretsync pipeline --config pipeline.yaml --output json +``` + +GitHub Actions uses the same contract through the published action. The action +does not own a separate API surface; it validates inputs, executes the pipeline, +and reports outputs suitable for CI workflows. + +### Kubernetes Scheduled Execution + +For Kubernetes, run SecretSync as a `CronJob`. Mount the pipeline configuration +from a ConfigMap or Secret and provide cloud credentials through the cluster +identity model. + +```text +kind: CronJob + -> Pod + -> secretsync pipeline --config /config/config.yaml + -> Vault / AWS Secrets Manager / S3 / AWS discovery APIs +``` + +The Helm chart is intentionally a runner chart. It should not grow a custom +resource, reconciler, or sidecar service unless those components are owned as a +new public runtime contract. -### Single Binary +## Integration Boundaries -The simplest deployment model is to deploy the service as a single binary, as shown in the High Level Diagram above. This is the easiest way to get started with the service, but is not recommended for production deployments. In this model, the service will run as a single process that both listens for webhook events and performs the sync operation. The upside of this model is the ease of deployment, however the downside is that you must grant the exposed webhook service access to the backend secret stores, which may not be ideal from a security perspective. +SecretSync owns the Go CLI, pipeline packages, release artifact, Docker action, +and Helm runner chart. Python applications should use the `extended-data` +connector unless they are explicitly experimenting with the local gopy binding +sources in this repository. -### Microservices +The stable cross-language contract is: -The recommended deployment model is to decouple the microservices and rely on a queue to provide inter-service communication. In this model, the webhook service is deployed as a separate process / container / pod from the sync operator. The webhook service listens for audit log events, filters out irrelevant events, and then queues the relevant events to be consumed by the sync operator. The sync operator then runs in a separate process / container / pod and consumes the queued events, syncing the secrets to the destination secret store(s). In this model, the webhook service only needs to be granted limited access to the queue, and the sync operator only needs to be granted access to the destination secret store(s). Besides the optional Kubernetes metrics endpoint, the sync operator does not expose any other services, and all communication is done through the queue. +```bash +secretsync pipeline --config pipeline.yaml --output json +``` -![Microservices Deployment](./architecture/HLA-microservice.drawio.png) \ No newline at end of file +The JSON result envelope contains pipeline success, target count, secret change +counts, duration, per-target results, and optional diff output. SecretSync +redacts common bearer tokens, password or token assignments, API key +assignments, client secrets, and matching URL query parameters from top-level +and per-target error strings before serializing this envelope. Consumers should +still treat diff and error fields as operationally sensitive and apply their own +policy before writing logs, CI comments, or chat responses. diff --git a/docs/ARCHITECTURE_AUDIT.md b/docs/ARCHITECTURE_AUDIT.md new file mode 100644 index 0000000..c966c5d --- /dev/null +++ b/docs/ARCHITECTURE_AUDIT.md @@ -0,0 +1,61 @@ +# Architecture Audit + +This file records the current architecture status of the standalone +`jbcom/secrets-sync` repository. It replaces earlier migration-era notes that +referenced old monorepo paths such as `stores/vault/vault.go`. + +## Current Shape + +SecretSync is a standalone Go module with: + +- CLI entry point in `cmd/secretsync`. +- Pipeline orchestration in `pkg/pipeline`. +- Vault and AWS clients in `pkg/client`. +- Diffing and exit-code behavior in `pkg/diff`. +- Circuit breaker, request context, and observability support in `pkg`. +- Docker action metadata in `action.yml`. +- Optional Python binding sources under `python/secretssync`. +- Helm runner chart under `deploy/charts/secretsync`, rendering a CronJob and + config mount for the same CLI pipeline contract. + +The main runtime path is the two-phase pipeline: + +1. Merge source secrets into a deterministic merge-store bundle. +2. Sync the bundle into destination stores, primarily AWS Secrets Manager. + +## Implemented Parity Items + +| Area | Current status | Primary implementation | +| --- | --- | --- | +| Deep merge semantics | Implemented: maps merge recursively, lists append, scalar and type conflicts override. | `pkg/utils/deepmerge.go`, `pkg/pipeline/merge.go`, `pkg/client/vault/vault.go` | +| Vault KV2 traversal | Implemented with breadth-first recursive listing, path validation, depth limits, secret-count limits, and queue compaction. | `pkg/client/vault/vault.go` | +| AWS planned-deletion filtering | Implemented with `IncludePlannedDeletion: aws.Bool(false)`. | `pkg/client/aws/aws.go` | +| Empty AWS secret filtering | Implemented through `NoEmptySecrets`. | `pkg/client/aws/aws.go` | +| Path conflict handling | Implemented for `/foo` versus `foo` before writes. | `pkg/client/aws/aws.go` | +| JSON-aware idempotency | Implemented through `SkipUnchanged` and JSON-normalized comparison. | `pkg/client/aws/aws.go`, `pkg/utils/deepmerge.go` | +| Target inheritance | Implemented with cycle detection and merge-store source paths. | `pkg/pipeline/inheritance.go`, `pkg/pipeline/graph.go` | +| Diff exit codes | Implemented and tested: no changes, changes, and error states map to stable exit codes. | `pkg/diff`, `pkg/pipeline/diff_integration_test.go` | +| Stable pipeline result output | Implemented for machine-readable action and CLI use. | `cmd/secretsync/cmd`, `pkg/pipeline` | + +## Release And Action Status + +- CI and release workflows are SHA-pinned to current stable action releases. +- Release-please owns the `secrets-sync-vX.Y.Z` component tag shape. +- GoReleaser builds binary release artifacts from release-created tags. +- The Docker action image tag remains `jbcom/secretssync:v1` until digest + refresh can be automated. + +## Future Release Work + +- The Marketplace and action docs should continue to use the component release + tag placeholder until the first standalone repository release exists. +- The Docker action should eventually move to a digest-pinned image reference + once release automation can update that digest as part of publication. +- Optional Python binding sources and any future native Kubernetes runtime + surface need separate release contracts before becoming first-class artifacts. + +## Development Rule + +Prefer visible breakage over compatibility shims. This repository is a clean +standalone line, so stale monorepo assumptions should be removed or made to fail +in tests rather than silently accepted. diff --git a/docs/ARCHITECTURE_GAP_ANALYSIS.md b/docs/ARCHITECTURE_GAP_ANALYSIS.md deleted file mode 100644 index 05a54de..0000000 --- a/docs/ARCHITECTURE_GAP_ANALYSIS.md +++ /dev/null @@ -1,559 +0,0 @@ -# Architecture Gap Analysis: Terraform Pipeline vs secretsync - -This document provides a comprehensive analysis of the requirements from the original Terraform-based `terraform-aws-secretsmanager` pipeline compared to the current `secretsync` implementation. - -## Executive Summary - -The current implementation has a solid foundation but has **critical gaps** in the merge semantics that would cause behavioral differences from the original pipeline. These must be addressed before the PR can be considered complete. - ---- - -## Part 1: Core Pipeline Behavior Analysis - -### 1.1 Deepmerge Strategy - -**Original Requirement (Part 1):** -```python -self.merger = Merger( - [(list, ["append"]), (dict, ["merge"]), (set, ["union"])], - ["override"], # fallback - ["override"], # type conflict -) -``` - -- **Lists**: APPEND (new items added, not replaced) -- **Dicts**: MERGE (recursive deep merge) -- **Sets**: UNION (combined) -- **Conflicts**: OVERRIDE (later values win) - -**Current Implementation (`stores/vault/vault.go:298-312`):** -```go -if vc.Merge { - sec, err := vc.GetSecret(ctx, s) - // ... - for k, v := range data { - secd[k] = v // SIMPLE OVERRIDE - NOT DEEPMERGE - } - data = secd -} -``` - -**GAP: CRITICAL** ❌ -- Current merge does simple key override: `secd[k] = v` -- Does NOT append lists -- Does NOT recursively merge nested dicts -- This will cause different behavior for secrets like: - ```json - // Source 1: {"tags": ["prod"], "config": {"a": 1}} - // Source 2: {"tags": ["v2"], "config": {"b": 2}} - // Expected: {"tags": ["prod", "v2"], "config": {"a": 1, "b": 2}} - // Current: {"tags": ["v2"], "config": {"b": 2}} // WRONG - ``` - -**Required Fix:** -- Implement proper deepmerge function with list append, dict merge, set union semantics -- Apply in `VaultClient.WriteSecret` when `Merge: true` - ---- - -### 1.2 Vault Secret Listing - -**Original Requirement (Part 1):** -```python -# BFS traversal using deque -# Paths stored WITHOUT leading slash: "api-keys/stripe" not "/api-keys/stripe" -# Directories end with "/" -# Uses KV2 API (secrets.kv.v2) -``` - -**Current Implementation (`stores/vault/vault.go:448-505`):** -```go -func (vc *VaultClient) ListSecretsOnce(ctx context.Context, p string) ([]string, error) { - // Uses metadata path correctly - pp = insertSliceString(pp, 1, "metadata") - // Returns keys from listing -} -``` - -**GAP: MINOR** ⚠️ -- Current implementation lists secrets but doesn't do recursive BFS traversal -- The sync framework handles recursion via regex patterns, but direct listing is flat -- Path format handling appears consistent (no leading slash) - -**Required Fix:** -- Verify path normalization in all code paths -- Consider adding recursive listing helper if needed for direct API usage - ---- - -### 1.3 AWS Secrets Manager Listing - -**Original Requirement (Part 1):** -```python -def list_aws_account_secrets( - self, - filters: Optional[list] = None, - get_secrets: Optional[bool] = None, - no_empty_secrets: Optional[bool] = None, # Skip empty/null secrets - execution_role_arn: Optional[str] = None, -): - # Paginated listing with IncludePlannedDeletion=False - # Skip empty secrets if requested -``` - -**Current Implementation (`stores/aws/aws.go:283-312`):** -```go -func (g *AwsClient) ListSecrets(ctx context.Context, p string) ([]string, error) { - params := &secretsmanager.ListSecretsInput{ - NextToken: nextToken, - } - // Pagination: YES ✓ - // IncludePlannedDeletion: NOT SET - // Filters: NOT SUPPORTED - // no_empty_secrets: NOT SUPPORTED -} -``` - -**GAP: MEDIUM** ⚠️ -- Missing `IncludePlannedDeletion: false` parameter -- Missing `Filters` support -- Missing `no_empty_secrets` option to skip null/empty values -- These can cause sync of deleted/empty secrets - -**Required Fix:** -- Add `IncludePlannedDeletion: aws.Bool(false)` to ListSecretsInput -- Add optional `Filters` parameter -- Add `no_empty_secrets` logic when fetching values - ---- - -### 1.4 Import Source Determination - -**Original Requirement (Part 1 & 3):** -```hcl -# NULL execution_role_arn → Vault mount -# Non-NULL execution_role_arn → AWS account -imports_config = { - for import_source in imports_raw_config : import_source => - try(coalesce(local.accounts_data[import_source]["execution_role_arn"]), null) -} -``` - -**Current Implementation (`pkg/pipeline/config.go:466-482`):** -```go -func (c *Config) GetSourcePath(importName string) string { - // Check if it's a direct source - if src, ok := c.Sources[importName]; ok { - if src.Vault != nil { - return src.Vault.Mount - } - } - // Check if it's another target (inheritance) - if _, ok := c.Targets[importName]; ok { - // ... - } - return importName -} -``` - -**GAP: MEDIUM** ⚠️ -- Current implementation distinguishes Source vs Target -- Does NOT distinguish based on presence of `execution_role_arn` -- The Source struct has both `Vault` and `AWS` fields, but resolution logic doesn't use execution_role_arn as the discriminator - -**Required Fix:** -- Add logic to check if import source has associated AWS account (via accounts lookup) -- If has execution_role_arn → read from AWS SM -- If no execution_role_arn → read from Vault mount - ---- - -### 1.5 Target Inheritance Model - -**Original Requirement (Part 1):** -```yaml -Serverless_Prod: - imports: - - Serverless_Stg # AWS account! Inherits CURRENT state from AWS - -# For secretsync using merge store: -# 1. Merge: analytics + analytics-engineers → merged-secrets/Serverless_Stg/ -# 2. Merge: merged-secrets/Serverless_Stg/ → merged-secrets/Serverless_Prod/ -# 3. Sync: merged-secrets/Serverless_Stg/ → AWS Serverless_Stg -# 4. Sync: merged-secrets/Serverless_Prod/ → AWS Serverless_Prod -``` - -**Current Implementation (`pkg/pipeline/config.go:451-463`):** -```go -func (c *Config) IsInheritedTarget(targetName string) bool { - target, ok := c.Targets[targetName] - for _, imp := range target.Imports { - if _, isTarget := c.Targets[imp]; isTarget { - return true // Correctly identifies inheritance - } - } - return false -} -``` - -**GAP: MINOR** ✓ -- Inheritance detection is implemented correctly -- `GetSourcePath` correctly returns merge store path for inherited targets -- Dependency graph correctly orders operations - -**Status: IMPLEMENTED** ✓ - ---- - -### 1.6 Path Conflict Handling (/foo vs foo) - -**Original Requirement (Part 1):** -```python -def sync_secret(self, client, secret_name: str, secret_value: Any): - # Handle path conflicts (with and without leading /) - alternate_path = secret_name[1:] if secret_name.startswith('/') else f'/{secret_name}' - if self.handle_deleted_secret(client, alternate_path) is not None: - self.safe_delete_secret(client, alternate_path) -``` - -**Current Implementation:** -- No explicit path conflict handling found in `stores/aws/aws.go` -- No normalization of leading slash - -**GAP: MEDIUM** ⚠️ -- Secrets could be duplicated with different path formats -- `/prod/database` and `prod/database` would be treated as different secrets - -**Required Fix:** -- Add path normalization function -- Before creating secret, check for alternate path format -- Delete conflicting alternate if exists - ---- - -### 1.7 JSON-Aware Comparison (Idempotency) - -**Original Requirement (Part 1):** -```python -def compare_secret_values(self, existing: str, new: str) -> bool: - """Compare as JSON if possible, otherwise string compare""" - try: - return json.loads(existing) == json.loads(new) - except json.JSONDecodeError: - return existing == new -``` - -**Current Implementation:** -- Sync always writes without comparison -- No idempotency check found - -**GAP: MEDIUM** ⚠️ -- Unnecessary writes to AWS SM -- Could trigger change events unnecessarily -- Higher API costs - -**Required Fix:** -- Before WriteSecret, compare existing value with new value -- Use JSON-aware comparison for dict/list secrets -- Skip write if values are equivalent - ---- - -## Part 2: tm_cli Interface Analysis - -### 2.1 Vault Authentication - -**Original Requirement (Part 2):** -```python -# Try token auth first -if vault_token and self._vault_client.is_authenticated(): - return self._vault_client -# Fallback to AppRole -if role_id and secret_id: - self._vault_client.auth.approle.login(...) -``` - -**Current Implementation (`stores/vault/vault.go:186-208`):** -```go -func (vc *VaultClient) NewToken(ctx context.Context) error { - if os.Getenv("VAULT_TOKEN") != "" { - vc.Client.SetToken(os.Getenv("VAULT_TOKEN")) - } - if err := vc.Login(ctx); err != nil { - return err - } -} -``` - -**GAP: MINOR** ✓ -- Token auth supported via VAULT_TOKEN env var -- Kubernetes auth supported -- AppRole not directly visible but can be added via AuthMethod - -**Status: MOSTLY IMPLEMENTED** ✓ -- Could add explicit AppRole support if needed - ---- - -### 2.2 Allowlist/Denylist Filtering - -**Original Requirement (Part 2):** -```python -if allowlist: - merged_data = {k: v for k, v in merged_data.items() if k in allowlist} -if denylist: - merged_data = {k: v for k, v in merged_data.items() if k not in denylist} -``` - -**Current Implementation (`internal/transforms/filter.go`):** -- Filter transforms exist for include/exclude patterns -- Applied at sync level via SecretSync spec - -**GAP: MINOR** ✓ -- Filtering available via transforms -- Syntax different but equivalent functionality - -**Status: IMPLEMENTED** ✓ (via transforms) - ---- - -### 2.3 Error Handling (Resilience) - -**Original Requirement (Part 2):** -```python -except InvalidPath as exc: - self.logger.warning(f"Invalid secret path {current_path}: {exc}") - # Continues to next secret, doesn't fail entire operation -``` - -**Current Implementation (`pkg/pipeline/pipeline.go:425-433`):** -```go -for _, r := range levelResults { - if !r.Success { - lastErr = r.Error - if !opts.ContinueOnError { - return results, lastErr - } - } -} -``` - -**GAP: MINOR** ✓ -- `ContinueOnError` option exists -- Errors logged and tracked -- Pipeline continues on failure if configured - -**Status: IMPLEMENTED** ✓ - ---- - -## Part 3: Configuration Format Analysis - -### 3.1 Config File Formats - -**Original Requirement (Part 3):** -```yaml -# Two syntax formats: -# 1. Explicit: target: {imports: [list]} -# 2. Shorthand: target: [list] # list IS the imports - -Serverless_Stg: - imports: - - analytics - -Serverless_Prod: - - Serverless_Stg # Shorthand -``` - -**Current Implementation (`pkg/pipeline/config.go`):** -- Only supports explicit format: `imports: [...]` -- YAML unmarshaling doesn't handle shorthand - -**GAP: MEDIUM** ⚠️ -- Migration from terraform-aws-secretsmanager configs won't work directly -- Users must manually convert shorthand to explicit format - -**Required Fix:** -- Add custom YAML unmarshaler for Target struct -- Detect if value is list (shorthand) vs map (explicit) -- Convert shorthand to explicit format during load - ---- - -### 3.2 accounts_by_json_key Integration - -**Original Requirement (Part 3):** -```json -{ - "Serverless_Stg": { - "account_id": "654654379445", - "execution_role_arn": "arn:aws:iam::654654379445:role/AWSControlTowerExecution", - "environment": "staging", - "ou_path": "Sandboxes/Analytics" - } -} -``` - -**Current Implementation:** -- No direct accounts_by_json_key lookup -- Account info embedded in Target struct -- Dynamic discovery via Organizations/Identity Center - -**GAP: MINOR** ⚠️ -- Different approach: static config vs dynamic discovery -- Migration command should map old format to new - -**Status: DIFFERENT APPROACH** - acceptable if migrate command handles conversion - ---- - -### 3.3 S3 Intermediate Storage - -**Original Requirement (Part 3):** -```python -s3_key = f"secrets/{target_account_id}.json" -s3.put_object( - Bucket=secrets_bucket, - Key=s3_key, - Body=json.dumps(merged_secrets) -) -``` - -**Current Implementation (`pkg/pipeline/s3_store.go`):** -```go -func (s *S3MergeStore) keyPath(targetName string) string { - if s.config.Prefix != "" { - return fmt.Sprintf("%s/%s.json", s.config.Prefix, targetName) - } - return fmt.Sprintf("secrets/%s.json", targetName) -} -``` - -**GAP: MINOR** ✓ -- S3 storage implemented -- Key format matches (`secrets/{target}.json`) -- Encryption supported (KMS or AES256) - -**Status: IMPLEMENTED** ✓ - ---- - -### 3.4 SSM Parameter Store Discovery - -**Original Requirement (Part 3):** -- External account lists from SSM Parameter Store -- Pattern: `ssm:/platform/analytics-engineer-sandboxes` - -**Current Implementation (`pkg/pipeline/discovery.go:297-309`):** -```go -func (d *DiscoveryService) getAccountsFromSSM(paramName string) ([]AccountInfo, error) { - // Placeholder - returns error - return nil, fmt.Errorf("SSM-based account discovery requires SSM client setup; parameter: %s", paramName) -} -``` - -**GAP: MEDIUM** ⚠️ -- Feature documented but not implemented -- Returns placeholder error - -**Required Fix:** -- Implement SSM client in AWSExecutionContext -- Parse parameter value (comma-separated or JSON array) -- Convert to AccountInfo list - ---- - -## Summary: Gap Resolution Status - -### CRITICAL - RESOLVED ✅ - -| # | Gap | File | Status | -|---|-----|------|--------| -| 1 | **Deepmerge semantics** | `stores/vault/vault.go`, `pkg/utils/deepmerge.go` | ✅ **FIXED** - Implemented proper deepmerge with list append, dict merge, set union | - -### HIGH - RESOLVED ✅ - -| # | Gap | File | Status | -|---|-----|------|--------| -| 2 | Path conflict handling | `stores/aws/aws.go` | ✅ **FIXED** - Added `getAlternatePath()` and conflict detection in `WriteSecret()` | -| 3 | JSON-aware comparison | `stores/aws/aws.go`, `pkg/utils/deepmerge.go` | ✅ **FIXED** - Added `CompareSecretsJSON()` and `SkipUnchanged` option | -| 4 | no_empty_secrets | `stores/aws/aws.go` | ✅ **FIXED** - Added `NoEmptySecrets` field and `isSecretEmpty()` check | -| 5 | IncludePlannedDeletion | `stores/aws/aws.go` | ✅ **FIXED** - Added `IncludePlannedDeletion: aws.Bool(false)` to `ListSecrets()` | - -### MEDIUM - RESOLVED ✅ - -| # | Gap | File | Status | -|---|-----|------|--------| -| 6 | Shorthand config format | `pkg/pipeline/config.go` | ✅ **FIXED** - Added `UnmarshalYAML` custom unmarshaler for Target | -| 7 | SSM discovery | `pkg/pipeline/discovery.go`, `pkg/pipeline/aws_context.go` | ✅ **FIXED** - Implemented `GetSSMParameter()` and full `getAccountsFromSSM()` | -| 8 | Import source resolution by execution_role_arn | `pkg/pipeline/config.go` | ⚠️ **DEFERRED** - Current approach uses Source type (Vault/AWS) which is clearer | - ---- - -## Implementation Status - -### Phase 1: Critical Fix ✅ COMPLETE - -1. **Implement proper deepmerge in Vault store** ✅ - - Created `pkg/utils/deepmerge.go` with proper semantics - - Lists: append ✅ - - Dicts: recursive merge ✅ - - Sets: union ✅ - - Conflicts: override ✅ - - Integrated into `VaultClient.WriteSecret` ✅ - - Added comprehensive unit tests in `pkg/utils/deepmerge_test.go` ✅ - -### Phase 2: High Priority Fixes ✅ COMPLETE - -2. **Path normalization in AWS store** ✅ - - Added `getAlternatePath()` function - - Check for alternate format before create in `WriteSecret()` - - Delete conflicting alternate if exists - -3. **Idempotency with JSON comparison** ✅ - - Added `CompareSecretsJSON()` function in `pkg/utils/deepmerge.go` - - Added `SkipUnchanged` option to `AwsClient` - - Check before write, skip if equivalent - -4. **AWS listing improvements** ✅ - - Added `IncludePlannedDeletion: aws.Bool(false)` to ListSecretsInput - - Added `NoEmptySecrets` option and `isSecretEmpty()` check - -### Phase 3: Medium Priority Fixes ✅ COMPLETE - -5. **Shorthand config support** ✅ - - Added custom `UnmarshalYAML` for Target struct - - Supports both `{imports: [...]}` and `[list]` formats - -6. **SSM discovery** ✅ - - Added `ssmClient` to AWSExecutionContext - - Added `GetSSMParameter()` method - - Implemented full `getAccountsFromSSM()` with support for: - - Comma-separated lists - - JSON string arrays - - JSON object arrays with id/name fields - ---- - -## Verification Checklist - -All items verified: - -- [x] Deepmerge: `{"tags": ["a"]}` + `{"tags": ["b"]}` = `{"tags": ["a", "b"]}` (unit test: TestDeepMerge_ListAppend) -- [x] Deepmerge: `{"config": {"x": 1}}` + `{"config": {"y": 2}}` = `{"config": {"x": 1, "y": 2}}` (unit test: TestDeepMerge_DictMerge) -- [x] Path handling: `/foo` and `foo` don't create duplicates (getAlternatePath + WriteSecret) -- [x] Idempotency: Unchanged secrets don't trigger writes (SkipUnchanged + CompareSecretsJSON) -- [x] Empty secrets: Not synced when no_empty_secrets is set (NoEmptySecrets + isSecretEmpty) -- [x] Deleted secrets: Not synced (IncludePlannedDeletion=false) -- [x] Inheritance: Serverless_Prod correctly inherits from Serverless_Stg (IsInheritedTarget + GetSourcePath) -- [x] Dynamic targets: Organizations discovery works (DiscoveryService) -- [x] S3 merge store: Secrets written to correct path (S3MergeStore) -- [x] Error resilience: Pipeline continues on single secret failure (ContinueOnError) -- [x] Shorthand config: Both `{imports: [...]}` and `[list]` formats supported -- [x] SSM discovery: Accounts can be loaded from SSM Parameter Store - -## Build & Test Status - -- ✅ `go build ./...` - PASSED -- ✅ `go test ./...` - ALL PASSED -- ✅ `golangci-lint run` - 0 ISSUES diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index d450622..341e5d2 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -1,174 +1,207 @@ # Deployment -This guide outlines the requirements and different deployment models for the Vault Secrets Sync service. +SecretSync deploys as a `secretsync pipeline` runner. The current production +surface is the CLI, the Docker image, the GitHub Action, or a Kubernetes +workload that invokes the same pipeline command with a mounted configuration +file. -## Getting Started +## Deployment Contract -The service can be configured with either a `YAML` or `JSON` configuration file, or via environment variables. The configuration is unmarshalled into the [`pipeline.Config`](../pkg/pipeline/types.go#L5) struct. An example of a fully configured `YAML` file can be found in [examples/config-full.yaml](../examples/config-full.yaml). To configure via environment variables, you must prefix the environment variable with `SECRETSYNC_` and use `_` to denote nested fields. For example, to set the `queue.type` field, you would set the environment variable `SECRETSYNC_QUEUE_TYPE`. Note: The legacy `VSS_` prefix is still supported for backwards compatibility. +- Use one pipeline configuration file as the source of truth. +- Validate the configuration before the first apply run. +- Run `--dry-run --diff` before writes in CI and scheduled jobs. +- Enable machine-readable output for automation with `--output json` or + `--output github`. +- Use `--exit-code` when a CI job must distinguish no changes, changes, and + failures. +- Enable metrics with `--metrics-port` for long-running or scheduled runners. -While various examples are provided, the operator intentionally does not ship with a "default" configuration file. This is to ensure that you are aware of the security implications of each component you are enabling. This guide will walk you through each section, what it does, and how to configure it. +## Configuration -### `queue` Configuration +Create a pipeline configuration with Vault source settings, a merge store, AWS +execution context, and targets. See [PIPELINE.md](./PIPELINE.md) for the full +schema. -The service requires a queue for communication between the `Event Server` and the `Sync Operator`. When deployed as a single binary, the `memory` queue will be used by default. This simply uses the operator's internal memory for queuing and processing. For small to moderately sized deployments this may be sufficient, however in more production grade deployments - and when deployed in microservices mode - an external queue must be configured. In all queues, both the event server and sync operator must be able to connect to the queue, and data will only ever flow in one direction (from the outside in, never from the inside out). - -Currently, the following queues are supported: +```yaml +vault: + address: https://vault.example.com/ + namespace: eng/data-platform + auth: + approle: + role_id: ${VAULT_ROLE_ID} + secret_id: ${VAULT_SECRET_ID} + +aws: + region: us-east-1 + execution_context: + type: delegated_admin + account_id: "123456789012" + control_tower: + enabled: true + execution_role: + name: AWSControlTowerExecution -- `memory`: The default queue, which uses the operator's internal memory for queuing and processing. This is only recommended for small to moderately sized deployments, and cannot be used in a microservices deployment model. -- `redis`: The Redis queue uses a Redis instance for queuing and processing. -- `nats`: The NATS queue uses a NATS instance for queuing and processing. -- `sqs`: The SQS queue uses an AWS SQS queue for queuing and processing. +sources: + analytics: + vault: + mount: analytics -An example of a fully configured `YAML` file can be found in [examples/config-full.yaml](../examples/config-full.yaml). Here's an example of a minimal configuration file: +merge_store: + vault: + mount: merged-secrets -```yaml -queue: - type: redis - params: - host: "redis" - port: 6379 - password: "" - db: 0 - tls: - ca: /etc/certs/ca.crt - cert: /etc/certs/client.crt - key: /etc/certs/client.key +targets: + Serverless_Stg: + account_id: "111111111111" + imports: + - analytics ``` -Fields not required in your environment can be omitted. For example, if you are not using TLS, you can omit the `tls` field. If you are not using a password, you can omit the `password` field. - -### `operator` Configuration +Validate before deploying: -The operator is responsible for reconciling the `SecretSync` CRD and handling sync operations. Here's an example of a minimal configuration file: - -```yaml -operator: - enabled: true - workerPoolSize: 10 - numSubscriptions: 10 +```bash +secretsync validate --config config.yaml +secretsync graph --config config.yaml ``` +## Local Or VM Runner -The `workerPoolSize` field is the number of workers that will be spawned to process the events from the queue. The `numSubscriptions` field is the number of subscriptions that will be created to the queue. The number of subscriptions should be equal to or greater than the number of workers. The `workerPoolSize` field should be set to a value that is appropriate for your environment. The default value is `10`. +Install the binary and run the same command used in CI: -### `event` Configuration +```bash +go install github.com/jbcom/secrets-sync/cmd/secretsync@latest + +secretsync pipeline \ + --config config.yaml \ + --dry-run \ + --diff \ + --output json \ + --exit-code +``` -The event server is responsible for listening for audit log events from Vault. The event server is required for the service to operate. It must be accessible by the respective vault instance audit log shippers, and must be able to communicate with the queue. Here's an example of a minimal configuration file: +For an apply run, remove `--dry-run` after reviewing the diff: -```yaml -event: - enabled: true - port: 8080 - security: - enabled: true - tls: - cert: /etc/certs/server.crt - key: /etc/certs/server.key - ca: /etc/certs/ca.crt - clientAuth: require +```bash +secretsync pipeline --config config.yaml --diff --output json ``` -Note that in the examples above, each service has an `enabled` field, this is where you can enable / disable particular components of the service. By default, all components are disabled. However you are still able to re-use a single configuration file across various microservice components by passing the corresponding CLI flag to enable the component rather than modifying the configuration file. +## Docker Runner -### `metrics` Configuration +Mount the configuration read-only and pass credentials through environment +variables, workload identity, or mounted secret files. -The service exposes a metrics endpoint on a dedicated service metrics port (separate from the kubernetes metrics port exposed when running in kubernetes operator mode) to expose Prometheus metrics. Here's an example of a minimal configuration file: - -```yaml -metrics: - port: 8082 +```bash +docker run --rm \ + -v "$PWD/config.yaml:/config.yaml:ro" \ + -e VAULT_ADDR=https://vault.example.com \ + -e VAULT_ROLE_ID="$VAULT_ROLE_ID" \ + -e VAULT_SECRET_ID="$VAULT_SECRET_ID" \ + jbcom/secretssync:v1 \ + pipeline --config /config.yaml --dry-run --diff --output json ``` -Note that the metrics endpoint also supports `security.tls` configuration, it has simply been omitted from the example for brevity. The metrics server also exposes a `/healthz` endpoint that can be used to check the health of the service and its dependencies. +Use the same image for scheduled container platforms. The image entry point is +`secretsync`, and the default command is `pipeline`. -### `stores` Configuration +## GitHub Actions -Each `SecretSync` configuration is entirely self-contained - the `spec` contains all the fields necessary to perform the sync. However you can use the `stores` configuration to set defaults for all `SecretSync` resources. This is useful if you have multiple `SecretSync` resources that share the same configuration. Here's an example of a minimal configuration file: +Use the published Docker action for CI and release pipelines. Keep the action +reference on a release tag and configure AWS/Vault credentials before invoking +SecretSync. ```yaml -stores: - vault: - address: "https://vault.example.com" - - github: - owner: "example-org" +jobs: + secrets: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: arn:aws:iam::123456789012:role/SecretSyncRunner + aws-region: us-east-1 + - uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z + with: + config: config.yaml + dry-run: "true" + compute-diff: "true" + output-format: json + exit-code: "true" ``` -Fields explictly defined in the `SecretSync` resource will take precedence over the defaults set in the `stores` configuration, however if a field is not provided in the `SecretSync` resource, the default value from the `stores` configuration will be used. Defaults are evaluated at runtime and are not persisted back to the backend. This can be both a feature and a bug, depending on your use case. Once you set a central default, be cognizant of the fact that changing the default _will_ change the behavior of existing `SecretSync` resources. For this reason it's generally recommended to not set global defaults and instead rely on fields being explicitly declared on the `SecretSync` resources themselves, unless you know for certain that you want to change the behavior of all resources at once. - -With this said, the fields defined will only work if the operator has the proper access to the stores. For more details on how to configure the operator to access the stores, see the [Security](./SECURITY.md) documentation. - -## Deploying in Kubernetes +See [GITHUB_ACTIONS.md](./GITHUB_ACTIONS.md) for the full action input +reference. -The service can be deployed in Kubernetes using the provided Helm chart. The Helm chart is located in the `deploy/charts` directory of the repository. The chart is designed to be as flexible as possible, and allows you to configure the service using a `values.yaml` file. The chart is designed to be deployed in a microservices architecture, where the webhook service is deployed in one container and the sync operator is deployed in another, however it does support a monolithic deployment as well. The chart also deploys a CRD to enable configuration of the sync service through native Kubernetes resources. +## Kubernetes CronJob -Once you have your `values.yaml` file configured, you can deploy the service using the following command: +For Kubernetes, run SecretSync as a scheduled job unless your environment has a +separate controller managing execution. Mount the pipeline configuration from a +ConfigMap or Secret and provide credentials through your cluster identity model. -```shell -helm install -n secretsync --create-namespace \ - secretsync ./deploy/charts/secretsync \ - -f /path/to/values.yaml +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: secretsync +spec: + schedule: "*/30 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + serviceAccountName: secretsync + containers: + - name: secretsync + image: jbcom/secretssync:v1 + args: + - pipeline + - --config + - /config/config.yaml + - --diff + - --output + - json + volumeMounts: + - name: config + mountPath: /config + readOnly: true + volumes: + - name: config + configMap: + name: secretsync-config ``` -Note this will install the `SecretSync` CRD by default. While recommended to use the CRD when deploying in Kubernetes it is _technically_ not required, and so if you do not want to install the CRDs with the rest of the chart, you can pass the `--skip-crds` flag to the `helm install` command. - -The chart will deploy one `Service` object for the event service with a `ClusterIP` type. This must be made accessible from the Vault audit log shipper(s) in whatever manner suits your environment. +For Kubernetes-authenticated Vault access, configure the `vault.auth.kubernetes` +section in the pipeline file and bind the service account to the expected Vault +role. For AWS, prefer IRSA, EKS Pod Identity, or another workload identity +mechanism over static access keys. -You can mount volumes, secrets, and env vars to any of the components through the values file, so if you need to mount a secret to the event server, you can do so through the `values.yaml` file. +## Metrics And Logs -If you're not a fan of using Helm to manage your resources, you can always replace `helm install` with `helm template` and pipe the output to `kubectl apply -f -` to apply the resources directly to your cluster. +Enable metrics when the runner stays alive long enough to be scraped, or when a +platform sidecar captures process metrics: -```shell -helm template -n secretsync \ - secretsync ./deploy/charts/secretsync \ - -f /path/to/values.yaml | kubectl apply -f - +```bash +secretsync pipeline --config config.yaml --metrics-addr 0.0.0.0 --metrics-port 9090 ``` -## Shipping Logs - -This service relies on the audit logs as shipped by HashiCorp Vault. You must have an [audit device](https://developer.hashicorp.com/vault/docs/audit) configured in your Vault instance to ship logs to the service. You must configure the webhook endpoint in your audit device to point to the `/events` endpoint of the service. It is recommended to include the `X-Vault-Tenant` header in the request to the service to identify the source of the event. This is especially important if you are syncing secrets from multiple Vault instances. This is discussed more in [Usage - Source Determination](./USAGE.md#source-determination). Below is a sample Fluentd configuration that ships logs to the service. If you have event server token-based security enabled, you will also need to include the `X-SecretSync-Token` header in the request. While your security posture may vary, it's generally recommended to use multiple layers of security, such as internal networking, IP whitelisting, service mesh RBAC, and token-based security. +Use JSON logs in centralized logging environments: -### Fluentd Configuration Example - -**Token based auth** - -```xml - - @type http - endpoint https://secretsync/events - headers {"x-vault-tenant": "https://vault.example.com", "x-secretsync-token": "99CFF209-9E67-4B22-880F-E15DAC3C1CEE"} - open_timeout 2 - - @type json - - - flush_interval 10s - - +```bash +secretsync pipeline --config config.yaml --log-format json --log-level info ``` -**TLS Client Cert Auth** - -```xml - - @type http - endpoint https://secretsync/events - headers {"x-vault-tenant": "https://vault.example.com"} - tls_ca_cert_path /path/to/ca.crt - tls_client_cert_path /path/to/client.crt - tls_private_key_path /path/to/client.key - open_timeout 2 - - @type json - - - flush_interval 10s - +SecretSync logs operational metadata, paths, targets, counts, durations, and +provider error context. It must not log raw secret values, raw Vault secret +payloads, raw AWS secret payloads, or raw client structures. +## Rollout Checklist - Once your fluentd is up and running, configure your vault audit device to ship logs to the fluentd endpoint. - - -```bash -vault audit enable socket address=fluentd:24224 socket_type=tcp -``` +1. Validate the configuration with `secretsync validate`. +2. Render and review the dependency graph with `secretsync graph`. +3. Run a dry-run diff with machine-readable output. +4. Confirm AWS role assumption and Vault authentication from the runner. +5. Run the apply command with `--diff` enabled for auditability. +6. Monitor exit status, logs, and metrics after each scheduled run. diff --git a/docs/ERROR_CONTEXT.md b/docs/ERROR_CONTEXT.md index 17c73a9..72f6207 100644 --- a/docs/ERROR_CONTEXT.md +++ b/docs/ERROR_CONTEXT.md @@ -93,12 +93,12 @@ time="2025-12-09T17:00:05Z" level=info msg="Starting merge phase" action=Pipelin time="2025-12-09T17:00:10Z" level=info msg="Pipeline execution completed successfully" request_id=e0958539-fae2-4567-9227-592a5b36983a duration_ms=10234 ``` -## Backward Compatibility +## Error Wrapping -The enhanced error context is fully backward compatible: -- Operations without request context still work (request_id will be empty) -- Error messages maintain the same error wrapping chain -- Existing error handling code continues to work unchanged +The enhanced error context preserves the Go error wrapping chain: +- Operations without request context still work with an empty request ID. +- Error messages keep their wrapped causes available to `errors.Is` and `errors.As`. +- Callers can opt into request-scoped diagnostics without changing the underlying error type. ## Benefits diff --git a/docs/FAQ.md b/docs/FAQ.md index 8c3e4fb..40754e0 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -19,7 +19,7 @@ SecretSync is an enterprise-grade secret synchronization pipeline that automates ### Is SecretSync production-ready? -Yes! SecretSync v1.2.0 is production-ready with: +Yes. The current SecretSync 2.x line is production-ready with: - 150+ comprehensive tests - Full CI/CD pipeline with integration tests - Circuit breakers and error handling @@ -96,17 +96,30 @@ path "secret/metadata/*" { Inheritance allows targets to import configuration from other targets: ```yaml +sources: + common-secrets: + vault: + mount: common + staging-overrides: + vault: + mount: staging + prod-overrides: + vault: + mount: production + targets: base: imports: [common-secrets] staging: - inherits: base # Gets everything from base - imports: [staging-overrides] # Plus staging-specific secrets + imports: + - base + - staging-overrides production: - inherits: staging # Gets base + staging - imports: [prod-overrides] # Plus production-specific secrets + imports: + - staging + - prod-overrides ``` ### Can I sync to multiple AWS accounts? @@ -116,14 +129,16 @@ Yes! Use cross-account IAM roles: ```yaml targets: dev-account: - aws_secretsmanager: - role_arn: "arn:aws:iam::111111111111:role/SecretSyncRole" - region: "us-east-1" + account_id: "111111111111" + role_arn: "arn:aws:iam::111111111111:role/SecretSyncRole" + region: "us-east-1" + imports: [shared-secrets] prod-account: - aws_secretsmanager: - role_arn: "arn:aws:iam::222222222222:role/SecretSyncRole" - region: "us-east-1" + account_id: "222222222222" + role_arn: "arn:aws:iam::222222222222:role/SecretSyncRole" + region: "us-east-1" + imports: [shared-secrets] ``` ### How do I handle different environments? @@ -158,9 +173,9 @@ targets: ## Features -### What's new in v1.2.0? +### What's included in the current feature set? -Major new features: +Major features: - **Enhanced AWS Organizations Discovery** with tag filtering and wildcards - **AWS Identity Center Integration** for permission set discovery - **Secret Versioning System** with S3-based storage and rollback @@ -199,8 +214,9 @@ targets: imports: [base-secrets] production: - inherits: staging # Reads from merge store - imports: [prod-overrides] + imports: + - staging + - prod-overrides ``` ### How does AWS Organizations discovery work? @@ -208,19 +224,23 @@ targets: Automatically discover and sync to accounts based on tags and OUs: ```yaml -discovery: - aws_organizations: - enabled: true - tag_filters: - - key: "Environment" - values: ["production", "staging"] - operator: "equals" - - key: "Team" - values: ["platform*"] - operator: "contains" - organizational_units: - - "ou-production-12345" - tag_logic: "AND" +dynamic_targets: + production_and_staging: + discovery: + organizations: + ous: + - ou-production-12345 + recursive: true + tag_filters: + - key: Environment + values: ["production", "staging"] + operator: equals + - key: Team + values: ["platform*"] + operator: contains + tag_combination: AND + imports: + - shared-secrets ``` ## Operations @@ -231,7 +251,7 @@ Use the GitHub Action: ```yaml - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml dry-run: 'false' @@ -339,13 +359,15 @@ Common causes and solutions: 3. **Region mismatch**: ```yaml # Ensure regions match - aws: - region: "us-east-1" - targets: - production: - aws_secretsmanager: - region: "us-east-1" # Must match - ``` + aws: + region: "us-east-1" + targets: + production: + account_id: "222222222222" + region: "us-east-1" + imports: + - shared-secrets + ``` ### "Secret not found in Vault" @@ -465,9 +487,9 @@ go test -race ./... ### Where can I get help? -- **Documentation**: [docs/](https://github.com/jbcom/secrets-sync/docs) +- **Documentation**: [docs/](https://github.com/jbcom/secrets-sync/tree/main/docs) - **GitHub Issues**: For bugs, questions, and feature requests -- **Examples**: [examples/](https://github.com/jbcom/secrets-sync/examples) +- **Examples**: [examples/](https://github.com/jbcom/secrets-sync/tree/main/examples) ### How do I request a feature? diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 7f6864c..dca002b 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -1,38 +1,29 @@ -# Getting Started with SecretSync +# Getting Started With SecretSync -This guide will walk you through setting up SecretSync from scratch to sync secrets from HashiCorp Vault to AWS Secrets Manager. +This guide sets up a current SecretSync pipeline that reads secrets from Vault, +merges them into a merge store, and syncs selected targets into AWS Secrets +Manager. ## Prerequisites -Before you begin, ensure you have: +- HashiCorp Vault with a KV v2 secrets engine. +- AWS credentials from an IAM role, workload identity, or access keys. +- Permission to read the configured Vault mounts. +- Permission to write the target AWS Secrets Manager secrets. -- **HashiCorp Vault** with KV2 secrets engine enabled -- **AWS Account** with Secrets Manager access -- **Vault credentials** (AppRole recommended) -- **AWS credentials** (IAM role or access keys) +## Step 1: Install -## Step 1: Installation - -Choose your preferred installation method: - -### Option A: Go Install +Choose one runtime: ```bash go install github.com/jbcom/secrets-sync/cmd/secretsync@latest ``` -### Option B: Docker - ```bash -# Pull image docker pull jbcom/secretssync:v1 - -# Create alias for easier usage alias secretsync='docker run --rm -v "$PWD":/workspace -w /workspace jbcom/secretssync:v1' ``` -### Option C: Build from Source - ```bash git clone https://github.com/jbcom/secrets-sync.git cd secrets-sync @@ -40,225 +31,140 @@ make build ./bin/secretsync version ``` -## Step 2: Basic Configuration +## Step 2: Create A Pipeline Config -Create a configuration file `config.yaml`: +Create `config.yaml`: ```yaml -# Basic SecretSync configuration vault: - address: "https://your-vault.example.com" - namespace: "admin" # Optional: if using Vault namespaces + address: https://vault.example.com/ + namespace: admin auth: approle: - role_id: "${VAULT_ROLE_ID}" - secret_id: "${VAULT_SECRET_ID}" + role_id: ${VAULT_ROLE_ID} + secret_id: ${VAULT_SECRET_ID} aws: - region: "us-east-1" - # Optional: role to assume for cross-account access - # role_arn: "arn:aws:iam::123456789012:role/SecretSyncRole" + region: us-east-1 + execution_context: + type: delegated_admin + account_id: "123456789012" + control_tower: + enabled: true + execution_role: + name: AWSControlTowerExecution -# Define where to read secrets from sources: app-secrets: vault: - path: "secret/data/myapp" # KV2 path + mount: secret + paths: + - myapp + +merge_store: + vault: + mount: merged-secrets -# Define where to write secrets to targets: production: - aws_secretsmanager: - region: "us-east-1" - # Optional: prefix for secret names - prefix: "myapp/" + account_id: "222222222222" + region: us-east-1 + secret_prefix: myapp/ imports: - app-secrets -``` - -## Step 3: Set Environment Variables -```bash -# Vault credentials -export VAULT_ROLE_ID="your-role-id" -export VAULT_SECRET_ID="your-secret-id" - -# AWS credentials (if not using IAM roles) -export AWS_ACCESS_KEY_ID="your-access-key" -export AWS_SECRET_ACCESS_KEY="your-secret-key" +pipeline: + merge: + parallel: 4 + sync: + parallel: 4 + delete_orphans: false + continue_on_error: true ``` -## Step 4: Validate Configuration +Targets can import from sources or from other targets. Target-to-target imports +are how you model inheritance: -Before running, validate your configuration: +```yaml +targets: + staging: + account_id: "111111111111" + imports: + - app-secrets -```bash -secretsync validate --config config.yaml + production: + account_id: "222222222222" + imports: + - staging ``` -This will check: -- Configuration syntax -- Vault connectivity -- AWS permissions -- Source/target accessibility - -## Step 5: Dry Run - -Perform a dry run to see what changes would be made: +## Step 3: Configure Credentials ```bash -secretsync pipeline --config config.yaml --dry-run -``` - -You should see output like: -``` -Pipeline Diff Summary -===================== - Added: 3 secrets - Modified: 0 secrets - Deleted: 0 secrets - -⚠️ CHANGES DETECTED - -Target: production - + myapp/database-password - + myapp/api-key - + myapp/jwt-secret +export VAULT_ROLE_ID="your-role-id" +export VAULT_SECRET_ID="your-secret-id" +export AWS_REGION="us-east-1" ``` -## Step 6: Execute Sync +Prefer AWS role assumption or workload identity for deployed runners. Use static +AWS access keys only when your environment cannot provide an identity. -If the dry run looks correct, execute the actual sync: +## Step 4: Validate And Inspect ```bash -secretsync pipeline --config config.yaml +secretsync validate --config config.yaml +secretsync graph --config config.yaml ``` -## Step 7: Verify Results +Validation checks the config structure and dependency graph. Add `--check-aws` +when you want validation to test AWS credentials and access. -Check AWS Secrets Manager to confirm your secrets were created: +## Step 5: Dry Run ```bash -# Using AWS CLI -aws secretsmanager list-secrets --query 'SecretList[?starts_with(Name, `myapp/`)]' - -# Or check in AWS Console -# Navigate to AWS Secrets Manager in your region +secretsync pipeline --config config.yaml --dry-run --diff --output json --exit-code ``` -## Next Steps +Exit codes are stable for automation: -### Enable Advanced Features +- `0`: no changes +- `1`: changes detected +- `2`: errors -#### 1. Add Observability (v1.1.0) +## Step 6: Apply -```bash -# Run with metrics endpoint -secretsync pipeline --config config.yaml --metrics-port 9090 - -# In another terminal, check metrics -curl http://localhost:9090/metrics -curl http://localhost:9090/health -``` - -#### 2. Enhanced Diff Output (v1.2.0) +After reviewing the dry-run diff, run the apply path: ```bash -# Side-by-side comparison -secretsync pipeline --config config.yaml --dry-run --format side-by-side - -# JSON output for automation -secretsync pipeline --config config.yaml --dry-run --format json -``` - -#### 3. Secret Versioning (v1.2.0) - -Add to your config: -```yaml -versioning: - enabled: true - s3_bucket: "my-secretsync-versions" - retention_days: 90 -``` - -#### 4. AWS Organizations Discovery (v1.2.0) - -```yaml -discovery: - aws_organizations: - enabled: true - tag_filters: - - key: "Environment" - values: ["production", "staging"] - operator: "equals" - cache_ttl: "1h" -``` - -### Set Up CI/CD - -#### GitHub Actions - -Create `.github/workflows/secretsync.yml`: - -```yaml -name: Sync Secrets -on: - schedule: - - cron: '0 */6 * * *' # Every 6 hours - workflow_dispatch: - -jobs: - sync: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - - steps: - - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} - aws-region: us-east-1 - - - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 - with: - config: config.yaml - env: - VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }} +secretsync pipeline --config config.yaml --diff --output json ``` ## Common Patterns -### Multi-Environment Setup +### Multi-Environment Inheritance ```yaml sources: base-secrets: vault: - path: "secret/data/base" - + mount: secret + paths: [base] prod-secrets: vault: - path: "secret/data/production" + mount: secret + paths: [production] targets: staging: - aws_secretsmanager: - region: "us-east-1" + account_id: "111111111111" imports: - base-secrets - + production: - aws_secretsmanager: - region: "us-east-1" + account_id: "222222222222" imports: - - base-secrets - - prod-secrets # Production-specific overrides + - staging + - prod-secrets ``` ### Cross-Account Sync @@ -266,90 +172,115 @@ targets: ```yaml targets: dev-account: - aws_secretsmanager: - region: "us-east-1" - role_arn: "arn:aws:iam::111111111111:role/SecretSyncRole" + account_id: "111111111111" + region: us-east-1 + role_arn: arn:aws:iam::111111111111:role/SecretSyncRole imports: - dev-secrets - + prod-account: - aws_secretsmanager: - region: "us-east-1" - role_arn: "arn:aws:iam::222222222222:role/SecretSyncRole" + account_id: "222222222222" + region: us-east-1 + role_arn: arn:aws:iam::222222222222:role/SecretSyncRole imports: - prod-secrets ``` -### Merge Store Pattern +### S3 Merge Store With Versioning ```yaml -# Use S3 as merge store for complex inheritance merge_store: s3: - bucket: "my-secretsync-merge-store" - prefix: "merged/" - region: "us-east-1" - -targets: - staging: - imports: [base-secrets] - - production: - inherits: staging # Inherit from staging's merged output - imports: [prod-overrides] + bucket: my-secretsync-merge-store + prefix: merged/ + kms_key_id: alias/secretsync + versioning: + enabled: true + retain_versions: 90 ``` -## Troubleshooting +### Dynamic AWS Organizations Targets -### Common Issues - -#### "Vault authentication failed" -- Verify `VAULT_ROLE_ID` and `VAULT_SECRET_ID` are correct -- Check Vault policies allow access to specified paths -- Ensure Vault address is reachable +```yaml +dynamic_targets: + production-accounts: + discovery: + organizations: + ous: + - ou-abcd-production + tag_filters: + - key: Environment + values: ["production"] + operator: equals + recursive: true + imports: + - app-secrets + region: us-east-1 + secret_prefix: myapp/ +``` -#### "AWS access denied" -- Verify AWS credentials are configured -- Check IAM permissions for Secrets Manager -- Ensure region is correct +Run with discovery enabled: -#### "Secret not found" -- Verify Vault path exists and is accessible -- Check KV2 engine is enabled at the mount -- Ensure path format is correct (`secret/data/path` for KV2) +```bash +secretsync pipeline --config config.yaml --discover --dry-run --diff +``` -### Debug Mode +## GitHub Actions -Enable debug logging for more details: +```yaml +name: Sync Secrets +on: + schedule: + - cron: "0 */6 * * *" + workflow_dispatch: -```bash -secretsync pipeline --config config.yaml --log-level debug +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} + aws-region: us-east-1 + - uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z + with: + config: config.yaml + dry-run: "true" + compute-diff: "true" + output-format: json + exit-code: "true" + env: + VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }} + VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }} ``` -### Validate Permissions +## Troubleshooting -Test individual components: +### Vault authentication failed -```bash -# Test Vault connectivity -vault auth -method=approle role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID -vault kv list secret/ +- Verify `VAULT_ROLE_ID` and `VAULT_SECRET_ID`. +- Confirm the AppRole can read the configured mounts and paths. +- Confirm Vault address and namespace are reachable from the runner. -# Test AWS connectivity -aws secretsmanager list-secrets --region us-east-1 -``` +### AWS access denied -## Getting Help +- Confirm the runner identity can assume the configured target role. +- Check Secrets Manager create, update, list, and delete permissions. +- Confirm the configured region and account IDs. -- **Documentation**: [Full docs](https://github.com/jbcom/secrets-sync/docs) -- **Examples**: [Configuration examples](https://github.com/jbcom/secrets-sync/examples) -- **Issues**: [GitHub Issues](https://github.com/jbcom/secrets-sync/issues) +### No changes detected unexpectedly -## What's Next? +- Run with `--output side-by-side` for a human diff. +- Check target imports and source path spelling. +- Run `secretsync graph --config config.yaml` to verify dependency order. -- Explore [advanced configuration options](./PIPELINE.md) -- Set up [monitoring and observability](./OBSERVABILITY.md) -- Learn about [deployment patterns](./DEPLOYMENT.md) -- Integrate with [GitHub Actions](./GITHUB_ACTIONS.md) +## Next Steps -Welcome to SecretSync! 🚀 +- Read [PIPELINE.md](./PIPELINE.md) for the full configuration reference. +- Read [DEPLOYMENT.md](./DEPLOYMENT.md) for production deployment patterns. +- Read [OBSERVABILITY.md](./OBSERVABILITY.md) for metrics and logging. +- Read [GITHUB_ACTIONS.md](./GITHUB_ACTIONS.md) for CI integration. diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.md index 2a14fee..daebf9a 100644 --- a/docs/GITHUB_ACTIONS.md +++ b/docs/GITHUB_ACTIONS.md @@ -21,16 +21,16 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml ``` @@ -47,9 +47,13 @@ All inputs correspond to CLI flags and are optional: | `merge-only` | Only run merge phase | `false` | `--merge-only` | | `sync-only` | Only run sync phase | `false` | `--sync-only` | | `discover` | Enable dynamic target discovery | `false` | `--discover` | -| `output-format` | Output format (human, json, github, compact) | `github` | `--output` | +| `output-format` | Output format (human, json, github, compact, side-by-side) | `github` | `--output` | | `compute-diff` | Show diff even without dry-run | `false` | `--diff` | | `exit-code` | Use exit codes for CI/CD | `false` | `--exit-code` | +| `continue-on-error` | Continue processing remaining targets after an error | `true` | `--continue-on-error` | +| `parallelism` | Maximum concurrent target operations (`0` uses config/default) | `0` | `--parallelism` | +| `metrics-addr` | Metrics server bind address | `0.0.0.0` | `--metrics-addr` | +| `metrics-port` | Metrics server port (`0` disables metrics) | `0` | `--metrics-port` | | `log-level` | Logging level (debug, info, warn, error) | `info` | `--log-level` | | `log-format` | Log format (text, json) | `text` | `--log-format` | @@ -77,16 +81,16 @@ jobs: pull-requests: write # For PR comments steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Validate Changes (Dry Run) - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml dry-run: 'true' @@ -124,16 +128,16 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml targets: ${{ github.event.inputs.targets != 'all' && github.event.inputs.targets || '' }} @@ -159,10 +163,10 @@ jobs: merge: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Merge Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml merge-only: 'true' @@ -191,16 +195,16 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Sync with Discovery - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml discover: 'true' @@ -231,17 +235,17 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Check for Changes id: check - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml dry-run: 'true' @@ -254,7 +258,7 @@ jobs: - name: Apply Changes if: steps.check.outcome == 'failure' - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml output-format: 'github' @@ -291,16 +295,16 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: configs/${{ github.event.inputs.environment }}.yaml output-format: 'github' @@ -359,7 +363,7 @@ targets: **Recommended:** ```yaml - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 @@ -392,7 +396,7 @@ jobs: sync-production: environment: production # Requires approval steps: - - uses: jbcom/secrets-sync@secretssync-v2.0.2 + - uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: production.yaml ``` @@ -512,9 +516,9 @@ Example policy: Ensure your config file is in the repository and the path is correct: ```yaml -- uses: actions/checkout@v4 # Required to access repository files +- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: path/to/config.yaml # Relative to repo root ``` @@ -524,7 +528,7 @@ Ensure your config file is in the repository and the path is correct: Verify environment variables are set correctly: ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml log-level: debug # Enable debug logging @@ -546,11 +550,36 @@ Check: Ensure `output-format` is set to `github`: ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: output-format: 'github' # Enables GitHub Actions annotations ``` +When diff computation is enabled, the action also writes modern +`$GITHUB_OUTPUT` values: + +| Output | Description | +| --- | --- | +| `changes` | Total added, removed, and modified secrets | +| `added` | Secrets that would be added or were added | +| `removed` | Secrets that would be removed or were removed | +| `modified` | Secrets that would be modified or were modified | +| `unchanged` | Secrets with no detected changes | +| `zero_sum` | `true` when no changes are detected | + +```yaml +- name: Check for Changes + id: check + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z + with: + dry-run: 'true' + compute-diff: 'true' + output-format: 'github' + +- name: Report Summary + run: echo "Changed secrets: ${{ steps.check.outputs.changes }}" +``` + ## Exit Codes When `exit-code: 'true'` is enabled: @@ -566,7 +595,7 @@ Example using exit codes: ```yaml - name: Check for Changes id: check - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: dry-run: 'true' exit-code: 'true' @@ -599,16 +628,16 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets[format('AWS_ROLE_{0}', matrix.environment)] }} aws-region: us-east-1 - name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: configs/${{ matrix.environment }}.yaml ``` @@ -628,7 +657,7 @@ jobs: sync: if: github.ref == 'refs/heads/main' steps: - - uses: jbcom/secrets-sync@secretssync-v2.0.2 + - uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z ``` ### Composite Actions @@ -646,12 +675,12 @@ inputs: runs: using: composite steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - - uses: jbcom/secrets-sync@secretssync-v2.0.2 + - uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: ${{ inputs.config }} output-format: 'github' @@ -662,7 +691,7 @@ runs: ## Support -- **Documentation**: [Full docs](https://github.com/jbcom/secrets-sync/docs) +- **Documentation**: [Full docs](https://github.com/jbcom/secrets-sync/tree/main/docs) - **Issues**: [GitHub Issues](https://github.com/jbcom/secrets-sync/issues) ## License diff --git a/docs/MARKETPLACE.md b/docs/MARKETPLACE.md index 53d4529..2dc2941 100644 --- a/docs/MARKETPLACE.md +++ b/docs/MARKETPLACE.md @@ -1,358 +1,164 @@ -# GitHub Marketplace Listing Guide +# GitHub Marketplace Guide -This document provides information for listing SecretSync on the GitHub Marketplace as a verified free action. +This guide describes the GitHub Marketplace surface for the standalone +`jbcom/secrets-sync` action. -## Marketplace Information +## Listing Summary -### Basic Information +| Field | Value | +| --- | --- | +| Name | `SecretSync` | +| Repository | `jbcom/secrets-sync` | +| Category | Deployment, Continuous Integration, Security | +| License | MIT | +| Pricing | Free | -**Name**: SecretSync -**Tagline**: Universal secrets synchronization pipeline for multi-cloud secret management -**Category**: Deployment and Continuous Integration -**Pricing**: Free +SecretSync synchronizes secrets from HashiCorp Vault into AWS Secrets Manager +with a two-phase merge and sync pipeline. It is designed for multi-account AWS +environments, AWS Organizations discovery, and CI/CD validation workflows. -### Description +## Supported Runtime Surface -SecretSync provides fully automated, real-time secret synchronization between HashiCorp Vault and AWS. Perfect for multi-account AWS environments, HashiCorp Vault users, and organizations managing secrets across AWS Organizations. - -**Key Features:** -- 🔄 Two-phase pipeline architecture (merge → sync) -- 🎯 Vault-to-AWS secrets synchronization -- 🌐 Multi-account AWS secret management -- 📊 GitHub-native diff annotations in PRs -- 🔒 OIDC authentication for AWS (no long-lived credentials) -- 🚀 Dynamic target discovery via AWS Organizations/Identity Center -- ⚡ Zero-configuration Docker action -- 🔐 Complete privacy - no data collection - -### Supported Stores - -- HashiCorp Vault (KV2) - source -- AWS Secrets Manager - target -- AWS S3 (merge store option) -- Kubernetes Secrets (operator mode) - -## Marketplace Requirements Checklist - -### ✅ Technical Requirements - -- [x] **action.yml file**: Present in repository root -- [x] **Docker-based action**: Uses Dockerfile for containerization -- [x] **Valid inputs**: All inputs properly documented with descriptions -- [x] **Branding**: Icon and color specified -- [x] **Multi-arch support**: Supports linux/amd64 and linux/arm64 - -### ✅ Documentation Requirements - -- [x] **README.md**: Comprehensive documentation with examples -- [x] **Usage examples**: Complete workflow examples -- [x] **Input documentation**: All inputs documented with defaults -- [x] **Quick start guide**: Easy getting started section -- [x] **Advanced examples**: Multiple use case examples - -### ✅ Legal and Policy Requirements - -- [x] **License**: MIT License (permissive, OSI-approved) -- [x] **Privacy Policy**: See [docs/PRIVACY.md](./PRIVACY.md) -- [x] **Support Information**: See [docs/SUPPORT.md](./SUPPORT.md) -- [x] **Security Policy**: See [docs/SECURITY.md](./SECURITY.md) -- [x] **Code of Conduct**: Implicit in professional conduct - -### ✅ Quality Requirements - -- [x] **Working action**: Fully functional and tested -- [x] **Error handling**: Proper error messages and exit codes -- [x] **Logging**: Comprehensive logging with multiple formats -- [x] **Security**: No hardcoded secrets, proper secret handling -- [x] **Performance**: Efficient execution with parallel processing - -### ✅ Marketplace Best Practices - -- [x] **Semantic versioning**: Using git tags (v1, v1.0.0, etc.) -- [x] **Clear naming**: Descriptive and searchable name -- [x] **Useful description**: Clear value proposition -- [x] **Good documentation**: Step-by-step guides and examples -- [x] **Community support**: GitHub Issues -- [x] **Regular updates**: Active maintenance and improvements - -## Publishing to Marketplace - -### Prerequisites - -1. **Repository Requirements** - - Public repository on GitHub - - Valid `action.yml` in root directory - - Proper branding (icon, color) - - Comprehensive README - -2. **Legal Requirements** - - License file (MIT) - - Privacy policy - - Support contact information - - Security policy - -3. **Quality Requirements** - - Working action with examples - - No security vulnerabilities - - Proper error handling - - Good documentation - -### Publishing Steps - -1. **Verify Action Works** - ```bash - # Test action locally - act -j test-action - ``` - -2. **Create Version Tags** - ```bash - # Create and push version tags - git tag -a v1.0.0 -m "Release v1.0.0" - git push origin v1.0.0 - - # Create major version tag (recommended for users) - git tag -fa v1 -m "Release v1" - git push origin v1 --force - ``` - -3. **Publish to Marketplace** - - Go to repository on GitHub - - Click "Releases" - - Click "Draft a new release" - - Choose the version tag (e.g., v1.0.0) - - Check "Publish this Action to the GitHub Marketplace" - - Select primary category: "Deployment" - - Add release notes - - Click "Publish release" - -4. **Verify Listing** - - Visit: https://github.com/marketplace/actions/secretsync - - Verify all information is correct - - Test the action from Marketplace - -### Recommended Version Tags +- GitHub Action using `action.yml`. +- Docker image `jbcom/secretssync:v1`. +- Go CLI `secretsync`. +- Vault KV2 sources. +- AWS Secrets Manager targets. +- Vault or S3 merge stores. +- GitHub-native output for PR validation and CI logs. -```bash -# Semantic version (specific) -v1.0.0, v1.0.1, v1.1.0, v2.0.0 - -# Major version (for users - auto-updates) -v1, v2 - -# Example: Release v1.2.3 -git tag -a v1.2.3 -m "Release v1.2.3 - Add OIDC support" -git push origin v1.2.3 +Avoid advertising stores or deployment modes that are not implemented in the +current standalone repository. -# Update major version pointer -git tag -fa v1 -m "Update v1 to v1.2.3" -git push origin v1 --force -``` - -## Marketplace Metadata +## Release Tags -### Action Metadata (action.yml) +Release-please manages releases for the root package named `secrets-sync`. +Marketplace examples should therefore use component release tags: ```yaml -name: 'SecretSync' -description: 'Universal secrets synchronization pipeline for multi-cloud secret management with Vault, AWS, GCP, and more' -author: 'jbcom' - -branding: - icon: 'lock' - color: 'blue' +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z ``` -### Category Selection - -**Primary Category**: Deployment -**Additional Categories**: -- Continuous Integration -- Security -- Utilities - -### Tags/Keywords - -- secrets-management -- vault -- aws-secrets-manager -- secret-sync -- multi-cloud -- devops -- security -- oidc -- ci-cd -- github-actions - -## Marketing Copy - -### Short Description (200 chars) - -Vault-to-AWS secrets sync. Two-phase pipeline with inheritance, dynamic discovery via AWS Organizations/Identity Center & GitHub-native diffs. Free, open source. - -### Long Description - -SecretSync revolutionizes Vault-to-AWS secrets management with a powerful two-phase pipeline architecture. Built for organizations managing secrets across multiple AWS accounts with HashiCorp Vault as the source of truth. - -**Perfect For:** -- Multi-account AWS environments (Control Tower, Organizations) -- HashiCorp Vault users syncing to AWS Secrets Manager -- Teams managing secrets across dev/staging/prod -- Organizations requiring secret inheritance hierarchies -- DevOps teams automating secret distribution - -**Key Benefits:** - -🔄 **Two-Phase Architecture** -Merge secrets from multiple Vault paths, then sync to multiple AWS accounts with inheritance support. - -🎯 **Vault → AWS Sync** -HashiCorp Vault (KV2) as source, AWS Secrets Manager as target, with S3 or Vault merge store options. - -🌐 **AWS-Native Patterns** -First-class support for AWS Control Tower, Organizations, and Identity Center patterns. - -📊 **GitHub-Native Integration** -Automatic PR annotations, diff reporting, and status checks. - -🔒 **Security First** -OIDC authentication, no long-lived credentials, complete audit trail, zero data collection. +Do not document old monorepo package tags using the `secretssync-v...` shape. -🚀 **Dynamic Discovery** -Automatically discover and sync to accounts via AWS Organizations or Identity Center. +`@main` may be useful for development testing, but it is not a stable +Marketplace recommendation. Moving major aliases such as `@v1` should only be +documented if the repository intentionally creates and maintains those aliases. -**Use Cases:** +## Marketplace Requirements -1. **Control Tower Environments**: Sync secrets to all AWS accounts in your organization -2. **Vault Distribution**: Push Vault secrets to AWS Secrets Manager across accounts -3. **Secret Inheritance**: Dev → Staging → Production with automatic propagation -4. **Multi-Region**: Sync secrets across AWS regions from central Vault -5. **Compliance**: Automated secret rotation with complete audit trail +- Repository is public. +- `action.yml` exists in the repository root. +- Action metadata has name, description, author, inputs, branding, and Docker + image reference. +- README includes a working action example. +- `LICENSE`, `SECURITY.md`, `CONTRIBUTING.md`, privacy docs, and support docs + are present. +- Workflow examples use least-privilege permissions. +- Third-party actions in maintained examples are pinned to exact commit SHAs. -**Zero Configuration** -Just add your config file and secrets - the action handles the rest. +## Publication Flow -## Badge and Shield Links +1. Merge normal changes to `main` using Conventional Commit prefixes. +2. Let release-please open or update the release PR. +3. Merge the release PR after review. +4. Confirm the release workflow created a `secrets-sync-vX.Y.Z` GitHub release. +5. Confirm GoReleaser uploaded binary assets and `checksums.txt`. +6. In the GitHub release UI, publish that release to Marketplace. +7. Verify the Marketplace page renders the README and action metadata correctly. -Add these to README for visibility: +Do not create a manual release or manual version tag during normal publication. -```markdown -[![GitHub Marketplace](https://img.shields.io/badge/Marketplace-SecretSync-blue.svg?colorA=24292e&colorB=0366d6&style=flat&longCache=true&logo=github)](https://github.com/marketplace/actions/secretsync) +## Recommended Usage Snippet -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - -[![GitHub release](https://img.shields.io/github/v/release/jbcom/secrets-sync?filter=secretssync-v*&label=release)](https://github.com/jbcom/secrets-sync/releases) - -[![GitHub stars](https://img.shields.io/github/stars/jbcom/secrets-sync.svg)](https://github.com/jbcom/secrets-sync/stargazers) +```yaml +name: Sync Secrets + +on: + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} + aws-region: us-east-1 + + - name: Sync Secrets + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z + with: + config: config.yaml + output-format: github + env: + VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }} + VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }} ``` -## Support URLs - -Add these to the Marketplace listing: - -- **Documentation**: https://github.com/jbcom/secrets-sync/docs -- **Issues**: https://github.com/jbcom/secrets-sync/issues -- **Support**: https://github.com/jbcom/secrets-sync/blob/main/docs/SUPPORT.md -- **Privacy Policy**: https://github.com/jbcom/secrets-sync/blob/main/docs/PRIVACY.md -- **Security**: https://github.com/jbcom/secrets-sync/blob/main/docs/SECURITY.md - -## Verification Requirements - -For verified publisher status: - -1. **Organization Account**: Must be published from an organization (not personal) -2. **Email Verification**: Organization email must be verified -3. **2FA Enabled**: Two-factor authentication required -4. **Quality Standards**: Meet all GitHub Marketplace quality requirements -5. **Active Maintenance**: Regular updates and responsive support - -## Monitoring and Maintenance - -### Post-Publication Tasks - -1. **Monitor Issues**: Respond to bug reports and questions -2. **Track Usage**: Use GitHub's marketplace insights -3. **Regular Updates**: Keep action up-to-date with dependencies -4. **Security Patches**: Respond quickly to security issues -5. **Documentation**: Keep docs updated with new features +## Metadata -### Marketplace Insights +`action.yml` should remain the source of truth for action metadata: -Track these metrics: -- Daily/monthly active users -- Total installations -- Popular use cases (from issues and support requests) -- User feedback and ratings -- Common problems/questions - -## Compliance and Policies - -### Data Privacy - -SecretSync is privacy-by-design: -- ✅ No data collection -- ✅ No external network calls (except to user's configured services) -- ✅ No telemetry or analytics -- ✅ Complete user control -- ✅ Open source and auditable - -See [Privacy Policy](./PRIVACY.md) for details. - -### Security - -- Regular dependency updates -- CodeQL scanning enabled -- Security policy documented -- Responsible disclosure process -- No known vulnerabilities - -See [Security Policy](./SECURITY.md) for details. +```yaml +name: "SecretSync" +author: "jbcom" +description: "Sync secrets from HashiCorp Vault to AWS Secrets Manager across multiple accounts" +branding: + icon: "lock" + color: "blue" +``` -### Support +## Listing Copy -- GitHub Issues for bug reports -- GitHub Issues for Q&A -- Email for security issues -- Community-driven support +Use concise copy that matches the implementation: -See [Support Guide](./SUPPORT.md) for details. +> SecretSync syncs HashiCorp Vault secrets into AWS Secrets Manager across +> multiple AWS accounts. It supports merge-first pipelines, AWS Organizations +> discovery, dry-run diff output, and GitHub-native CI feedback. -## Frequently Asked Questions +## Verification -### Can I publish beta/pre-release versions? +After publication: -Yes! Use pre-release tags: ```bash -git tag -a v1.0.0-beta.1 -m "Beta release" +gh release view secrets-sync-vX.Y.Z --repo jbcom/secrets-sync +gh workflow run ci.yml --repo jbcom/secrets-sync ``` -### How do I unpublish from Marketplace? - -You can delist the action in your repository settings under "Marketplace". +Also check: -### Can I charge for this action? +- Marketplace page links to the standalone repository. +- The README usage example references `secrets-sync-vX.Y.Z`. +- Inputs shown by Marketplace match `action.yml`. +- No docs mention old `secretssync-v...` monorepo package tags. -This action is MIT licensed and free. Paid versions require different licensing. +## FAQ -### How do version tags work? +### Can users pin `@main`? -Users can reference: -- `@secretssync-v2.0.2` - Current package release tag (recommended) -- `@secretssync-vX.Y.Z` - Exact package release tag (pinned) -- `@main` - Latest commit (not recommended) +They can, but documentation should recommend a component release tag because +`main` is mutable. -### What if my action has dependencies? +### Should we publish a `v1` alias? -Docker actions (like this one) bundle all dependencies in the container. +Only if maintainers decide to update that alias intentionally for every +supported release. Release-please will not maintain it automatically. -## Additional Resources +### Does the Marketplace release publish binary assets? -- [GitHub Actions Documentation](https://docs.github.com/en/actions) -- [Creating a Docker Container Action](https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action) -- [Publishing Actions to Marketplace](https://docs.github.com/en/actions/creating-actions/publishing-actions-in-github-marketplace) -- [Marketplace Requirements](https://docs.github.com/en/actions/creating-actions/publishing-actions-in-github-marketplace#requirements-for-publishing-an-action) -- [Action Metadata Syntax](https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions) +No. GoReleaser publishes binary archives and checksums from the release +workflow. Marketplace publication exposes the action metadata from the GitHub +release. ---- +### Does the action send data to jbcom? -**Ready to publish?** Follow the Publishing Steps above and your action will be live on the GitHub Marketplace! +No. The action runs in the user's GitHub Actions environment and talks directly +to the configured Vault and AWS accounts. See `docs/PRIVACY.md`. diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index 8d488c9..0514f3a 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -63,6 +63,33 @@ Once enabled, metrics are exposed at: - **Metrics**: `http://localhost:9090/metrics` - **Health check**: `http://localhost:9090/health` +## Logging Safety + +Logs and metrics are designed for operational visibility without exposing the +secret bytes being synchronized. SecretSync records request IDs, operation +names, durations, counts, paths, targets, account identifiers, and provider +error context. It must not log raw secret values, raw Vault secret payloads, raw +AWS secret payloads, or raw client structures. + +Use JSON logs when shipping to a centralized platform: + +```bash +secretsync pipeline --config config.yaml --log-format json --log-level info +``` + +Debug and trace logging can reveal more operational metadata and should be sent +only to secured sinks with appropriate retention. If a provider returns +credentials in an error string, treat that upstream behavior as a provider or +configuration issue and rotate the exposed credential. + +Machine-readable `secretsync pipeline --output json` result envelopes redact +common secret-bearing fragments from top-level and per-target error strings +before serialization. Treat `error_message`, per-target `error`, and +`diff_output` as operationally sensitive when forwarding them to logs, +dashboards, CI comments, or chat systems. +GitHub Actions annotation output escapes workflow-command data in target names +and secret paths before writing groups, notices, or warnings. + ## Available Metrics ### Vault Metrics diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index c8531cc..1545afe 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -269,7 +269,8 @@ dynamic_targets: dev_accounts: discovery: organizations: - ou: "ou-xxxx-development" + ous: + - "ou-xxxx-development" recursive: true # Include accounts in child OUs imports: - dev-secrets @@ -352,15 +353,15 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Run Pipeline - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml targets: ${{ inputs.targets || '' }} @@ -376,7 +377,7 @@ jobs: ```yaml secrets-sync: stage: deploy - image: golang:1.25 + image: golang:1.26 before_script: - go install github.com/jbcom/secrets-sync/cmd/secretsync@latest script: diff --git a/docs/PUBLISHING_CHECKLIST.md b/docs/PUBLISHING_CHECKLIST.md index 31c263c..1f1f294 100644 --- a/docs/PUBLISHING_CHECKLIST.md +++ b/docs/PUBLISHING_CHECKLIST.md @@ -1,316 +1,115 @@ -# Publishing SecretSync to GitHub Marketplace - Checklist +# Publishing Checklist -This is a step-by-step checklist for publishing SecretSync to the GitHub Marketplace. +SecretSync releases are automated from `main` with release-please and +GoReleaser. Do not hand-edit versions, changelog entries, release tags, or +GitHub releases during the normal release path. -## Pre-Publishing Checklist +## Release Model -### ✅ Repository Requirements +- `release-please` owns version detection, changelog updates, release PRs, and + Git tags. +- The root package name is `secrets-sync`, so component release tags use the + `secrets-sync-vX.Y.Z` shape. +- GoReleaser runs only after release-please reports that a release was created. +- GoReleaser builds binary archives and checksums. Container and Marketplace + publication are separate release surfaces. +- The Docker action currently references `docker://jbcom/secretssync:v1` from + `action.yml`; digest pinning should be added only when release automation can + refresh that digest reliably. -- [x] Repository is public -- [x] `action.yml` exists in repository root -- [x] Action has valid metadata (name, description, author) -- [x] Branding is configured (icon: lock, color: blue) -- [x] Dockerfile builds successfully -- [x] README has usage examples -- [x] LICENSE file exists (MIT) +## Maintainer Preflight -### ✅ Action Configuration - -- [x] All inputs documented with descriptions -- [x] Default values specified for optional inputs -- [x] Docker image reference is correct (`image: 'Dockerfile'`) -- [x] Entrypoint script is executable -- [x] Environment variables properly mapped - -### ✅ Documentation - -- [x] README.md is comprehensive -- [x] Usage examples provided -- [x] Input parameters documented -- [x] Example workflows included -- [x] Security best practices documented -- [x] Troubleshooting section exists - -### ✅ Legal and Compliance - -- [x] MIT License in place -- [x] Privacy policy created (docs/PRIVACY.md) -- [x] Support documentation created (docs/SUPPORT.md) -- [x] Security policy exists (docs/SECURITY.md) -- [x] Contributing guidelines created (CONTRIBUTING.md) - -### ✅ Quality Assurance - -- [x] Action inputs validated -- [x] Entrypoint script syntax validated -- [x] Example configurations provided -- [x] Error handling implemented -- [x] Logging configured -- [x] No hardcoded secrets - -## Publishing Steps - -### Step 1: Final Testing - -Before publishing, test the action: +Run these before merging a release PR or manually dispatching release workflow +diagnostics: ```bash -# 1. Validate entrypoint script -sh -n entrypoint.sh - -# 2. Test Docker build (if environment allows) +go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... +go test ./... +go build -o bin/secretsync ./cmd/secretsync +goreleaser check docker build -t secretsync-test . - -# 3. Create a test workflow in .github/workflows/test-action.yml -# Example test workflow: -``` - -```yaml -name: Test Action -on: [push] -jobs: - test: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} - aws-region: us-east-1 - - - name: Test Action - uses: ./ - with: - config: examples/action-test-config.yaml - dry-run: 'true' - env: - VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }} ``` -### Step 2: Create Version Tag +If `goreleaser` is not installed locally, use the pinned workflow action as the +source of truth and verify the GitHub check output. -```bash -# 1. Ensure all changes are committed -git status - -# 2. Create annotated tag for first release -git tag -a v1.0.0 -m "Release v1.0.0 - Initial GitHub Marketplace release" - -# 3. Push tag to GitHub -git push origin v1.0.0 - -# 4. Create major version tag (for users) -git tag -fa v1 -m "Release v1 - Initial release" -git push origin v1 --force -``` - -### Step 3: Create GitHub Release +## Workflow Hygiene -1. Go to: https://github.com/jbcom/secrets-sync/releases -2. Click "Draft a new release" -3. Select tag: `v1.0.0` -4. Release title: `v1.0.0 - GitHub Marketplace Release` -5. Release description: +- Keep `.github/workflows/*.yml` actions pinned to exact commit SHAs. +- Update the adjacent version comments when refreshing an action SHA. +- Use `gh` to verify latest stable action releases before changing pins. +- Do not grant broad workflow permissions; keep top-level `permissions: {}` and + add job-scoped permissions only where needed. -```markdown -## 🚀 Initial GitHub Marketplace Release +Current workflow action pins: -SecretSync is now available as a GitHub Action! This release provides a Docker-based action for universal secrets synchronization across multiple cloud providers. +| Action | Stable version | Commit SHA | +| --- | --- | --- | +| `actions/checkout` | `v6.0.3` | `df4cb1c069e1874edd31b4311f1884172cec0e10` | +| `actions/setup-go` | `v6.4.0` | `4a3601121dd01d1626a1e23e37211e3254c1c06c` | +| `googleapis/release-please-action` | `v5.0.0` | `45996ed1f6d02564a971a2fa1b5860e934307cf7` | +| `goreleaser/goreleaser-action` | `v7.2.2` | `5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89` | -### ✨ Features +## Publishing Flow -- 🔄 Two-phase pipeline architecture (merge → sync) -- 🎯 Support for 8+ secret stores (Vault, AWS, GCP, GitHub, Doppler, K8s, S3) -- 🌐 Multi-cloud and multi-account secret management -- 📊 GitHub-native diff annotations in PRs -- 🔒 OIDC authentication for AWS (no long-lived credentials) -- 🚀 Dynamic target discovery via AWS Organizations/Identity Center -- ⚡ Zero-configuration Docker action -- 🔐 Complete privacy - no data collection - -### 📖 Quick Start +1. Land normal feature, fix, docs, and maintenance commits using Conventional + Commit prefixes. +2. Let the release workflow open or update the release-please PR. +3. Review the release PR for correct changelog and manifest updates. +4. Merge the release PR. +5. Confirm the release workflow created a `secrets-sync-vX.Y.Z` GitHub release. +6. Confirm GoReleaser uploaded archives and `checksums.txt`. +7. Verify the action can be referenced with: ```yaml -- name: Sync Secrets - uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml - env: - VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }} -``` - -### 📚 Documentation - -- [GitHub Actions Guide](./GITHUB_ACTIONS.md) -- [Quick Reference](./ACTION_QUICK_REFERENCE.md) -- [Example Workflows](../examples/github-action-workflow.yml) -- [Privacy Policy](./PRIVACY.md) -- [Support](./SUPPORT.md) - -### 🔒 Security - -SecretSync collects zero data and runs entirely within your GitHub Actions environment. See [Privacy Policy](./PRIVACY.md) for details. - -### 🤝 Contributing - -We welcome contributions! See [CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines. - -### 📄 License - -MIT License - See [LICENSE](../LICENSE) ``` -6. Check "✓ Publish this Action to the GitHub Marketplace" -7. Select primary category: **Deployment** -8. Optionally add secondary categories: - - Continuous Integration - - Security -9. Click "Publish release" - -### Step 4: Verify Marketplace Listing - -1. Visit: https://github.com/marketplace/actions/secretsync -2. Verify all information displays correctly: - - Name, description, and icon - - Input parameters - - Usage examples - - Documentation links - - Author information - -3. Check that: - - README renders properly - - Examples are clear - - Links work - - Badges display - -### Step 5: Post-Publication Tasks - -1. **Update README with Marketplace badge** - ```markdown - [![GitHub Marketplace](https://img.shields.io/badge/Marketplace-SecretSync-blue.svg?colorA=24292e&colorB=0366d6&style=flat&longCache=true&logo=github)](https://github.com/marketplace/actions/secretsync) - ``` - -2. **Announce release** - - Open a tracking issue for launch feedback - - Tweet/share on social media - - Update any external documentation - -3. **Monitor feedback** - - Watch for issues - - Respond to questions - - Track usage metrics (if available) +## Marketplace Publication -4. **Set up monitoring** - - Confirm issue templates and labels are ready - - Set up issue templates - - Configure automated responses +GitHub Marketplace publication is completed from a GitHub release in the UI. +Use the release that release-please created. Do not create a parallel manual +release just to publish the Marketplace listing. -## Post-Publication Maintenance +Checklist: -### Regular Updates +- Repository is public. +- `action.yml` exists at repository root. +- `action.yml` metadata, inputs, branding, and Docker image reference are valid. +- `README.md`, `docs/GITHUB_ACTIONS.md`, and + `docs/ACTION_QUICK_REFERENCE.md` show the component tag shape. +- Security, privacy, support, contributing, and license documents are present. +- A real release exists for the tag being published. -- [ ] Monitor security vulnerabilities -- [ ] Update dependencies regularly -- [ ] Respond to issues and PRs -- [ ] Release bug fixes promptly -- [ ] Add new features based on feedback - -### Version Management - -When releasing new versions: +## Post-Release Verification ```bash -# 1. Update CHANGELOG.md -# 2. Create new version tag -git tag -a v1.1.0 -m "Release v1.1.0 - Add feature X" -git push origin v1.1.0 - -# 3. Update major version tag -git tag -fa v1 -m "Update v1 to v1.1.0" -git push origin v1 --force - -# 4. Create GitHub release with changelog +gh release view secrets-sync-vX.Y.Z --repo jbcom/secrets-sync +gh workflow run ci.yml --repo jbcom/secrets-sync ``` -### Marketplace Updates - -To update marketplace listing: - -1. Update README or action.yml as needed -2. Create new release with updated information -3. Marketplace will automatically reflect changes - -## Troubleshooting - -### Common Issues - -**Issue**: Action doesn't appear in Marketplace after publishing -- **Solution**: Check that "Publish to Marketplace" was checked during release - -**Issue**: Docker build fails for users -- **Solution**: Test multi-platform builds, ensure dependencies are available - -**Issue**: Inputs not working as expected -- **Solution**: Verify entrypoint.sh handles all inputs correctly - -**Issue**: Users report authentication errors -- **Solution**: Check documentation is clear, add troubleshooting guide - -## Support Channels - -After publishing, provide support through: - -1. **GitHub Issues**: Bug reports and feature requests -2. **GitHub Issues**: Questions and community support -3. **Email**: Security issues (private reporting) -4. **Documentation**: Keep docs updated with common questions - -## Metrics to Track - -Monitor these metrics post-publication: - -- Daily/monthly active users -- Total installations -- Issue resolution time -- PR merge rate -- Community engagement -- User feedback/ratings - -## Continuous Improvement - -Based on feedback: - -- [ ] Add requested features -- [ ] Improve documentation -- [ ] Fix reported bugs -- [ ] Optimize performance -- [ ] Enhance security - -## Checklist Summary +Also verify: -✅ All pre-publication requirements met -✅ Action tested and working -✅ Documentation complete -✅ Legal requirements satisfied -✅ Version tags created -✅ GitHub release created -✅ Marketplace listing verified -✅ Post-publication tasks completed +- Release assets are present for supported OS and architecture combinations. +- `checksums.txt` is attached. +- Marketplace examples render correctly. +- The latest documentation does not reference old monorepo package tags using + the `secretssync-v...` shape. -## Next Steps +## Manual Repairs -1. Create version tag: `git tag -a v1.0.0 -m "Initial release"` -2. Push tag: `git push origin v1.0.0` -3. Create GitHub release with Marketplace checkbox -4. Verify listing appears correctly -5. Monitor for feedback and issues +Manual tags are a repair path, not the release process. If a release workflow +fails after release-please creates a tag: ---- +1. Keep the failed tag intact while diagnosing unless the release is proven + unrecoverable. +2. Prefer rerunning the failed workflow job. +3. If a bad GitHub release was published, delete only the bad release artifacts + needed for repair. +4. Document the repair in the PR or release notes. -**Ready to publish?** Follow the steps above to make SecretSync available on the GitHub Marketplace! 🚀 +Do not create moving major aliases such as `v1` unless the repository decides +to maintain them deliberately; release-please will not update those aliases by +default. diff --git a/docs/PYTHON_BINDINGS.md b/docs/PYTHON_BINDINGS.md index a0ed1a4..02015b3 100644 --- a/docs/PYTHON_BINDINGS.md +++ b/docs/PYTHON_BINDINGS.md @@ -1,10 +1,17 @@ -# Python Bindings +# Python Integration -SecretSync provides Python bindings via [gopy](https://github.com/go-python/gopy), enabling seamless integration with Python applications, AI agents, and the Extended Data Library packages. +SecretSync is available to Python through the `extended-data[secrets]` +connector. That connector executes the supported `secretsync` CLI contract and +returns mapping-style `ExtendedDict` payloads for configuration inspection, +dry-run, merge, sync, and full pipeline operations. + +This repository also includes direct [gopy](https://github.com/go-python/gopy) +binding sources under `python/` for local experiments. Those bindings are not +the runtime adapter contract used by `extended-data`. ## Overview -The Python bindings expose the core SecretSync functionality: +The `extended-data` connector exposes the core SecretSync functionality: - **Pipeline execution**: Run merge, sync, or full pipeline operations - **Configuration validation**: Validate YAML configuration files @@ -22,17 +29,17 @@ pip install extended-data[secrets] ``` This provides: -- Native bindings when available (fastest) -- CLI fallback when bindings aren't installed +- CLI execution through the stable `secretsync` command +- Mapping-style `ExtendedDict` results - AI framework integrations (LangChain, CrewAI, Strands) - MCP server support To execute the full pipeline from Python, keep the `secretsync` CLI installed -or build the native bindings in the current environment. +and available on `PATH`. -### Option 2: Build Native Bindings +### Option 2: Build Direct gopy Bindings -For maximum performance, build the native Python bindings: +For local experiments with direct Go-to-Python bindings: ```bash # Prerequisites @@ -47,15 +54,19 @@ make python-bindings make python-install ``` -### Option 3: CLI-only Mode +### CLI Requirement -If you have the `secretsync` CLI installed, the Python connector will use it automatically: +The `extended-data` connector requires the `secretsync` CLI: ```bash go install github.com/jbcom/secrets-sync/cmd/secretsync@latest pip install extended-data[secrets] ``` +The connector relies on `secretsync pipeline --output json`, which emits the +same stable result envelope for dry-run and apply runs. Diff data is nested +under `diff` and `diff_output` when diff computation is enabled. + ## Usage ### Basic Usage @@ -66,19 +77,18 @@ from extended_data.secrets import SecretsConnector # Initialize connector connector = SecretsConnector() -# Check which mode is active -print(f"Native bindings: {connector.native_available}") +# Check that the CLI can be found print(f"CLI available: {connector.cli_available}") # Validate a configuration file -is_valid, message = connector.validate_config("pipeline.yaml") -if not is_valid: - print(f"Invalid config: {message}") +validation = connector.validate_config("pipeline.yaml") +if not validation["valid"]: + print(f"Invalid config: {validation['message']}") # Get configuration info info = connector.get_config_info("pipeline.yaml") -print(f"Sources: {info.sources}") -print(f"Targets: {info.targets}") +print(f"Sources: {info['sources']}") +print(f"Targets: {info['targets']}") ``` ### Dry Run @@ -91,16 +101,16 @@ from extended_data.secrets import SecretsConnector connector = SecretsConnector() result = connector.dry_run("pipeline.yaml") -print(f"Would process {result.target_count} targets") -print(f" Secrets to add: {result.secrets_added}") -print(f" Secrets to modify: {result.secrets_modified}") -print(f" Secrets to remove: {result.secrets_removed}") -print(f" Unchanged: {result.secrets_unchanged}") +print(f"Would process {result['target_count']} targets") +print(f" Secrets to add: {result['secrets_added']}") +print(f" Secrets to modify: {result['secrets_modified']}") +print(f" Secrets to remove: {result['secrets_removed']}") +print(f" Unchanged: {result['secrets_unchanged']}") -# View the diff output -if result.diff_output: - print("\nDiff:") - print(result.diff_output) +# Diff output may contain secret values. Inspect it only in a secure terminal +# or through a redacted reporting path. +if result["diff_output"]: + print("Diff output returned; not printing it from the example.") ``` ### Running the Pipeline @@ -126,10 +136,10 @@ options = SyncOptions( ) result = connector.run_pipeline("pipeline.yaml", options) -if result.success: - print(f"Synced {result.secrets_added} secrets in {result.duration_ms}ms") +if result["success"]: + print(f"Synced {result['secrets_added']} secrets in {result['duration_ms']}ms") else: - print(f"Error: {result.error_message}") + print("Pipeline failed. Re-run secretsync directly in a secure terminal for diagnostics.") ``` ### Merge and Sync Phases @@ -226,33 +236,24 @@ Configure in your MCP client: } ``` -## Performance - -| Mode | Relative Speed | Use Case | -|------|----------------|----------| -| Native bindings | 1x (fastest) | Production workloads | -| CLI subprocess | ~2-5x slower | Development, compatibility | - -The connector automatically uses native bindings when available. - ## Error Handling ```python -from extended_data.secrets import SecretsConnector, SyncResult +from extended_data.secrets import SecretsConnector connector = SecretsConnector() result = connector.run_pipeline("pipeline.yaml") -if not result.success: - print(f"Pipeline failed: {result.error_message}") +if not result["success"]: + print("Pipeline failed. Re-run secretsync directly in a secure terminal for diagnostics.") # Check detailed results import json - if result.results_json: - details = json.loads(result.results_json) + if result["results_json"]: + details = json.loads(result["results_json"]) for target_result in details: if not target_result.get("success"): - print(f" {target_result['target']}: {target_result.get('error')}") + print(f" {target_result['target']}: failed") ``` ## Environment Variables diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 81d4a51..3c5a7c5 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -2,13 +2,14 @@ This roadmap outlines the planned development direction for SecretSync. It's a living document that evolves based on community feedback and changing requirements. -## Current Status: v1.2.0 (December 2025) +## Current Status: v2.x -✅ **Production Ready** - All core features implemented and battle-tested +✅ **Production Ready** - The current 2.x line is the supported release line for +the independent SecretSync repository. ## Upcoming Releases -### v1.3.0 - Observability & Integrations (Q1 2026) +### v2.1.0 - Observability & Integrations **Theme**: Enhanced monitoring and ecosystem integrations @@ -36,7 +37,7 @@ This roadmap outlines the planned development direction for SecretSync. It's a l - **Configuration Templates**: Pre-built templates for common patterns - **IDE Extensions**: VS Code extension for configuration editing -### v1.4.0 - Enterprise Features (Q2 2026) +### v2.2.0 - Enterprise Features **Theme**: Advanced enterprise capabilities and governance @@ -64,7 +65,7 @@ This roadmap outlines the planned development direction for SecretSync. It's a l - **Batch Operations**: Bulk secret operations for efficiency - **Rate Limiting**: Intelligent rate limiting and backoff -### v1.5.0 - Ecosystem & Platform (Q3 2026) +### v2.3.0 - Ecosystem & Platform **Theme**: Platform features and ecosystem growth @@ -74,11 +75,11 @@ This roadmap outlines the planned development direction for SecretSync. It's a l - **Real-time Monitoring**: Live pipeline execution monitoring - **User Management**: Built-in user authentication and authorization -#### 🔧 Operator Enhancements -- **Kubernetes Operator v2**: Enhanced CRD-based management -- **GitOps Integration**: ArgoCD/Flux integration for configuration management +#### 🔧 Kubernetes Runtime +- **GitOps Examples**: ArgoCD/Flux examples for scheduled pipeline runners - **Helm Chart Improvements**: Advanced deployment options -- **Multi-Cluster Support**: Manage secrets across multiple clusters +- **Multi-Cluster Patterns**: Run scoped pipeline jobs per cluster or account boundary +- **Native API Research**: Evaluate a future controller only if it can be owned end to end #### 🌐 API & Integrations - **REST API**: Full REST API for programmatic access @@ -92,7 +93,7 @@ This roadmap outlines the planned development direction for SecretSync. It's a l - **Shell Integration**: Bash/Zsh completion and integration - **Configuration Management**: CLI-based configuration management -## Future Considerations (v2.0+) +## Future Considerations (v3.0+) ### Major Architecture Evolution @@ -129,7 +130,7 @@ Based on community feedback, we're prioritizing: - **GitHub Issues**: Share your use cases and requirements - **Feature Requests**: Create detailed feature requests with business justification - **User Surveys**: Participate in periodic user surveys -- **Community Calls**: Join monthly community calls (coming in v1.3.0) +- **Community Calls**: Join monthly community calls when scheduled ### 🤝 Contributions - **Code Contributions**: Implement features you need @@ -157,19 +158,12 @@ Based on community feedback, we're prioritizing: - **Release Candidates**: 1 week before major releases - **Early Access**: Available for enterprise partners -## Backwards Compatibility +## Clean Break Policy -### Compatibility Promise -- **Configuration**: Backwards compatible within major versions -- **API**: Semantic versioning with deprecation notices -- **CLI**: Backwards compatible with deprecation warnings -- **Migration Tools**: Automated migration for breaking changes - -### Deprecation Policy -- **6 Month Notice**: Minimum 6 months notice for deprecations -- **Migration Guides**: Detailed migration documentation -- **Automated Tools**: CLI tools to assist with migrations -- **Support**: Extended support for deprecated features +- **Configuration**: Prefer one current shape over compatibility aliases. +- **API**: Breaking changes are acceptable when they keep the implementation honest. +- **CLI**: Removed flags and fields should fail loudly with clear replacement guidance. +- **Migration Docs**: Document replacement configuration rather than carrying shims. ## Success Metrics diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 28895ee..11b74ae 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -8,26 +8,46 @@ As this service is effectively an event bus for your various secret stores and t The service is designed to be secure by default, preferrring more explicit configuration as opposed to insecure defaults. This does mean initial set up will not be one-liner fire-and-forget. That is intentional, as it forces you to think about the security implications of the decisions you are making rather than starting up the process insecurely, forgetting about it, and then being surprised when something goes wrong. If started with no configuration and no command line flags, the service will exit with an error. All components of the service default to disabled, you must explicitly enable the components you wish to use. -### Segregation of Duties +### Logging and Diagnostics + +SecretSync logs operational metadata for troubleshooting, not secret material. +Vault and AWS client initialization logs are limited to explicit non-secret +fields such as address, path, auth method, region, role ARN, cache settings, and +boolean capability flags. Raw secret values, raw Vault secret response objects, +raw AWS secret response objects, and raw client structures must not be written +to logs at any level. + +Diagnostics may include secret paths, target names, account IDs, error classes, +request IDs, durations, counts, and operation names. Treat those logs as +operationally sensitive, but they should not contain the bytes being synced. +Keep `--log-level debug` and `--log-level trace` restricted to trusted +operators and secured log sinks. + +Machine-readable `secretsync pipeline --output json` result envelopes redact +common secret-bearing diagnostic fragments in top-level and per-target error +strings before serialization, including bearer tokens, password or token +assignments, API key assignments, client secrets, and matching URL query +parameters. Downstream consumers should still treat `error_message`, per-target +`error`, and `diff_output` as operationally sensitive and avoid copying them to +untrusted logs, comments, or chat systems without their own policy checks. +GitHub Actions annotation output escapes workflow-command data in target names +and secret paths before writing groups, notices, or warnings. -By default, all components of the app default to disabled. This is to ensure that you are only enabling the components you need. To run in "single binary mode", you must explicitly enable each component. This is to ensure that you are aware of the security implications of each component you are enabling. This also enables you to more easily deploy the service in a microservices architecture, where you can run the webhook service in one container and the sync operator in another. +### Segregation of Duties +SecretSync runs as an explicit pipeline command. Split duties by separating +configuration authors, CI/CD approvers, runtime identities, and target account +roles. In Kubernetes, prefer a scheduled job with a dedicated service account +and narrowly scoped projected credentials. ## Security Configuration -### API Server - -The service exposes one API process, the `Event Server` which is responsible for handling audit log events from Vault. - -The API server can be secured either with mTLS, an auth token string, or both. Of course you can also - and are recommended to - further secure communication via service mesh, network policies, or other network security controls. - -If in doubt, it is recommended to use mTLS for all communication with the service. This will ensure that all communication is encrypted and that the client is authenticated. If you are running in a Kubernetes cluster, you can use `cert-manager` to automatically provision certificates for your services. If using token auth, the token must be passed as a `X-Vault-Secret-Sync-Token` header in the request. +### Pipeline Runner -### Queue - -If deployed in HA / microservice mode, the service will rely on a queue to communicate between the `Event Server` and the `Sync Operator`. - -The event server only needs to publish to the queue, and the sync operator only needs to consume from the queue. All appropriate measures should be taken to secure the queue, including network policies, authentication, and encryption. +The supported runtime surface is `secretsync pipeline`. It does not expose an +ingress API by default. Network access should be outbound-only to the configured +secret stores and cloud provider APIs unless your environment adds its own +wrapping service. ### Secret Stores @@ -37,15 +57,22 @@ It is recommended to use a service account with the least privileges necessary t All operations performed by the service include an `X-Vault-Sync: true` header to identify the action as being performed by the sync service. -As the operator itself is effectively `root` for lack of a better analogy, it is critical to ensure that the operator is only deployed in environments where it can be trusted. Furthermore, as the operator will dutifully do what it is told to do, it is critical to ensure proper RBAC and policy is in place around modifying operator and / or sync configurations. If deployed in a Kubernetes cluster, it is recommended to use the Kubernetes native RBAC system to limit access to the operator and sync configurations. +The pipeline runtime is highly privileged relative to the stores it can read +from and write to. Keep configuration changes behind review, protect the merge +store, and scope runtime identities to only the source paths and target accounts +the pipeline needs. Every sync operation will instantiate a new client to the source and destination secret store, and will close the client after the operation is complete. At no time are client objects reused between sync operations. This is to ensure that the client is not left open and vulnerable to attack. #### AWS -If you are running in AWS EKS, you can use IAM roles to grant the operator access to the AWS Secrets Manager. If you are running in a different environment, you can use the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables to provide the operator with the necessary credentials. If you are using access keys, it is recommended to rotate these regularly, and utilize a project such as [External Secrets Operator](https://external-secrets.io/latest/) to manage the lifecycle of the access keys into the operator. +If you are running in AWS EKS, use workload identity or IRSA for the pipeline +runner. If you are running in a different environment, you can use +`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, but prefer short-lived +credentials and rotate any static keys regularly. -For cross-account access you must configure an IAM role in your target account which can be assumed by the identity associated with the operator. +For cross-account access you must configure an IAM role in your target account +which can be assumed by the identity associated with the pipeline runner. Your role will need to have the following permissions: @@ -103,49 +130,21 @@ and an example trusted entity configuration: } ``` -#### GCP - -If you are running in GCP GKE, you can use GCP service accounts to grant the operator access to GCP Secret Manager. If you are running in a different environment, you can use the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to provide the operator with the necessary credentials. If you are using service accounts, it is recommended to rotate these regularly, and utilize a project such as [External Secrets Operator](https://external-secrets.io/latest/) to manage the lifecycle of the service account keys into the operator. - -This identity associated with the operator must be granted the following permissions: - -- `secretmanager.versions.access` -- `secretmanager.versions.add` -- `secretmanager.versions.create` -- `secretmanager.versions.destroy` -- `secretmanager.versions.disable` -- `secretmanager.versions.enable` -- `secretmanager.versions.get` -- `secretmanager.versions.list` -- `secretmanager.versions.restore` -- `secretmanager.versions.update` -- `secretmanager.secrets.access` -- `secretmanager.secrets.addVersion` -- `secretmanager.secrets.create` -- `secretmanager.secrets.delete` -- `secretmanager.secrets.get` -- `secretmanager.secrets.list` -- `secretmanager.secrets.update` - -#### GitHub - -GitHub requires a GitHub App installed in the account with access to the level of secrets you desire. When you create your GitHub App, it will have an `installId`, `appId`, and `privateKey`. You will need to provide these to the operator to authenticate with GitHub. It is recommended to rotate the private key regularly, and utilize a project such as [External Secrets Operator](https://external-secrets.io/latest/) to manage the lifecycle of the private key into the operator. - -Heere is an example store configuration: - -```yaml -stores: - github: - installId: 12345 - appId: 67890 - privateKeyPath: "/path/to/private/key" -``` - ### HashiCorp Vault -If you are running in a Kubernetes cluster, you can use the Kubernetes auth method to authenticate the operator with Vault. If you are running in a different environment, you can use the `VAULT_TOKEN` environment variable to provide the operator with the necessary token. If you are using tokens, it is recommended to rotate these regularly, and utilize a project such as [External Secrets Operator](https://external-secrets.io/latest/) to manage the lifecycle of the tokens into the operator. +If you are running in Kubernetes, you can use the Kubernetes auth method to +authenticate the pipeline runner with Vault. If you are running in a different +environment, use AppRole or `VAULT_TOKEN`. Rotate long-lived credentials +regularly and prefer a secret manager or workload identity mechanism to inject +them into the runtime environment. ## Vulnerability Reporting -If you believe you have found a security vulnerability in this project, please report it privately to the project maintainers. If you are unsure whether the issue is a security vulnerability, please report it anyway. We take all reports seriously and will respond promptly to your inquiry. Please do not disclose the issue publicly until we have had a chance to address it. You can report a security vulnerability by emailing [robert@lestak.sh](mailto:robert@lestak.sh). Please include the word "SECURITY" in the subject line. \ No newline at end of file +If you believe you have found a security vulnerability in this project, please +report it privately through +[GitHub Security Advisories](https://github.com/jbcom/secrets-sync/security/advisories) +or email [security@jbcom.dev](mailto:security@jbcom.dev). If you are unsure +whether the issue is a security vulnerability, please report it anyway. We take +all reports seriously and will respond promptly to your inquiry. Please do not +disclose the issue publicly until we have had a chance to address it. diff --git a/docs/SUPPORT.md b/docs/SUPPORT.md index 02306cb..bc33033 100644 --- a/docs/SUPPORT.md +++ b/docs/SUPPORT.md @@ -93,7 +93,7 @@ When reporting bugs, please include: # If using GitHub Action # Include the version/tag from your workflow - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z ``` 2. **Configuration** (sanitized - remove secrets!) @@ -225,17 +225,16 @@ Yes! SecretSync is free and open source (MIT License). ### Can I use this in production? -Yes! SecretSync is production-ready. Many organizations use it daily. +SecretSync is intended for production use when Vault, AWS IAM, OIDC, and +release pinning are configured for your environment. Run dry-run validation in +CI before applying changes. ### How do I upgrade? For GitHub Actions: ```yaml -# Pin to the current package release tag (recommended) -uses: jbcom/secrets-sync@secretssync-v2.0.2 - -# Pin to an exact package release tag (most stable) -uses: jbcom/secrets-sync@secretssync-v2.0.2 +# Pin to an exact package release tag +uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z # Track the branch tip (not recommended for production) uses: jbcom/secrets-sync@main diff --git a/docs/TWO_PHASE_ARCHITECTURE.md b/docs/TWO_PHASE_ARCHITECTURE.md index c7ad03b..d9fb880 100644 --- a/docs/TWO_PHASE_ARCHITECTURE.md +++ b/docs/TWO_PHASE_ARCHITECTURE.md @@ -5,7 +5,7 @@ The secrets synchronization pipeline operates in two distinct phases: 1. **MERGE Phase** (optional): Aggregate secrets from multiple sources into a unified pool -2. **SYNC Phase**: Propagate secrets from source(s) to target(s) +2. **SYNC Phase**: Propagate merged bundles into AWS Secrets Manager targets ``` ┌──────────────────────────────────────────────────────────────────────────────┐ @@ -25,7 +25,7 @@ The secrets synchronization pipeline operates in two distinct phases: │ │ └─────────┘ │ │ • Aggregation │ │ │ │ │ ┌─────────┐ │ │ • Deduplication │ │ │ │ │ │ Source3 │──┘ │ • Inheritance │ │ │ -│ │ │ (HTTP) │ │ • DeepMerge │ │ │ +│ │ │ (Vault) │ │ • DeepMerge │ │ │ │ │ └─────────┘ └────────┬─────────┘ │ │ │ │ │ │ │ │ └───────────────────────────────┼──────────────────────────────────────┘ │ @@ -40,10 +40,10 @@ The secrets synchronization pipeline operates in two distinct phases: │ │ │ SOURCE │────▶│ Target1 (AWS) │ │ │ │ │ │ (Merge Store │ └─────────────────┘ │ │ │ │ │ or Direct) │ ┌─────────────────┐ │ │ -│ │ │ │────▶│ Target2 (Vault)│ │ │ +│ │ │ │────▶│ Target2 (AWS) │ │ │ │ │ │ │ └─────────────────┘ │ │ │ │ │ │ ┌─────────────────┐ │ │ -│ │ │ │────▶│ Target3 (GCP) │ │ │ +│ │ │ │────▶│ Target3 (AWS) │ │ │ │ │ └──────────────────┘ └─────────────────┘ │ │ │ │ │ │ │ └───────────────────────────────────────────────────────────────────────┘ │ @@ -84,23 +84,19 @@ For each target in topological order: ### SYNC Phase -**Purpose**: Propagate secrets from source to target stores. +**Purpose**: Propagate merged secrets into AWS Secrets Manager targets. **Source determination**: - If MERGE phase ran: Merge store becomes the source automatically - If SYNC-only: Explicitly configured source **Supported Store Combinations**: -| Source | Target | Status | -|--------|--------|--------| -| Vault | AWS Secrets Manager | ✅ Supported | -| Vault | Vault | ✅ Supported | -| Vault | GCP Secret Manager | ✅ Supported | -| Vault | GitHub Secrets | ✅ Supported | -| Vault | Kubernetes Secrets | ✅ Supported | -| AWS SM | AWS SM | ✅ Supported | -| AWS SM | Vault | ✅ Supported | -| S3 | AWS SM | ✅ Supported (via S3 merge store) | +| Source or Merge Store | Target | Status | +|-----------------------|--------|--------| +| Vault KV2 | AWS Secrets Manager | ✅ Supported | +| AWS Secrets Manager | AWS Secrets Manager | ✅ Supported | +| Vault merge store | AWS Secrets Manager | ✅ Supported | +| S3 merge store | AWS Secrets Manager | ✅ Supported | **Sync Process**: ``` @@ -213,19 +209,32 @@ secretsync pipeline --config config.yaml --merge-only Both phases support diff computation: ```bash -# Dry-run with diff output +# Dry-run with machine-readable result and nested diff output secretsync pipeline --config config.yaml --dry-run --output json # Output: { - "dry_run": true, - "summary": { - "added": 5, - "modified": 2, - "removed": 0, - "unchanged": 43 - }, - "targets": [...] + "success": true, + "target_count": 2, + "secrets_processed": 50, + "secrets_added": 5, + "secrets_modified": 2, + "secrets_removed": 0, + "secrets_unchanged": 43, + "duration_ms": 1284, + "results": [...], + "diff_output": "{...}", + "diff": { + "dry_run": true, + "summary": { + "added": 5, + "modified": 2, + "removed": 0, + "unchanged": 43, + "total": 50 + }, + "targets": [...] + } } ``` @@ -263,12 +272,14 @@ targets: imports: [common-secrets] Staging: - inherits: Base - imports: [staging-secrets] + imports: + - Base + - staging-secrets Production: - inherits: Staging - imports: [production-secrets] + imports: + - Staging + - production-secrets ``` **Processing Order**: diff --git a/docs/USAGE.md b/docs/USAGE.md index 72dfb23..1cf78bf 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -1,297 +1,131 @@ # Usage -## YAML Configuration +SecretSync is a pipeline runner. It reads configured sources, merges secret +material into a merge store, and syncs the resulting bundles into AWS Secrets +Manager targets. -```yaml -apiVersion: secretsync.extendeddata.dev/v1alpha1 -kind: SecretSync -metadata: - name: "example-sync" - namespace: "default" -spec: - dryRun: false - syncDelete: false - suspend: false - source: - address: "https://vault.example.com" - path: "foo/bar/(.*)" - namespace: "robertlestak/example" - filters: - regex: - include: - - "foo/bar/hello-[0-9]+" - exclude: - - "foo/bar/no[^abc]+" - paths: - include: - - "foo/bar/hello" - exclude: - - "foo/bar/no" - dest: - - vault: - address: "https://vault2.example.com" - path: "hello/world/$1" - namespace: "robertlestak/example" - - vault: - address: "https://vault3.example.com" - path: "another/vault" - - aws: - name: "example-secret" - region: "us-west-2" - roleArn: "arn:aws:iam::123456789012:role/role-name" - encryptionKey: "alias/aws/secretsmanager" - replicaRegions: ["us-east-1"] - - github: - repo: "example-repo" - owner: "robertlestak" - - gcp: - project: "example-project" - name: "example-secret" - - http: - url: "https://example.com/my/app" - method: "POST" - headerSecret: "default/header-secret" - headers: - Content-Type: "application/json" - template: | - { - "custom": { - "{{ .Key }}": "{{ .Value }}" - } - } - webhooks: - - event: success - url: "https://example.com/success" - method: "POST" - headers: - Content-Type: "application/json" - template: | - { - "custom": { - "event": "{{ .Event }}", - "message": "{{ .Message }}", - "name": "{{ .SecretSync.Name }}" - "source.address": "{{ .SecretSync.Spec.Source.Address }}" - } - } - - event: failure - url: "https://example.com/failure" - method: "POST" - headers: - Content-Type: "application/json" - template: | - { - "custom": { - "event": "{{ .Event }}", - "message": "{{ .Message }}", - "name": "{{ .SecretSync.Name }}" - "source.address": "{{ .SecretSync.Spec.Source.Address }}" - } - } -``` - -`metadata` is that can be used to identify the sync operation. Both can be ommitted, however if `name` is omitted, to trigger a manual sync you will first need to authenticate and access the `/configs` endpoint to get the auto-generated name for your respective config. - -`spec.source` and `spec.dest` are the source and destination of the sync operation. - -`authMethod` and `role` can be omitted to use the default global method and role, if defined. If they are specified, they will be used to authenticate to the Vault instance. Optionally, `token` can be provided as a mustache template referencing an environment variable. If provided, this will always override configured auth methods. - -`path` can either be provided as a path to a single vault kv secret, or as a regex string. If regex, all matching child secrets will be synced. Wildcarding will recurse into child paths, so `foo/test/(.*)` will sync `foo/test/foo` and `foo/test/foo/bar`, but not `foo/test2/foo`. However `foo/test(.*)` will sync `foo/test` and `foo/test2`. You can also use capture groups to rewrite the path in the destination. For example, `foo/(test)/(bar)` will sync `foo/test/bar` and rewrite it to `test/bar` in the destination. - -If you set `source.cidr` to the CIDR in which the source vault is deployed (as seen from ingestion point - so if this is a public Vault, this will be the outbound NAT/IGW), it will enable multiple source vaults to sync through a single instance of the operator. You can also set `x-vault-tenant` header in the log shipping config to specify the source vault from which that log is coming from. - -### Source Determination - -By default, the Vault Audit log contains no contextual information about what Vault it is coming from. To enable this operator to connect to multiple vaults, multi-tenant source determination logic is used based on the following order of precendence. - -- `X-Vault-Tenant` Header: If an `x-vault-tenant` header is set to a Vault URL (as defined in the example fluentd config below), this will be used to perform an exact lookup against configured Vault Source Addresses. -- `X-Forwarded-For` Header: If an `x-forwarded-for` header is set (and the `x-vault-tenant` is not), the operator will perform a source lookup by finding the `source.cidr` which contains the caller IP. -- `Remote IP Address`: If both the `x-vault-tenant` and `x-forwarded-for` headers do not exist, the operator will use the caller's IP address and will perform a source lookup by finding the `source.cidr` which contains the caller IP. - - -### Filters - -Filters can be applied to the sync to include or exclude secrets based on either a regex pattern or a path pattern. The path filter is an explicit match, while the regex filter is a regex pattern match. If both filters are present, the secret must match both filters to be included in the sync. - -```yaml - filters: - regex: - include: - - "foo/bar/hello-[0-9]+" - exclude: - - "foo/bar/no[^abc]+" - paths: - include: - - "foo/bar/hello" - exclude: - - "foo/bar/no" -``` - - -### Destination Configuration - -The destination is configured in the same way as the source, with the exception that the `driver` field can be specified. If no driver is specified, the default driver is `vault`. - -#### Vault (Driver: `vault`) - -The Vault destination driver will write the secret to the target Vault instance. - -```yaml - dest: - - vault: - address: "https://vault.example.com" - path: "foo/test2/(.*)" - namespace: "" - authMethod: "" - role: "" - ttl: 1m # optional, defaults to token default lease time - merge: false # optional, default false. false will overwrite existing secrets with values from vault, merge will merge the two, overwriting only the keys that are present in the new secret -``` +## Configuration -#### GitHub (Driver: `github`) - -The GitHub destination driver will write the secret to a GitHub repository or organization. - -```yaml - dest: - - github: - repo: "example-repo" - env: "" # optional, default empty. Set to a specific environment to sync to within a repo if needed - owner: "robertlestak" # optional, will default to the company org - org: false # optional, default false. set to true to set org secret rather than repo secret - merge: false # optional, default true. false will overwrite existing secrets with values from vault, merge will merge the two -``` - -Note that since GitHub secrets do not have a concept of pathing, if you are syncing a wildcard source path, the secrets will be overwritten in the destination repository with a last-write-wins strategy. - -#### AWS Secrets Manager (Driver: `aws`) - -The AWS destination driver will write the secret to AWS Secrets Manager. +Use the current pipeline configuration shape: ```yaml - dest: - - aws: - name: "example-secret" - region: "us-west-2" # optional, default us-east-1 - roleArn: "arn:aws:iam::123456789012:role/role-name" # optional, default empty. Set to a specific role to assume when writing to secrets manager - encryptionKey: "alias/aws/secretsmanager" # optional, default empty. Set to a specific KMS key to use for encryption - replicaRegions: [] # optional, default empty. Set to a list of regions to replicate the secret to +vault: + address: https://vault.example.com/ + namespace: platform/secrets + auth: + approle: + mount: approle + role_id: ${VAULT_ROLE_ID} + secret_id: ${VAULT_SECRET_ID} + +aws: + region: us-east-1 + +sources: + shared: + vault: + mount: shared + paths: + - app/* + production-overrides: + vault: + mount: production + +merge_store: + s3: + bucket: secretsync-merge-store + prefix: merged/ + kms_key_id: alias/secretsync-merge-store + +targets: + staging: + account_id: "111111111111" + region: us-east-1 + secret_prefix: /staging/ + imports: + - shared + production: + account_id: "222222222222" + region: us-east-1 + secret_prefix: /production/ + imports: + - staging + - production-overrides + +dynamic_targets: + analytics_sandboxes: + discovery: + organizations: + ous: + - ou-xxxx-sandbox + recursive: true + tag_filters: + - key: Team + values: + - analytics + operator: equals + imports: + - shared + secret_prefix: /sandbox/ + +pipeline: + merge: + parallel: 4 + sync: + parallel: 4 + delete_orphans: false + dry_run: false + continue_on_error: true ``` -#### GCP Secret Manager (Driver: `gcp`) +`sources` define where secrets are read from. `merge_store` defines the +intermediate bundle store. `targets` define AWS accounts and the source or +target imports that feed them. A target imports another target by listing that +target name in `imports`. -The GCP destination driver will write the secret to GCP Secret Manager in the specified project. +## Validate -```yaml - dest: - - gcp: - project: "example-project" - name: "example-secret" - replicationLocations: [] # optional, default empty. Set to a list of regions to replicate the secret to. If empty, all regions will be used +```bash +secretsync validate --config config.yaml ``` +Validation checks required targets, account ID formats, merge store settings, +dynamic discovery settings, and target inheritance cycles. -Note that since GCP Secret Manager does not support the `/` character, the sync operator will replace `/` with `-` in the secret name. This generally only applies when using a wildcard source path. - - - -#### HTTP (Driver: `http`) +## Plan -The HTTP destination driver will make an HTTP request to the specified URL with the secret data as the body of the request. By default this will be a POST request with a JSON body, but the method, headers, and body can be customized. Note that this will be sending your secrets in plain text to the specified URL, so ensure that the destination is within your control and secure. - -```yaml - dest: - - http: - url: "https://example.com/my/app" - method: "POST" # optional, default POST. Set to the HTTP method to use for the request - headerSecret: "default/header-secret" # optional, default empty. Set to the name of a kubernetes secret in the format namespace/name to use as the header KV pairs for the request - headers: # optional, default empty. Set to a map of headers to include in the request - Content-Type: "application/json" - template: | # optional, default empty. Set to a template to use for the request body. The template is a Go template with the following variables available: .Key, .Value, .Namespace, .Path, .Secret, .Timestamp - { - "custom": { - "{{ .Key }}": "{{ .Value }}" - } - } +```bash +secretsync pipeline --config config.yaml --dry-run --diff --output json ``` -#### Webhooks +Dry runs load the same configuration and compute the same target graph as an +apply run, but skip writes to destination stores. -Webhooks can be configured to send a POST request to a specified URL when a sync event occurs. The event can be either `success` or `failure`, and the request will include a JSON body with information about the event. The template can be customized to include any information from the sync event. +## Apply -```yaml - webhooks: - - event: success - url: "https://example.com/success" - method: "POST" # optional, default POST. Set to the HTTP method to use for the request - headers: # optional, default empty. Set to a map of headers to include in the request - Content-Type: "application/json" - template: | # optional, default empty. Set to a template to use for the request body. The template is a Go template with the following variables available: .Event, .Message, .SecretSync - { - "custom": { - "event": "{{ .Event }}", - "message": "{{ .Message }}", - "name": "{{ .SecretSync.Name }}" - "source.address": "{{ .SecretSync.Spec.Source.Address }}" - } - } - - event: failure - url: "https://example.com/failure" - method: "POST" # optional, default POST. Set to the HTTP method to use for the request - headers: # optional, default empty. Set to a map of headers to include in the request - Content-Type: "application/json" - template: | # optional, default empty. Set to a template to use for the request body. The template is a Go template with the following variables available: .Event, .Message, .SecretSync - { - "custom": { - "event": "{{ .Event }}", - "message": "{{ .Message }}", - "name": "{{ .SecretSync.Name }}" - "source.address": "{{ .SecretSync.Spec.Source.Address }}" - } - } +```bash +secretsync pipeline --config config.yaml --diff --output human ``` -## Operations - -### Kubernetes +Use `--targets staging,production` to limit a run to selected targets and their +dependencies. Use `--merge-only` or `--sync-only` when an operational workflow +needs to split the two phases. -When deployed in Kubernetes, `SecretSync` operations are exposed through the Kubernetes API. `SecretSync` resources can be created, updated, and deleted through the Kubernetes API. - -```yaml -cat < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/architecture/HLA-microservice.drawio.png b/docs/architecture/HLA-microservice.drawio.png deleted file mode 100644 index 6cf5f64..0000000 Binary files a/docs/architecture/HLA-microservice.drawio.png and /dev/null differ diff --git a/docs/architecture/HLA.drawio b/docs/architecture/HLA.drawio deleted file mode 100644 index 7952b68..0000000 --- a/docs/architecture/HLA.drawio +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/architecture/HLA.drawio.png b/docs/architecture/HLA.drawio.png deleted file mode 100644 index 2de24a9..0000000 Binary files a/docs/architecture/HLA.drawio.png and /dev/null differ diff --git a/docs/development/contributing.md b/docs/development/contributing.md index e28c972..f7e905d 100644 --- a/docs/development/contributing.md +++ b/docs/development/contributing.md @@ -15,6 +15,9 @@ go mod download ## Running Tests ```bash +# Vulnerability scan +go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... + # Unit tests go test ./... diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 4e13e10..b2935de 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -2,13 +2,13 @@ ## Requirements -- Go 1.25+ +- Go 1.26.4+ - Docker (optional, for containerized runs or the GitHub Action image) ## Install the CLI ```bash -# Install the latest CLI from the monorepo module path +# Install the latest CLI from the standalone module path go install github.com/jbcom/secrets-sync/cmd/secretsync@latest ``` @@ -34,10 +34,11 @@ make build ## GitHub Action -Use the packaged action from the monorepo subdirectory and pin to a package tag: +Use the packaged action from this standalone repository and pin to a release +tag. Replace `X.Y.Z` with a published release version: ```yaml -- uses: jbcom/secrets-sync@secretssync-v2.0.2 +- uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml ``` diff --git a/docs/testing/organizations-discovery-integration-tests.md b/docs/testing/organizations-discovery-integration-tests.md index 4e32304..cd50ff4 100644 --- a/docs/testing/organizations-discovery-integration-tests.md +++ b/docs/testing/organizations-discovery-integration-tests.md @@ -56,7 +56,8 @@ dynamic_targets: test_ou_discovery: discovery: organizations: - ou: "ou-xxxx-development" + ous: + - "ou-xxxx-development" imports: - test-secrets ``` @@ -70,7 +71,7 @@ dynamic_targets: **Validation:** ```bash # Run the pipeline in dry-run mode -./vss pipeline --config test-config.yaml --dry-run +secretsync pipeline --config test-config.yaml --dry-run # Verify discovered targets in output # Should show accounts directly in the OU only @@ -84,7 +85,8 @@ dynamic_targets: test_recursive_discovery: discovery: organizations: - ou: "ou-xxxx-workloads" + ous: + - "ou-xxxx-workloads" recursive: true imports: - test-secrets @@ -104,7 +106,7 @@ aws organizations list-organizational-units-for-parent --parent-id ou-xxxx-workl # Then check each child OU # Compare with discovery output -./vss pipeline --config test-config.yaml --dry-run | grep "Discovered target" +secretsync pipeline --config test-config.yaml --dry-run | grep "Discovered target" ``` ### Test 3: Tag-Based Filtering (All Accounts) @@ -137,7 +139,7 @@ for account in $(aws organizations list-accounts --query 'Accounts[].Id' --outpu done # Compare with discovery output -./vss pipeline --config test-config.yaml --dry-run +secretsync pipeline --config test-config.yaml --dry-run ``` ### Test 4: Combined OU and Tag Filtering @@ -148,7 +150,8 @@ dynamic_targets: test_combined_filtering: discovery: organizations: - ou: "ou-xxxx-production" + ous: + - "ou-xxxx-production" tags: Environment: production imports: @@ -173,7 +176,7 @@ for account in $(aws organizations list-accounts-for-parent --parent-id ou-xxxx- done # Run discovery -./vss pipeline --config test-config.yaml --dry-run +secretsync pipeline --config test-config.yaml --dry-run ``` ### Test 5: Recursive OU with Tag Filtering @@ -184,7 +187,8 @@ dynamic_targets: test_recursive_with_tags: discovery: organizations: - ou: "ou-xxxx-workloads" + ous: + - "ou-xxxx-workloads" recursive: true tags: AutoManaged: enabled @@ -202,7 +206,7 @@ dynamic_targets: ```bash # This requires checking all accounts in the OU hierarchy # and verifying they have the required tags -./vss pipeline --config test-config.yaml --dry-run --log-level debug +secretsync pipeline --config test-config.yaml --dry-run --log-level debug ``` ### Test 6: Account Name Resolution @@ -227,7 +231,7 @@ dynamic_targets: **Validation:** ```bash # Check that discovered targets have meaningful names -./vss pipeline --config test-config.yaml --dry-run | grep "Discovered target" +secretsync pipeline --config test-config.yaml --dry-run | grep "Discovered target" # Names should be sanitized: # "Analytics Sandbox" → "Analytics_Sandbox" @@ -256,7 +260,7 @@ dynamic_targets: ### Enable Debug Logging ```bash -./vss pipeline --config config.yaml --dry-run --log-level debug +secretsync pipeline --config config.yaml --dry-run --log-level debug ``` ### Common Issues diff --git a/docs_markdown_test.go b/docs_markdown_test.go new file mode 100644 index 0000000..b1c0e60 --- /dev/null +++ b/docs_markdown_test.go @@ -0,0 +1,304 @@ +package secretsync_test + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestMarkdownFencedCodeBlocksAreBalanced(t *testing.T) { + var offenders []string + for _, root := range []string{"README.md", "docs"} { + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".md" { + return nil + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + if strings.Count(string(content), "```")%2 != 0 { + offenders = append(offenders, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + + if len(offenders) > 0 { + t.Fatalf("markdown files have unbalanced fenced code blocks: %s", strings.Join(offenders, ", ")) + } +} + +func TestDeploymentGuideUsesCurrentPipelineSurface(t *testing.T) { + paths := []string{"docs/ARCHITECTURE.md", "docs/DEPLOYMENT.md"} + for _, required := range []string{ + "secretsync pipeline", + "--dry-run", + "--diff", + "--output json", + "kind: CronJob", + } { + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if !strings.Contains(string(content), required) { + t.Fatalf("%s should document current deployment surface %q", path, required) + } + } + } + deploymentGuide, err := os.ReadFile("docs/DEPLOYMENT.md") + if err != nil { + t.Fatalf("read docs/DEPLOYMENT.md: %v", err) + } + if !strings.Contains(string(deploymentGuide), "jbcom/secrets-sync@secrets-sync-vX.Y.Z") { + t.Fatal("docs/DEPLOYMENT.md should document the GitHub Action release tag") + } + + for _, forbidden := range []string{ + "Vault Secrets Sync service", + "Event Server", + "Sync Operator", + "-operator", + "-events", + "memory queue", + "microservices mode", + "REST webhook endpoint", + "SecretSync resources", + } { + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if strings.Contains(string(content), forbidden) { + t.Fatalf("%s should not document stale deployment surface %q", path, forbidden) + } + } + } +} + +func TestStaleArchitectureDiagramsAreNotPublished(t *testing.T) { + if _, err := os.Stat("docs/architecture"); !os.IsNotExist(err) { + t.Fatal("docs/architecture should not publish stale operator architecture diagrams") + } +} + +func TestArchitectureAuditCurrentShapeReferencesExistingPaths(t *testing.T) { + content, err := os.ReadFile("docs/ARCHITECTURE_AUDIT.md") + if err != nil { + t.Fatalf("read docs/ARCHITECTURE_AUDIT.md: %v", err) + } + + text := string(content) + start := strings.Index(text, "## Current Shape") + end := strings.Index(text, "## Future Release Work") + if start < 0 || end < 0 || end <= start { + t.Fatal("docs/ARCHITECTURE_AUDIT.md should have a current shape section before future release work") + } + + currentShape := text[start:end] + for _, forbidden := range []string{ + "api/v1alpha1", + "Kubernetes API types", + } { + if strings.Contains(currentShape, forbidden) { + t.Fatalf("architecture audit should not document removed current path %q", forbidden) + } + } + if !strings.Contains(currentShape, "deploy/charts/secretsync") { + t.Fatal("architecture audit should document the Helm runner chart path") + } + + for _, match := range regexp.MustCompile("`([^`]+)`").FindAllStringSubmatch(currentShape, -1) { + path := match[1] + if !isArchitectureAuditRepoPath(path) { + continue + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("architecture audit references missing current path %s: %v", path, err) + } + } +} + +func TestArchitectureAuditDoesNotKeepMigrationGapFilename(t *testing.T) { + if _, err := os.Stat("docs/ARCHITECTURE_GAP_ANALYSIS.md"); !os.IsNotExist(err) { + t.Fatal("docs/ARCHITECTURE_GAP_ANALYSIS.md should not be published in the standalone repository") + } +} + +func TestArchitectureAuditIsDiscoverableFromPublicDocs(t *testing.T) { + for _, path := range []string{"README.md", "docs/ARCHITECTURE.md"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if !strings.Contains(string(content), "ARCHITECTURE_AUDIT.md") { + t.Fatalf("%s should link to docs/ARCHITECTURE_AUDIT.md", path) + } + } +} + +func TestContributingGuideUsesCurrentRepositoryShape(t *testing.T) { + content, err := os.ReadFile("CONTRIBUTING.md") + if err != nil { + t.Fatalf("read CONTRIBUTING.md: %v", err) + } + + text := string(content) + for _, required := range []string{ + "pkg/client/", + "pkg/driver", + "pkg/pipeline", + "driver.DriverName", + } { + if !strings.Contains(text, required) { + t.Fatalf("CONTRIBUTING.md should document current repository shape %q", required) + } + } + + for _, forbidden := range []string{ + "stores/newstore", + "github.com/jbcom/secrets-sync/pkg/store", + "├── stores/", + } { + if strings.Contains(text, forbidden) { + t.Fatalf("CONTRIBUTING.md should not document removed store surface %q", forbidden) + } + } +} + +func TestPublicGitHubDirectoryLinksUseTreeURLs(t *testing.T) { + brokenDirectoryLink := regexp.MustCompile(`https://github\.com/jbcom/secrets-sync/(docs|examples)(?:[)\s]|$)`) + var offenders []string + + for _, root := range []string{"README.md", "docs", "CONTRIBUTING.md", "SECURITY.md"} { + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".md" { + return nil + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + if brokenDirectoryLink.Match(content) { + offenders = append(offenders, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + + if len(offenders) > 0 { + t.Fatalf("GitHub directory links should use /tree/main/... URLs:\n%s", strings.Join(offenders, "\n")) + } +} + +func isArchitectureAuditRepoPath(path string) bool { + if strings.HasPrefix(path, "jbcom/") || strings.Contains(path, ":") { + return false + } + for _, prefix := range []string{"cmd/", "deploy/", "docs/", "pkg/", "python/"} { + if strings.HasPrefix(path, prefix) { + return true + } + } + return path == "pkg" || path == "action.yml" || strings.HasSuffix(path, ".go") +} + +func TestGettingStartedUsesCurrentPipelineConfigShape(t *testing.T) { + paths := []string{"README.md", "docs/GETTING_STARTED.md"} + + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := string(content) + for _, required := range []string{ + "merge_store:", + "account_id:", + "secret_prefix:", + "dynamic_targets:", + } { + if !strings.Contains(text, required) { + t.Fatalf("%s should document current pipeline config %q", path, required) + } + } + for _, forbidden := range []string{ + "aws_secretsmanager:", + "inherits:", + "discovery:\n aws_organizations:", + "versioning:\n enabled: true\n s3_bucket:", + "secretsync versions", + "secretsync sync --version", + } { + if strings.Contains(text, forbidden) { + t.Fatalf("%s should not document stale config shape %q", path, forbidden) + } + } + } +} + +func TestPythonDocsUseExtendedDataCLIContract(t *testing.T) { + attributeStyleResult := regexp.MustCompile(`\bresult\.`) + paths := []string{"README.md", "docs/PYTHON_BINDINGS.md"} + forbidden := []string{ + "native_available", + "Native bindings", + "native bindings", + "CLI fallback", + "bindings aren't installed", + "is_valid, message", + "print(result[\"diff_output\"])", + } + + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + text := string(content) + + for _, phrase := range forbidden { + if strings.Contains(text, phrase) { + t.Fatalf("%s should not document stale Python integration surface %q", path, phrase) + } + } + if attributeStyleResult.MatchString(text) { + t.Fatalf("%s should use mapping-style result access, not result.attribute examples", path) + } + } + + content, err := os.ReadFile("docs/PYTHON_BINDINGS.md") + if err != nil { + t.Fatalf("read docs/PYTHON_BINDINGS.md: %v", err) + } + text := string(content) + for _, required := range []string{ + "connector.cli_available", + "validation = connector.validate_config", + "result[\"success\"]", + "result['secrets_added']", + "secretsync pipeline --output json", + } { + if !strings.Contains(text, required) { + t.Fatalf("docs/PYTHON_BINDINGS.md should document current Python contract %q", required) + } + } +} diff --git a/docs_security_test.go b/docs_security_test.go new file mode 100644 index 0000000..476c861 --- /dev/null +++ b/docs_security_test.go @@ -0,0 +1,100 @@ +package secretsync_test + +import ( + "os" + "strings" + "testing" +) + +func TestSecurityDocsDocumentLoggingContract(t *testing.T) { + required := []string{ + "raw secret values", + "raw Vault secret", + "raw AWS secret", + "raw client structures", + "machine-readable `secretsync pipeline --output json` result envelopes redact", + "GitHub Actions annotation output escapes workflow-command data", + } + + for _, path := range []string{"docs/SECURITY.md", "docs/OBSERVABILITY.md"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := strings.ToLower(strings.Join(strings.Fields(string(content)), " ")) + for _, phrase := range required { + if !strings.Contains(text, strings.ToLower(phrase)) { + t.Fatalf("%s must document logging contract phrase %q", path, phrase) + } + } + } +} + +func TestSecurityDocsUseProjectReportingContacts(t *testing.T) { + required := []string{ + "https://github.com/jbcom/secrets-sync/security/advisories", + "security@jbcom.dev", + } + + for _, path := range []string{"SECURITY.md", "docs/SECURITY.md"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := string(content) + if strings.Contains(text, "robert@lestak.sh") { + t.Fatalf("%s should not use the old fork-era security contact", path) + } + for _, phrase := range required { + if !strings.Contains(text, phrase) { + t.Fatalf("%s must document reporting contact %q", path, phrase) + } + } + } +} + +func TestPublicUsageDocsDoNotUseForkEraOwners(t *testing.T) { + for _, path := range []string{"docs/USAGE.md"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := string(content) + for _, forbidden := range []string{ + "robertlestak", + "vault-secret-sync", + } { + if strings.Contains(text, forbidden) { + t.Fatalf("%s should not use fork-era owner or package identifier %q", path, forbidden) + } + } + } +} + +func TestSecurityPolicyDocumentsCurrentMajorOnly(t *testing.T) { + content, err := os.ReadFile("SECURITY.md") + if err != nil { + t.Fatalf("read SECURITY.md: %v", err) + } + + text := string(content) + if !strings.Contains(text, "| 2.x") { + t.Fatalf("SECURITY.md should document current 2.x support") + } + for _, oldVersion := range []string{"| 1.2.x", "| 1.1.x", "| 1.0.x"} { + if strings.Contains(text, oldVersion) { + t.Fatalf("SECURITY.md should not advertise old support line %q", oldVersion) + } + } + for _, oldExample := range []string{"1.2.1", "1.2.2"} { + if strings.Contains(text, oldExample) { + t.Fatalf("SECURITY.md should not use unsupported 1.x patch example %q", oldExample) + } + } + if !strings.Contains(text, "2.0.1") || !strings.Contains(text, "2.0.2") { + t.Fatalf("SECURITY.md should use a supported 2.x patch-release example") + } +} diff --git a/docs_version_test.go b/docs_version_test.go new file mode 100644 index 0000000..2d86f05 --- /dev/null +++ b/docs_version_test.go @@ -0,0 +1,76 @@ +package secretsync_test + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDocsDoNotAdvertiseOldCurrentVersion(t *testing.T) { + forbiddenByPath := map[string][]string{ + "docs/ROADMAP.md": { + "Current Status: v1.2.0", + "Future Considerations (v2.0+)", + "### v1.3.0", + "### v1.4.0", + "### v1.5.0", + "coming in v1.3.0", + }, + "docs/FAQ.md": { + "SecretSync v1.2.0 is production-ready", + }, + } + + for path, forbidden := range forbiddenByPath { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := string(content) + for _, phrase := range forbidden { + if strings.Contains(text, phrase) { + t.Fatalf("%s should not advertise old current-version phrase %q", path, phrase) + } + } + } +} + +func TestPublicDocsDoNotAdvertiseOldFeatureReleaseLabels(t *testing.T) { + forbidden := []string{"v1.1.0", "v1.2.0"} + paths := []string{"README.md"} + + if err := filepath.WalkDir("docs", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".md" { + return nil + } + paths = append(paths, path) + return nil + }); err != nil { + t.Fatalf("walk docs: %v", err) + } + + var offenders []string + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := string(content) + for _, phrase := range forbidden { + if strings.Contains(text, phrase) { + offenders = append(offenders, path+": "+phrase) + } + } + } + + if len(offenders) > 0 { + t.Fatalf("public docs should not advertise old feature-release labels:\n%s", strings.Join(offenders, "\n")) + } +} diff --git a/examples/config-full.yaml b/examples/config-full.yaml index d954743..ad25b23 100644 --- a/examples/config-full.yaml +++ b/examples/config-full.yaml @@ -1,92 +1,103 @@ +# Full SecretSync pipeline configuration example. + log: - level: "debug" # The log level for the application. Can be one of "debug", "info", "warn", "error", "fatal", or "panic". - format: "json" # The format of the log output. Can be one of "json" or "text" - events: true # Whether to log events. + level: info + format: json -# Configuration for the event server. -events: - # Whether the event server is enabled. - enabled: true - # The port the event server listens on. - port: 8080 - # Security settings for the event server. - security: - # Whether security is enabled for the event server. - enabled: true - # The token used for authentication. - token: "your-token" - # TLS configuration for the event server. - tls: - certFile: "/path/to/certfile" - keyFile: "/path/to/keyfile" - # Whether to deduplicate events. - dedupe: true +vault: + address: https://vault.example.com/ + namespace: platform/secrets + auth: + approle: + mount: approle + role_id: ${VAULT_ROLE_ID} + secret_id: ${VAULT_SECRET_ID} -# Configuration for the operator. -operator: - # Whether the operator is enabled. - enabled: true - workerPoolSize: 10 - # The number of subscriptions to use. - numSubscriptions: 10 +aws: + region: us-east-1 + execution_context: + type: delegated_admin + account_id: "123456789012" + delegation: + services: + - organizations.amazonaws.com + - sso.amazonaws.com + control_tower: + enabled: true + execution_role: + name: AWSControlTowerExecution + organizations: + auto_discover: true + identity_center: + enabled: true + auto_discover: true -# Configuration for the stores. -stores: - aws: - region: "us-west-2" +sources: + shared: + vault: + mount: shared + paths: + - app/* + - platform/* + analytics: + vault: + mount: analytics + paths: + - services/* + production-overrides: + vault: + mount: production - github: - installId: 12345 - appId: 67890 - privateKeyPath: "/path/to/private/key" +merge_store: + s3: + bucket: secretsync-merge-store + prefix: merged/ + kms_key_id: alias/secretsync-merge-store + versioning: + enabled: true + retain_versions: 20 -# Configuration for the queue. -queue: - # The type of queue to use. - type: "your-queue-type" - # Parameters for the queue. - params: - param1: "value1" - param2: "value2" +targets: + staging: + account_id: "111111111111" + region: us-east-1 + secret_prefix: /staging/ + imports: + - shared + - analytics + production: + account_id: "222222222222" + region: us-east-1 + secret_prefix: /production/ + imports: + - staging + - production-overrides -# Configuration for the metrics server. -metrics: - # The port the metrics server listens on. - port: 9090 - # Security settings for the metrics server. - security: - # Whether security is enabled for the metrics server. - enabled: true - # The token used for authentication. - token: "your-token" - # TLS configuration for the metrics server. - tls: - certFile: "/path/to/certfile" - keyFile: "/path/to/keyfile" +dynamic_targets: + analytics_sandboxes: + discovery: + organizations: + ous: + - ou-xxxx-sandbox + recursive: true + tag_filters: + - key: Team + values: + - analytics + operator: equals + exclude_statuses: + - SUSPENDED + imports: + - shared + - analytics + secret_prefix: /sandbox/ + region: us-east-1 -notifications: - email: - enabled: true - host: "smtp.example.com" - port: 587 - username: "your-email@example.com" - password: "your-email-password" - from: "your-email@example.com" - to: "recipient@example.com" - subject: "Notification Subject" - body: "This is the notification body." - slack: - enabled: true - url: "https://hooks.slack.example.com/services/xxx/xxx/xxx" - message: "This is the notification message." - webhook: - enabled: true - url: "https://example.com/webhook" - method: "POST" - headers: - Content-Type: "application/json" - body: | - { - "status": "{{ .Status }}", - "message": "{{ .Message }}" - } +pipeline: + merge: + parallel: 4 + sync: + parallel: 4 + delete_orphans: false + dry_run: false + continue_on_error: true diff --git a/examples/github-action-workflow.yml b/examples/github-action-workflow.yml index 5f3d841..89f780e 100644 --- a/examples/github-action-workflow.yml +++ b/examples/github-action-workflow.yml @@ -3,8 +3,8 @@ name: Sync Secrets with SecretSync # This is a complete example workflow showing how to use SecretSync # as a GitHub Action for automated secrets synchronization. # -# Pin to a package release tag for reproducible runs. -# This example uses the latest current package tag. +# Pin third-party actions to exact commits and pin SecretSync to a released +# component tag for reproducible runs. on: # Scheduled sync every 6 hours @@ -58,16 +58,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 - name: Validate Configuration (Dry Run) - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: config.yaml dry-run: 'true' @@ -87,7 +87,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Determine Configuration id: config @@ -99,7 +99,7 @@ jobs: fi - name: Configure AWS Credentials (OIDC) - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 @@ -107,7 +107,7 @@ jobs: role-session-name: GitHubActions-SecretSync-${{ github.run_id }} - name: Run SecretSync - uses: jbcom/secrets-sync@secretssync-v2.0.2 + uses: jbcom/secrets-sync@secrets-sync-vX.Y.Z with: config: ${{ steps.config.outputs.config }} targets: ${{ github.event.inputs.targets || '' }} diff --git a/examples/organizations-discovery.yaml b/examples/organizations-discovery.yaml index 34ec03a..b0435c1 100644 --- a/examples/organizations-discovery.yaml +++ b/examples/organizations-discovery.yaml @@ -11,7 +11,8 @@ dynamic_targets: development_accounts: discovery: organizations: - ou: "ou-xxxx-development" + ous: + - "ou-xxxx-development" imports: - shared-dev-secrets exclude: @@ -24,7 +25,8 @@ dynamic_targets: workload_accounts: discovery: organizations: - ou: "ou-xxxx-workloads" + ous: + - "ou-xxxx-workloads" recursive: true # Includes all nested OUs imports: - platform-secrets @@ -39,8 +41,10 @@ dynamic_targets: discovery: organizations: tags: - Environment: production - ManagedBy: platform-team + Environment: + - production + ManagedBy: + - platform-team imports: - prod-secrets - compliance-keys @@ -54,10 +58,13 @@ dynamic_targets: production_web_apps: discovery: organizations: - ou: "ou-xxxx-production" + ous: + - "ou-xxxx-production" tags: - Workload: web-application - CostCenter: engineering + Workload: + - web-application + CostCenter: + - engineering imports: - web-app-secrets - ssl-certificates @@ -69,11 +76,14 @@ dynamic_targets: sandbox_environments: discovery: organizations: - ou: "ou-xxxx-sandboxes" + ous: + - "ou-xxxx-sandboxes" recursive: true # Search all child OUs too tags: - AutoCleanup: enabled - Owner: analytics-team + AutoCleanup: + - enabled + Owner: + - analytics-team imports: - sandbox-defaults - testing-credentials @@ -87,7 +97,8 @@ dynamic_targets: custom_role_accounts: discovery: organizations: - ou: "ou-xxxx-special" + ous: + - "ou-xxxx-special" imports: - special-secrets # {{.AccountID}} will be replaced with each discovered account ID @@ -108,8 +119,10 @@ dynamic_targets: # Second: also get accounts from Organizations by tags organizations: tags: - Platform: analytics - Environment: shared + Platform: + - analytics + Environment: + - shared # Third: also get accounts from an SSM parameter accounts_list: source: "ssm:/platform/analytics-accounts" @@ -127,10 +140,12 @@ dynamic_targets: dev_envs: discovery: organizations: - ou: "ou-xxxx-dev" + ous: + - "ou-xxxx-dev" recursive: true tags: - AutoProvision: enabled + AutoProvision: + - enabled imports: - dev-shared-config - testing-data @@ -140,9 +155,11 @@ dynamic_targets: staging_envs: discovery: organizations: - ou: "ou-xxxx-staging" + ous: + - "ou-xxxx-staging" tags: - Environment: staging + Environment: + - staging imports: - dev-shared-config # Inherit from dev - staging-specific # Add staging-specific secrets @@ -152,10 +169,13 @@ dynamic_targets: prod_envs: discovery: organizations: - ou: "ou-xxxx-production" + ous: + - "ou-xxxx-production" tags: - Environment: production - Compliance: required + Environment: + - production + Compliance: + - required imports: - prod-secrets - pci-compliance-keys diff --git a/examples/pipeline-config.yaml b/examples/pipeline-config.yaml index 3c93948..016514d 100644 --- a/examples/pipeline-config.yaml +++ b/examples/pipeline-config.yaml @@ -117,11 +117,11 @@ sources: # mount: secrets # Import existing secrets from AWS account - # legacy-secrets: + # imported-secrets: # aws: # account_id: "999999999999" # region: us-east-1 - # prefix: "/legacy/" + # prefix: "/imported/" # ============================================================================= # Merge Store @@ -209,7 +209,8 @@ dynamic_targets: # dev_environments: # discovery: # organizations: - # ou: "ou-xxxx-development" + # ous: + # - "ou-xxxx-development" # # Or by tags: # # tags: # # Environment: development diff --git a/examples/vault-to-aws-secrets.yaml b/examples/vault-to-aws-secrets.yaml index abf1ae1..844a288 100644 --- a/examples/vault-to-aws-secrets.yaml +++ b/examples/vault-to-aws-secrets.yaml @@ -1,15 +1,34 @@ ---- -apiVersion: secretsync.extendeddata.dev/v1alpha1 -kind: SecretSync -metadata: - name: vault-to-aws-secrets - namespace: default -spec: - source: - address: "https://vault.example.com" - path: "hello/world" - namespace: "foo/bar" - dest: - - aws: - name: "example-secret" - region: "us-west-2" \ No newline at end of file +# Minimal Vault-to-AWS Secrets Manager pipeline. + +vault: + address: https://vault.example.com/ + namespace: platform/secrets + auth: + token: + token: ${VAULT_TOKEN} + +aws: + region: us-west-2 + +sources: + app-secrets: + vault: + mount: secret + paths: + - hello/world + +merge_store: + vault: + mount: merged-secrets + +targets: + example-account: + account_id: "111111111111" + region: us-west-2 + secret_prefix: /example/ + imports: + - app-secrets + +pipeline: + dry_run: false + continue_on_error: false diff --git a/examples/vault-to-github.yaml b/examples/vault-to-github.yaml deleted file mode 100644 index 6fce8e3..0000000 --- a/examples/vault-to-github.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: secretsync.extendeddata.dev/v1alpha1 -kind: SecretSync -metadata: - name: vault-to-github - namespace: default -spec: - source: - address: "https://vault.example.com" - path: "hello/world" - namespace: "foo/bar" - dest: - - github: - repo: "foobar" - owner: "helloworld" - - github: - repo: "barfoo" - owner: "helloworld" \ No newline at end of file diff --git a/examples/vault-to-vault.yaml b/examples/vault-to-vault.yaml deleted file mode 100644 index d9d0768..0000000 --- a/examples/vault-to-vault.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: secretsync.extendeddata.dev/v1alpha1 -kind: SecretSync -metadata: - name: vault-to-vault - namespace: default -spec: - source: - address: "https://vault.example.com" - path: "hello/world/(.*)" - namespace: "foo/bar" - dest: - - vault: - address: "https://vault2.example.com" - path: "rewritten/path/$1" - namespace: "robertlestak/example" \ No newline at end of file diff --git a/examples_config_test.go b/examples_config_test.go new file mode 100644 index 0000000..20b00d9 --- /dev/null +++ b/examples_config_test.go @@ -0,0 +1,81 @@ +package secretsync_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jbcom/secrets-sync/pkg/pipeline" +) + +func TestExamplePipelineConfigsLoadAndValidate(t *testing.T) { + paths, err := filepath.Glob("examples/*.yaml") + if err != nil { + t.Fatalf("glob examples: %v", err) + } + if len(paths) == 0 { + t.Fatal("expected pipeline config examples") + } + + for _, path := range paths { + t.Run(path, func(t *testing.T) { + cfg, err := pipeline.LoadConfigWithoutAutoDetect(path) + if err != nil { + t.Fatalf("load %s: %v", path, err) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("validate %s: %v", path, err) + } + }) + } +} + +func TestPublicDocsAndExamplesDoNotAdvertiseRemovedAPISurfaces(t *testing.T) { + forbidden := []string{ + "apiVersion: secretsync.extendeddata.dev", + "kind: SecretSync", + "aws_secretsmanager:", + "inherits:", + "Kubernetes operator with CRD support", + "secretsync-events", + "secretsync-operator", + "Event Server", + "Sync Operator", + "memory queue", + "microservices mode", + "Vault | GCP Secret Manager | ✅ Supported", + "Vault | GitHub Secrets | ✅ Supported", + } + + for _, root := range []string{"README.md", "docs", "examples"} { + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + switch filepath.Ext(path) { + case ".md", ".yaml", ".yml": + default: + return nil + } + + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + text := string(content) + for _, phrase := range forbidden { + if strings.Contains(text, phrase) { + t.Fatalf("%s should not advertise removed API surface %q", path, phrase) + } + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } +} diff --git a/go.mod b/go.mod index 423b061..d8bc1bf 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/jbcom/secrets-sync -go 1.25.3 +go 1.26.4 require ( github.com/aws/aws-sdk-go-v2 v1.41.6 @@ -80,9 +80,9 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/protobuf v1.36.9 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 59e90a2..24ee62d 100644 --- a/go.sum +++ b/go.sum @@ -178,12 +178,12 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= diff --git a/helm_chart_test.go b/helm_chart_test.go new file mode 100644 index 0000000..e34072f --- /dev/null +++ b/helm_chart_test.go @@ -0,0 +1,138 @@ +package secretsync_test + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestHelmChartUsesSecretSyncAPI(t *testing.T) { + paths := []string{ + "deploy/charts/secretsync/Chart.yaml", + "deploy/charts/secretsync/values.yaml", + "docs/USAGE.md", + } + + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + text := string(content) + for _, forbidden := range []string{ + "vaultsecretsync.lestak.sh", + "VaultSecretSync", + "vaultsecretsync", + " vss", + "- vss", + } { + if strings.Contains(text, forbidden) { + t.Fatalf("%s should not preserve old VaultSecretSync API surface %q", path, forbidden) + } + } + } +} + +func TestHelmChartUsesPipelineRunner(t *testing.T) { + files := map[string]string{ + "chart": readTestFile(t, "deploy/charts/secretsync/Chart.yaml"), + "values": readTestFile(t, "deploy/charts/secretsync/values.yaml"), + "configmap": readTestFile(t, "deploy/charts/secretsync/templates/configmap.yaml"), + "cronjob": readTestFile(t, "deploy/charts/secretsync/templates/cronjob.yaml"), + } + + for _, forbidden := range []string{ + "dependencies:", + "secretsync-events", + "secretsync-operator", + "Legacy config format", + "Kubernetes operator", + "-operator", + "-events", + } { + for name, text := range files { + if strings.Contains(text, forbidden) { + t.Fatalf("helm %s should not contain removed surface %q", name, forbidden) + } + } + } + + required := map[string][]string{ + "values": { + "pipeline:", + "enabled: false", + "schedule: \"\"", + "config: {}", + "continueOnError: true", + }, + "configmap": { + ".Values.pipeline.config", + ".Values.pipeline.existingConfigMap", + }, + "cronjob": { + "kind: CronJob", + "- pipeline", + "- --config", + "/config/config.yaml", + "--dry-run={{ .Values.pipeline.dryRun }}", + "--continue-on-error={{ .Values.pipeline.continueOnError }}", + }, + } + + for name, needles := range required { + for _, needle := range needles { + if !strings.Contains(files[name], needle) { + t.Fatalf("helm %s missing %q", name, needle) + } + } + } +} + +func TestHelmChartDoesNotShipDeadSubcharts(t *testing.T) { + for _, path := range []string{ + "deploy/charts/secretsync/charts/secretsync-events", + "deploy/charts/secretsync/charts/secretsync-operator", + } { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("%s should not exist after removing the unsupported operator/events runtimes", path) + } + } +} + +func TestHelmTemplatesDoNotUseRemovedCLIFlags(t *testing.T) { + err := filepath.WalkDir("deploy/charts/secretsync", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + switch filepath.Ext(path) { + case ".yaml", ".yml", ".tpl": + default: + return nil + } + text := readTestFile(t, path) + for _, forbidden := range []string{"-operator", "-events"} { + if strings.Contains(text, forbidden) { + t.Fatalf("%s should not use removed CLI flag %q", path, forbidden) + } + } + return nil + }) + if err != nil { + t.Fatalf("walk helm chart: %v", err) + } +} + +func readTestFile(t *testing.T, path string) string { + t.Helper() + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(content) +} diff --git a/pkg/client/aws/aws.go b/pkg/client/aws/aws.go index e8d214e..7953452 100644 --- a/pkg/client/aws/aws.go +++ b/pkg/client/aws/aws.go @@ -159,7 +159,13 @@ func NewClient(cfg *AwsClient) (*AwsClient, error) { breakerName := fmt.Sprintf("aws-secretsmanager-%s-%s", vc.Name, vc.Region) vc.breaker = circuitbreaker.New(circuitbreaker.DefaultConfig(breakerName)) - l.Debugf("client=%+v", vc) + l.WithFields(log.Fields{ + "name": vc.Name, + "region": vc.Region, + "roleArn": vc.RoleArn, + "cacheTTL": vc.CacheTTL, + "hasKmsKey": vc.EncryptionKey != "", + }).Debug("client initialized") l.Trace("end") return vc, nil } diff --git a/pkg/client/aws/aws_test.go b/pkg/client/aws/aws_test.go index 274e835..e2024b6 100644 --- a/pkg/client/aws/aws_test.go +++ b/pkg/client/aws/aws_test.go @@ -110,6 +110,13 @@ func TestAwsClient_Validate(t *testing.T) { } } +func TestAwsClient_DoesNotLogRawClientStruct(t *testing.T) { + content, err := os.ReadFile("aws.go") + require.NoError(t, err) + + assert.NotContains(t, string(content), `Debugf("client=%+v"`) +} + func TestAwsClient_Driver(t *testing.T) { client := &AwsClient{} assert.Equal(t, driver.DriverNameAws, client.Driver()) diff --git a/pkg/client/vault/vault.go b/pkg/client/vault/vault.go index d307cac..400d51a 100644 --- a/pkg/client/vault/vault.go +++ b/pkg/client/vault/vault.go @@ -124,7 +124,13 @@ func NewClient(cfg *VaultClient) (*VaultClient, error) { breakerName := fmt.Sprintf("vault-%s", vc.Address) vc.breaker = circuitbreaker.New(circuitbreaker.DefaultConfig(breakerName)) - l.Tracef("client=%+v", vc) + l.WithFields(log.Fields{ + "address": vc.Address, + "path": vc.Path, + "authMethod": vc.AuthMethod, + "namespace": vc.Namespace, + "merge": vc.Merge, + }).Trace("client initialized") l.Trace("end") return vc, nil } @@ -279,21 +285,14 @@ func (vc *VaultClient) GetKVSecretOnce(ctx context.Context, s string) (map[strin observability.RecordDuration(observability.VaultAPICallDuration, startTime, "get_secret", status) }() - l := log.WithFields(log.Fields{ - "address": vc.Address, - "role": vc.Role, - "path": s, - "method": vc.AuthMethod, - }) - var secrets map[string]interface{} if s == "" { observability.RecordError(observability.VaultErrors, "get_secret", "invalid_path") - return secrets, errors.New("secret path required") + return nil, errors.New("secret path required") } ss := strings.Split(s, "/") if len(ss) < 2 { observability.RecordError(observability.VaultErrors, "get_secret", "invalid_path") - return secrets, errors.New("secret path must be in kv/path/to/secret format") + return nil, errors.New("secret path must be in kv/path/to/secret format") } ss = insertSliceString(ss, 1, "data") //log.Debugf("headers_sent=%+v", vc.Client.Headers()) @@ -301,7 +300,7 @@ func (vc *VaultClient) GetKVSecretOnce(ctx context.Context, s string) (map[strin s = strings.Join(ss, "/") if c == nil { observability.RecordError(observability.VaultErrors, "get_secret", "not_initialized") - return secrets, errors.New("vault client not initialized") + return nil, errors.New("vault client not initialized") } // Ensure circuit breaker is initialized @@ -313,7 +312,7 @@ func (vc *VaultClient) GetKVSecretOnce(ctx context.Context, s string) (map[strin }) if err != nil { observability.RecordError(observability.VaultErrors, "get_secret", "api_error") - return secrets, circuitbreaker.WrapError(err, vc.breaker.Name(), vc.breaker.State()) + return nil, circuitbreaker.WrapError(err, vc.breaker.Name(), vc.breaker.State()) } secret := result @@ -321,7 +320,6 @@ func (vc *VaultClient) GetKVSecretOnce(ctx context.Context, s string) (map[strin observability.RecordError(observability.VaultErrors, "get_secret", "not_found") return nil, errors.New("secret not found: " + s) } - l.Tracef("secret=%+v", secret) if secret.Data["data"] == nil { observability.RecordError(observability.VaultErrors, "get_secret", "no_data") return nil, errors.New("secret data not found: " + s) @@ -386,7 +384,6 @@ func (vc *VaultClient) WriteSecret(ctx context.Context, meta metav1.ObjectMeta, if err != nil { return nil, err } - var secrets map[string]interface{} if vc.Merge { sec, getErr := vc.GetSecret(ctx, s) if getErr != nil { @@ -419,18 +416,17 @@ func (vc *VaultClient) WriteSecret(ctx context.Context, meta metav1.ObjectMeta, if terr != nil { return nil, terr } - secrets, err = vc.WriteSecretWithLatestCAS(ctx, s, data) + _, err = vc.WriteSecretWithLatestCAS(ctx, s, data) if err != nil { terr := vc.NewToken(ctx) if terr != nil { return nil, terr } - secrets, err = vc.WriteSecretWithLatestCAS(ctx, s, data) + _, err = vc.WriteSecretWithLatestCAS(ctx, s, data) if err != nil { return nil, err } } - l.Tracef("secrets=%+v", secrets) return nil, err } diff --git a/pkg/client/vault/vault_test.go b/pkg/client/vault/vault_test.go index e169cab..794e2ef 100644 --- a/pkg/client/vault/vault_test.go +++ b/pkg/client/vault/vault_test.go @@ -3,6 +3,7 @@ package vault import ( "context" "encoding/json" + "os" "strings" "testing" @@ -118,6 +119,21 @@ func TestVaultClient_Driver(t *testing.T) { assert.Equal(t, driver.DriverNameVault, client.Driver()) } +func TestVaultClient_DoesNotLogRawSecretPayloads(t *testing.T) { + content, err := os.ReadFile("vault.go") + require.NoError(t, err) + + source := string(content) + forbidden := []string{ + `Tracef("secret=%+v"`, + `Tracef("secrets=%+v"`, + `Tracef("client=%+v"`, + } + for _, pattern := range forbidden { + assert.NotContains(t, source, pattern) + } +} + func TestVaultClient_GetPath(t *testing.T) { client := &VaultClient{Path: "secret/data/test"} assert.Equal(t, "secret/data/test", client.GetPath()) diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index e59bf9d..1137b9a 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -387,14 +387,6 @@ func formatHuman(diff *PipelineDiff) string { func formatGitHub(diff *PipelineDiff) string { var sb strings.Builder - // Summary as workflow output - sb.WriteString(fmt.Sprintf("::set-output name=changes::%d\n", diff.Summary.Added+diff.Summary.Removed+diff.Summary.Modified)) - sb.WriteString(fmt.Sprintf("::set-output name=added::%d\n", diff.Summary.Added)) - sb.WriteString(fmt.Sprintf("::set-output name=removed::%d\n", diff.Summary.Removed)) - sb.WriteString(fmt.Sprintf("::set-output name=modified::%d\n", diff.Summary.Modified)) - sb.WriteString(fmt.Sprintf("::set-output name=unchanged::%d\n", diff.Summary.Unchanged)) - sb.WriteString(fmt.Sprintf("::set-output name=zero_sum::%t\n", diff.IsZeroSum())) - if diff.IsZeroSum() { sb.WriteString("::notice::✅ Zero-sum: No changes detected\n") } else { @@ -409,17 +401,18 @@ func formatGitHub(diff *PipelineDiff) string { continue } - sb.WriteString(fmt.Sprintf("::group::Target: %s (%d changes)\n", td.Target, + sb.WriteString(fmt.Sprintf("::group::Target: %s (%d changes)\n", escapeGitHubCommandData(td.Target), td.Summary.Added+td.Summary.Removed+td.Summary.Modified)) for _, c := range td.Changes { + safePath := escapeGitHubCommandData(c.Path) switch c.ChangeType { case ChangeTypeAdded: - sb.WriteString(fmt.Sprintf("::notice::+ %s (new secret)\n", c.Path)) + sb.WriteString(fmt.Sprintf("::notice::+ %s (new secret)\n", safePath)) case ChangeTypeRemoved: - sb.WriteString(fmt.Sprintf("::warning::- %s (removed)\n", c.Path)) + sb.WriteString(fmt.Sprintf("::warning::- %s (removed)\n", safePath)) case ChangeTypeModified: - sb.WriteString(fmt.Sprintf("::notice::~ %s (modified)\n", c.Path)) + sb.WriteString(fmt.Sprintf("::notice::~ %s (modified)\n", safePath)) } } @@ -429,6 +422,15 @@ func formatGitHub(diff *PipelineDiff) string { return sb.String() } +func escapeGitHubCommandData(value string) string { + replacer := strings.NewReplacer( + "%", "%25", + "\r", "%0D", + "\n", "%0A", + ) + return replacer.Replace(value) +} + func formatCompact(diff *PipelineDiff) string { if diff.IsZeroSum() { return fmt.Sprintf("ZERO-SUM: %d secrets unchanged", diff.Summary.Unchanged) diff --git a/pkg/diff/diff_test.go b/pkg/diff/diff_test.go index 66fcf8f..1dbd952 100644 --- a/pkg/diff/diff_test.go +++ b/pkg/diff/diff_test.go @@ -135,14 +135,14 @@ func TestDiffSecrets_ComplexScenario(t *testing.T) { "api-keys/stripe": map[string]interface{}{"KEY": "sk_xxx"}, "api-keys/datadog": map[string]interface{}{"API_KEY": "dd_xxx"}, "database/postgres": map[string]interface{}{"HOST": "old.db.com", "PASSWORD": "old"}, - "legacy/config": map[string]interface{}{"OLD": "value"}, + "retired/config": map[string]interface{}{"OLD": "value"}, } desired := map[string]interface{}{ "api-keys/stripe": map[string]interface{}{"KEY": "sk_xxx"}, // unchanged "api-keys/datadog": map[string]interface{}{"API_KEY": "dd_yyy"}, // modified "database/postgres": map[string]interface{}{"HOST": "new.db.com", "PASSWORD": "new"}, // modified "api-keys/newrelic": map[string]interface{}{"KEY": "nr_xxx"}, // added - // legacy/config removed + // retired/config removed } changes := DiffSecrets(current, desired) @@ -254,17 +254,46 @@ func TestFormatDiff_GitHub(t *testing.T) { output := FormatDiff(diff, OutputFormatGitHub) - if !strings.Contains(output, "::set-output name=changes::3") { - t.Error("expected changes output") - } - if !strings.Contains(output, "::set-output name=zero_sum::false") { - t.Error("expected zero_sum output") + if strings.Contains(output, "::set-output") { + t.Error("GitHub format should not use deprecated set-output commands") } if !strings.Contains(output, "::warning::") { t.Error("expected warning annotation") } } +func TestFormatDiff_GitHubEscapesCommandData(t *testing.T) { + diff := &PipelineDiff{ + Targets: []TargetDiff{ + { + Target: "prod%\n::warning::injected", + Changes: []SecretChange{ + { + Path: "app/secret%\r\n::error::leak", + ChangeType: ChangeTypeModified, + }, + }, + Summary: ChangeSummary{Modified: 1, Total: 1}, + }, + }, + Summary: ChangeSummary{Modified: 1, Total: 1}, + } + + output := FormatDiff(diff, OutputFormatGitHub) + + if strings.Contains(output, "prod%\n") || strings.Contains(output, "app/secret%\r\n") { + t.Fatalf("GitHub command data was not escaped:\n%s", output) + } + if strings.Contains(output, "\n::warning::injected") || strings.Contains(output, "\n::error::leak") { + t.Fatalf("GitHub command data allowed injected workflow command:\n%s", output) + } + for _, expected := range []string{"prod%25%0A::warning::injected", "app/secret%25%0D%0A::error::leak"} { + if !strings.Contains(output, expected) { + t.Fatalf("GitHub output missing escaped value %q:\n%s", expected, output) + } + } +} + func TestFormatDiff_GitHubZeroSum(t *testing.T) { diff := &PipelineDiff{ Summary: ChangeSummary{Unchanged: 5, Total: 5}, @@ -272,8 +301,8 @@ func TestFormatDiff_GitHubZeroSum(t *testing.T) { output := FormatDiff(diff, OutputFormatGitHub) - if !strings.Contains(output, "::set-output name=zero_sum::true") { - t.Error("expected zero_sum=true output") + if strings.Contains(output, "::set-output") { + t.Error("GitHub format should not use deprecated set-output commands") } if !strings.Contains(output, "::notice::") { t.Error("expected notice annotation for zero-sum") diff --git a/pkg/diff/versioning_test.go b/pkg/diff/versioning_test.go index ed6cc29..c93f5fd 100644 --- a/pkg/diff/versioning_test.go +++ b/pkg/diff/versioning_test.go @@ -153,8 +153,8 @@ func TestFormatDiffWithVersions(t *testing.T) { assert.Contains(t, output, "(was v5)", "Should show version for removed secret") } -// TestBackwardCompatibility tests that diff works without version information (v1.2.0 - Requirement 24) -func TestBackwardCompatibility(t *testing.T) { +// TestDiffWithoutVersionMetadata tests the current no-version diff path. +func TestDiffWithoutVersionMetadata(t *testing.T) { current := map[string]interface{}{ "app/api-key": map[string]interface{}{"key": "old-value"}, } @@ -162,7 +162,7 @@ func TestBackwardCompatibility(t *testing.T) { "app/api-key": map[string]interface{}{"key": "new-value"}, } - // Test without version information (backward compatibility) + // Test without version information. changes := DiffSecrets(current, desired) assert.Len(t, changes, 1) assert.Equal(t, ChangeTypeModified, changes[0].ChangeType) diff --git a/pkg/pipeline/aws_context_test.go b/pkg/pipeline/aws_context_test.go index 88b7885..67e4950 100644 --- a/pkg/pipeline/aws_context_test.go +++ b/pkg/pipeline/aws_context_test.go @@ -141,13 +141,13 @@ func TestOrganizationsDiscoveryTagFiltering(t *testing.T) { // TestDiscoveryWithTagsAndOU tests discovery configuration with both OU and tags func TestDiscoveryWithTagsAndOU(t *testing.T) { cfg := &OrganizationsDiscovery{ - OU: "ou-abc-12345678", + OUs: []string{"ou-abc-12345678"}, Tags: map[string][]string{"Environment": {"production"}}, Recursive: true, } // Verify configuration is valid - assert.Equal(t, "ou-abc-12345678", cfg.OU) + assert.Equal(t, []string{"ou-abc-12345678"}, cfg.OUs) assert.True(t, cfg.Recursive) assert.NotNil(t, cfg.Tags) assert.ElementsMatch(t, []string{"production"}, cfg.Tags["Environment"]) @@ -161,7 +161,7 @@ func TestDiscoveryWithTagsOnly(t *testing.T) { // When no OU is specified, all accounts should be listed and filtered by tags // Now supports multiple values per tag for OR matching - assert.Equal(t, "", cfg.OU) + assert.Empty(t, cfg.OUs) assert.False(t, cfg.Recursive) assert.NotNil(t, cfg.Tags) assert.ElementsMatch(t, []string{"sandbox", "development"}, cfg.Tags["Environment"]) diff --git a/pkg/pipeline/config_test.go b/pkg/pipeline/config_test.go index 864d322..7aaf4b9 100644 --- a/pkg/pipeline/config_test.go +++ b/pkg/pipeline/config_test.go @@ -249,7 +249,7 @@ func TestConfigValidate(t *testing.T) { "sandboxes": { Discovery: DiscoveryConfig{ Organizations: &OrganizationsDiscovery{ - OU: "ou-xxxx-sandboxes", + OUs: []string{"ou-xxxx-sandboxes"}, Recursive: true, }, }, diff --git a/pkg/pipeline/discovery_organizations.go b/pkg/pipeline/discovery_organizations.go index a3cac34..2980788 100644 --- a/pkg/pipeline/discovery_organizations.go +++ b/pkg/pipeline/discovery_organizations.go @@ -10,7 +10,6 @@ import ( func (d *DiscoveryService) discoverFromOrganizations(cfg *OrganizationsDiscovery) ([]AccountInfo, error) { l := log.WithFields(log.Fields{ "action": "discoverFromOrganizations", - "ou": cfg.OU, "ous": cfg.OUs, "recursive": cfg.Recursive, }) @@ -22,12 +21,7 @@ func (d *DiscoveryService) discoverFromOrganizations(cfg *OrganizationsDiscovery var accounts []AccountInfo - // Collect all OUs to process (legacy single OU + new multiple OUs) - var ousToProcess []string - if cfg.OU != "" { - ousToProcess = append(ousToProcess, cfg.OU) - } - ousToProcess = append(ousToProcess, cfg.OUs...) + ousToProcess := cfg.OUs // Discover by OUs if len(ousToProcess) > 0 { @@ -59,12 +53,12 @@ func (d *DiscoveryService) discoverFromOrganizations(cfg *OrganizationsDiscovery accounts = append(accounts, allAccounts...) } - // Filter by tags if specified (legacy format) + // Filter by simple tag map if specified if len(cfg.Tags) > 0 { accounts = filterAccountsByTags(accounts, cfg.Tags) } - // Filter by enhanced tag filters if specified (v1.2.0) + // Filter by tag predicates if specified if len(cfg.TagFilters) > 0 { combination := cfg.TagCombination if combination == "" { diff --git a/pkg/pipeline/discovery_ou_test.go b/pkg/pipeline/discovery_ou_test.go index 14210a8..3a9814c 100644 --- a/pkg/pipeline/discovery_ou_test.go +++ b/pkg/pipeline/discovery_ou_test.go @@ -1,20 +1,21 @@ package pipeline import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestOrganizationsDiscovery_MultipleOUsConfig(t *testing.T) { - t.Run("legacy single OU", func(t *testing.T) { + t.Run("single OU uses OUs list", func(t *testing.T) { cfg := &OrganizationsDiscovery{ - OU: "ou-prod-123", + OUs: []string{"ou-prod-123"}, } - // Test that the configuration is properly structured - assert.Equal(t, "ou-prod-123", cfg.OU) - assert.Empty(t, cfg.OUs) + assert.Equal(t, []string{"ou-prod-123"}, cfg.OUs) }) t.Run("multiple OUs", func(t *testing.T) { @@ -22,21 +23,18 @@ func TestOrganizationsDiscovery_MultipleOUsConfig(t *testing.T) { OUs: []string{"ou-prod-123", "ou-staging-456", "ou-dev-789"}, } - assert.Empty(t, cfg.OU) assert.Len(t, cfg.OUs, 3) assert.Contains(t, cfg.OUs, "ou-prod-123") assert.Contains(t, cfg.OUs, "ou-staging-456") assert.Contains(t, cfg.OUs, "ou-dev-789") }) - t.Run("legacy OU + multiple OUs", func(t *testing.T) { - cfg := &OrganizationsDiscovery{ - OU: "ou-prod-123", - OUs: []string{"ou-staging-456", "ou-dev-789"}, - } + t.Run("removed single OU yaml is rejected", func(t *testing.T) { + cfg := &OrganizationsDiscovery{} + + err := yaml.Unmarshal([]byte("ou: ou-prod-123\n"), cfg) - assert.Equal(t, "ou-prod-123", cfg.OU) - assert.Len(t, cfg.OUs, 2) + assert.ErrorContains(t, err, "organizations.ou has been removed") }) t.Run("OU caching enabled", func(t *testing.T) { @@ -48,6 +46,24 @@ func TestOrganizationsDiscovery_MultipleOUsConfig(t *testing.T) { }) } +func TestRepositoryExamplesRejectRemovedOrganizationsOUShape(t *testing.T) { + paths := []string{ + filepath.Join("..", "..", "examples", "pipeline-config.yaml"), + filepath.Join("..", "..", "tests", "integration", "fixtures", "pipeline-config.yaml"), + } + + for _, path := range paths { + t.Run(path, func(t *testing.T) { + data, err := os.ReadFile(path) + assert.NoError(t, err) + + var doc yaml.Node + assert.NoError(t, yaml.Unmarshal(data, &doc)) + assert.False(t, hasRemovedOrganizationsOUShape(&doc), "%s uses removed organizations.ou", path) + }) + } +} + func TestDiscoveryService_CacheInitialization(t *testing.T) { discovery := &DiscoveryService{ ouCache: make(map[string][]AccountInfo), @@ -72,3 +88,43 @@ func TestDiscoveryService_CacheInitialization(t *testing.T) { assert.Len(t, cached, 1) assert.Equal(t, "111111111111", cached[0].ID) } + +func hasRemovedOrganizationsOUShape(node *yaml.Node) bool { + if node == nil { + return false + } + + if node.Kind == yaml.MappingNode { + for i := 0; i < len(node.Content)-1; i += 2 { + key := node.Content[i] + value := node.Content[i+1] + if key.Value == "organizations" && mappingNodeHasKey(value, "ou") { + return true + } + if hasRemovedOrganizationsOUShape(value) { + return true + } + } + return false + } + + for _, child := range node.Content { + if hasRemovedOrganizationsOUShape(child) { + return true + } + } + return false +} + +func mappingNodeHasKey(node *yaml.Node, keyName string) bool { + if node == nil || node.Kind != yaml.MappingNode { + return false + } + + for i := 0; i < len(node.Content)-1; i += 2 { + if node.Content[i].Value == keyName { + return true + } + } + return false +} diff --git a/pkg/pipeline/pipeline.go b/pkg/pipeline/pipeline.go index 4189e5f..4bdec93 100644 --- a/pkg/pipeline/pipeline.go +++ b/pkg/pipeline/pipeline.go @@ -89,7 +89,7 @@ func DefaultOptions() Options { return Options{ Operation: OperationPipeline, DryRun: false, - ContinueOnError: false, + ContinueOnError: true, Parallelism: 4, ComputeDiff: false, } diff --git a/pkg/pipeline/pipeline_test.go b/pkg/pipeline/pipeline_test.go index 285396c..a4279f4 100644 --- a/pkg/pipeline/pipeline_test.go +++ b/pkg/pipeline/pipeline_test.go @@ -90,6 +90,16 @@ func TestNew(t *testing.T) { } } +func TestDefaultOptions(t *testing.T) { + opts := DefaultOptions() + + assert.Equal(t, OperationPipeline, opts.Operation) + assert.False(t, opts.DryRun) + assert.True(t, opts.ContinueOnError) + assert.Equal(t, 4, opts.Parallelism) + assert.False(t, opts.ComputeDiff) +} + func TestPipeline_Operations(t *testing.T) { tests := []struct { name string diff --git a/pkg/pipeline/types.go b/pkg/pipeline/types.go index 6d8efb8..a0c1a8f 100644 --- a/pkg/pipeline/types.go +++ b/pkg/pipeline/types.go @@ -1,6 +1,12 @@ // Package pipeline provides unified configuration and orchestration for secrets syncing pipelines. package pipeline +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + // Config represents the unified pipeline configuration type Config struct { Log LogConfig `mapstructure:"log" yaml:"log"` @@ -248,19 +254,36 @@ type IdentityCenterDiscovery struct { // OrganizationsDiscovery discovers accounts from AWS Organizations type OrganizationsDiscovery struct { - OU string `mapstructure:"ou" yaml:"ou"` Tags map[string][]string `mapstructure:"tags" yaml:"tags"` Recursive bool `mapstructure:"recursive" yaml:"recursive"` NameMatching *NameMatchingConfig `mapstructure:"name_matching" yaml:"name_matching"` - // Enhanced filtering (v1.2.0) - OUs []string `mapstructure:"ous" yaml:"ous"` // Multiple OUs support + OUs []string `mapstructure:"ous" yaml:"ous"` TagFilters []TagFilter `mapstructure:"tag_filters" yaml:"tag_filters"` TagCombination string `mapstructure:"tag_combination" yaml:"tag_combination"` // "AND" or "OR", default "AND" ExcludeStatuses []string `mapstructure:"exclude_statuses" yaml:"exclude_statuses"` // e.g., ["SUSPENDED", "CLOSED"] CacheOUStructure bool `mapstructure:"cache_ou_structure" yaml:"cache_ou_structure"` // Cache OU hierarchy } +// UnmarshalYAML rejects removed single-OU configuration instead of silently ignoring it. +func (o *OrganizationsDiscovery) UnmarshalYAML(value *yaml.Node) error { + if value.Kind == yaml.MappingNode { + for i := 0; i < len(value.Content)-1; i += 2 { + if value.Content[i].Value == "ou" { + return fmt.Errorf("organizations.ou has been removed; use organizations.ous with one or more OU IDs") + } + } + } + + type organizationsDiscovery OrganizationsDiscovery + var decoded organizationsDiscovery + if err := value.Decode(&decoded); err != nil { + return err + } + *o = OrganizationsDiscovery(decoded) + return nil +} + // TagFilter represents a single tag filtering condition with wildcard support type TagFilter struct { Key string `mapstructure:"key" yaml:"key"` diff --git a/pkg/utils/deepmerge_test.go b/pkg/utils/deepmerge_test.go index 1da2c3e..8f5d669 100644 --- a/pkg/utils/deepmerge_test.go +++ b/pkg/utils/deepmerge_test.go @@ -374,8 +374,8 @@ func TestDeepMerge_MultipleImports(t *testing.T) { } } -// TestDeepMerge_FSCCompatibility tests specific FSC use cases for FlipsideCrypto compatibility -func TestDeepMerge_FSCCompatibility(t *testing.T) { +// TestDeepMerge_FSCUseCases tests specific FSC deep-merge use cases. +func TestDeepMerge_FSCUseCases(t *testing.T) { t.Run("3+ level deep nesting", func(t *testing.T) { // FSC has deeply nested config structures dst := map[string]interface{}{ @@ -553,7 +553,7 @@ func TestDeepMerge_FSCCompatibility(t *testing.T) { } }) - t.Run("JSON round-trip compatibility", func(t *testing.T) { + t.Run("JSON round trip", func(t *testing.T) { // Verify that merging works correctly through JSON serialization dstJSON := []byte(`{ "api_keys": {"stripe": "sk_test_123"}, diff --git a/python/secretssync/secretssync.go b/python/secretssync/secretssync.go index bcab42c..5da4b06 100644 --- a/python/secretssync/secretssync.go +++ b/python/secretssync/secretssync.go @@ -73,7 +73,7 @@ func DefaultSyncOptions() *SyncOptions { DryRun: false, Operation: OperationPipeline, Targets: "", - ContinueOnError: false, + ContinueOnError: true, Parallelism: 4, ComputeDiff: false, OutputFormat: OutputFormatHuman, @@ -194,7 +194,7 @@ func RunPipeline(configPath string, opts *SyncOptions) *SyncResult { } // Process results - result.TargetCount = len(results) + result.TargetCount = countUniqueTargets(results) result.Success = err == nil for _, r := range results { @@ -303,6 +303,16 @@ func splitTargets(targets string) []string { return result } +func countUniqueTargets(results []pipeline.Result) int { + targets := make(map[string]struct{}) + for _, result := range results { + if result.Target != "" { + targets[result.Target] = struct{}{} + } + } + return len(targets) +} + // ConfigInfo returns information about a configuration file type ConfigInfo struct { Valid bool // Whether the configuration is valid diff --git a/python/secretssync/secretssync_test.go b/python/secretssync/secretssync_test.go new file mode 100644 index 0000000..3b5b758 --- /dev/null +++ b/python/secretssync/secretssync_test.go @@ -0,0 +1,40 @@ +package secretssync + +import ( + "testing" + + "github.com/jbcom/secrets-sync/pkg/pipeline" +) + +func TestDefaultSyncOptions(t *testing.T) { + opts := DefaultSyncOptions() + + if opts.Operation != OperationPipeline { + t.Fatalf("Operation = %q, want %q", opts.Operation, OperationPipeline) + } + if opts.DryRun { + t.Fatal("DryRun = true, want false") + } + if !opts.ContinueOnError { + t.Fatal("ContinueOnError = false, want true") + } + if opts.Parallelism != 4 { + t.Fatalf("Parallelism = %d, want 4", opts.Parallelism) + } + if opts.OutputFormat != OutputFormatHuman { + t.Fatalf("OutputFormat = %q, want %q", opts.OutputFormat, OutputFormatHuman) + } +} + +func TestCountUniqueTargets(t *testing.T) { + results := []pipeline.Result{ + {Target: "prod", Phase: "merge"}, + {Target: "prod", Phase: "sync"}, + {Target: "staging", Phase: "sync"}, + {Phase: "sync"}, + } + + if got := countUniqueTargets(results); got != 2 { + t.Fatalf("countUniqueTargets() = %d, want 2", got) + } +} diff --git a/release_config_test.go b/release_config_test.go new file mode 100644 index 0000000..6490f22 --- /dev/null +++ b/release_config_test.go @@ -0,0 +1,25 @@ +package secretsync_test + +import ( + "os" + "strings" + "testing" +) + +func TestGoReleaserUsesProductNameInReleaseNotes(t *testing.T) { + content, err := os.ReadFile(".goreleaser.yml") + if err != nil { + t.Fatalf("read .goreleaser.yml: %v", err) + } + + text := string(content) + if !strings.Contains(text, "project_name: secretsync") { + t.Fatalf(".goreleaser.yml should keep lowercase secretsync artifact naming") + } + if !strings.Contains(text, "## SecretSync {{ .Tag }}") { + t.Fatalf(".goreleaser.yml should use SecretSync product casing in release notes") + } + if strings.Contains(text, "## secretsync {{ .Tag }}") { + t.Fatalf(".goreleaser.yml should not use lowercase product name in release notes") + } +} diff --git a/scripts/break-fork.sh b/scripts/break-fork.sh deleted file mode 100755 index 4e2c732..0000000 --- a/scripts/break-fork.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# break-fork.sh - Script to break the fork and rename to secretsync -# -# Usage: ./scripts/break-fork.sh [new-org] [new-repo] -# Example: ./scripts/break-fork.sh jbcom secretsync - -set -euo pipefail - -NEW_ORG="${1:-jbcom}" -NEW_REPO="${2:-secretsync}" -OLD_MODULE="github.com/robertlestak/vault-secret-sync" -NEW_MODULE="github.com/${NEW_ORG}/${NEW_REPO}" - -echo "=== Breaking Fork: vault-secret-sync → ${NEW_REPO} ===" -echo "" -echo "Old module: ${OLD_MODULE}" -echo "New module: ${NEW_MODULE}" -echo "" - -# Confirm -read -p "Continue? (y/N) " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Aborted." - exit 1 -fi - -echo "" -echo "Step 1: Updating go.mod..." -sed -i "s|module ${OLD_MODULE}|module ${NEW_MODULE}|g" go.mod - -echo "Step 2: Updating all Go imports..." -find . -name "*.go" -type f -exec sed -i "s|${OLD_MODULE}|${NEW_MODULE}|g" {} + - -echo "Step 3: Updating documentation..." -find . -name "*.md" -type f -exec sed -i "s|${OLD_MODULE}|${NEW_MODULE}|g" {} + -find . -name "*.md" -type f -exec sed -i "s|vault-secret-sync|${NEW_REPO}|g" {} + - -echo "Step 4: Updating Helm charts..." -find deploy/charts -name "*.yaml" -type f -exec sed -i "s|vault-secret-sync|${NEW_REPO}|g" {} + -find deploy/charts -name "Chart.yaml" -type f -exec sed -i "s|vault-secret-sync|${NEW_REPO}|g" {} + - -echo "Step 5: Updating Dockerfile..." -sed -i "s|vault-secret-sync|${NEW_REPO}|g" Dockerfile - -echo "Step 6: Updating GitHub workflows..." -find .github -name "*.yml" -type f -exec sed -i "s|vault-secret-sync|${NEW_REPO}|g" {} + - -echo "Step 7: Running go mod tidy..." -go mod tidy - -echo "Step 8: Verifying build..." -go build ./... - -echo "" -echo "=== Fork Break Complete ===" -echo "" -echo "Next steps:" -echo "1. Create new GitHub repo: https://github.com/${NEW_ORG}/${NEW_REPO}" -echo " - Do NOT create as a fork" -echo " - Create empty (no README, no .gitignore)" -echo "" -echo "2. Remove old git history and push fresh:" -echo " rm -rf .git" -echo " git init" -echo " git add -A" -echo " git commit -m 'Initial commit: ${NEW_REPO} - Universal Secrets Sync'" -echo " git remote add origin https://github.com/${NEW_ORG}/${NEW_REPO}.git" -echo " git push -u origin main" -echo "" -echo "3. (Optional) Archive old repo with redirect notice" -echo "" -echo "4. Update Docker Hub / container registry" -echo "" -echo "5. Update Helm chart repository" diff --git a/tests/integration/README.md b/tests/integration/README.md index 30e4149..1ccbc1c 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -112,7 +112,7 @@ The GitHub Actions CI workflow uses the docker-compose stack for integration tes - Syncs to AWS Secrets Manager - Validates final output -2. **fsc_compatibility_test.go** - FlipsideCrypto compatibility +2. **fsc_pipeline_test.go** - FSC pipeline shape - Multi-tier target inheritance (Staging → Production → Demo) - AWS Organizations account discovery - Fuzzy account name matching diff --git a/tests/integration/fixtures/pipeline-config.yaml b/tests/integration/fixtures/pipeline-config.yaml index 63b5808..410407c 100644 --- a/tests/integration/fixtures/pipeline-config.yaml +++ b/tests/integration/fixtures/pipeline-config.yaml @@ -68,7 +68,8 @@ dynamic_targets: dev_accounts: discovery: organizations: - ou: "ou-xxxx-development" + ous: + - "ou-xxxx-development" recursive: true # Fuzzy match account names to target configs name_matching: diff --git a/tests/integration/fsc_compatibility_test.go b/tests/integration/fsc_pipeline_test.go similarity index 98% rename from tests/integration/fsc_compatibility_test.go rename to tests/integration/fsc_pipeline_test.go index 3fd551c..c7bebd3 100644 --- a/tests/integration/fsc_compatibility_test.go +++ b/tests/integration/fsc_pipeline_test.go @@ -1,4 +1,4 @@ -// Package integration provides FSC-compatible end-to-end tests. +// Package integration provides FSC pipeline-shape end-to-end tests. // These tests validate the complete pipeline with realistic patterns: // - Multi-tier target inheritance (Staging → Production → Demo) // - Deepmerge strategies (list append, dict merge, scalar override) @@ -46,8 +46,8 @@ type AccountData struct { } `json:"organizational_units"` } -// TestFSCCompatibilityFullPipeline validates the complete FSC-compatible workflow -func TestFSCCompatibilityFullPipeline(t *testing.T) { +// TestFSCPipelineFullFlow validates the complete FSC-style workflow. +func TestFSCPipelineFullFlow(t *testing.T) { skipIfNoIntegrationEnv(t) ctx := context.Background() diff --git a/workflow_pinning_test.go b/workflow_pinning_test.go new file mode 100644 index 0000000..82bece8 --- /dev/null +++ b/workflow_pinning_test.go @@ -0,0 +1,229 @@ +package secretsync_test + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +var ( + actionRefPattern = regexp.MustCompile(`^\s*(?:-\s*)?uses:\s*([^#\s]+)(?:\s+#\s*(\S+))?`) + actionVersionCommentPattern = regexp.MustCompile(`^v\d+\.\d+\.\d+$`) + pinnedSHAPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + publishingChecklistPinPattern = regexp.MustCompile("^\\|\\s*`([^`]+)`\\s*\\|\\s*`([^`]+)`\\s*\\|\\s*`([0-9a-f]{40})`\\s*\\|$") +) + +type workflowActionPin struct { + Version string + SHA string +} + +func TestWorkflowActionsArePinnedToExactSHAs(t *testing.T) { + pins := workflowActionPins(t) + if len(pins) == 0 { + t.Fatalf("expected workflow action pins") + } +} + +func TestPublishingChecklistMatchesWorkflowActionPins(t *testing.T) { + workflowPins := workflowActionPins(t) + checklistPins := publishingChecklistPins(t) + + if len(checklistPins) == 0 { + t.Fatalf("docs/PUBLISHING_CHECKLIST.md must list current workflow action pins") + } + if len(workflowPins) != len(checklistPins) { + t.Fatalf("publishing checklist pin count mismatch: workflow=%d checklist=%d", len(workflowPins), len(checklistPins)) + } + for action, workflowPin := range workflowPins { + checklistPin, ok := checklistPins[action] + if !ok { + t.Fatalf("docs/PUBLISHING_CHECKLIST.md missing workflow action %s", action) + } + if checklistPin != workflowPin { + t.Fatalf("docs/PUBLISHING_CHECKLIST.md pin for %s = %+v, want %+v", action, checklistPin, workflowPin) + } + } + for action := range checklistPins { + if _, ok := workflowPins[action]; !ok { + t.Fatalf("docs/PUBLISHING_CHECKLIST.md lists non-workflow action %s", action) + } + } +} + +func TestMaintainedDocsPinThirdPartyActions(t *testing.T) { + paths := markdownAndExampleWorkflowFiles(t) + var offenders []string + + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + for index, line := range strings.Split(string(content), "\n") { + matches := actionRefPattern.FindStringSubmatch(line) + if matches == nil { + continue + } + + uses := strings.TrimSpace(matches[1]) + if shouldSkipDocumentedActionPin(uses) { + continue + } + + _, ref, found := strings.Cut(uses, "@") + version := "" + if len(matches) > 2 { + version = matches[2] + } + if !found || !pinnedSHAPattern.MatchString(ref) { + offenders = append(offenders, path+":"+itoa(index+1)+": "+uses) + continue + } + if !actionVersionCommentPattern.MatchString(version) { + offenders = append(offenders, path+":"+itoa(index+1)+": missing stable version comment for "+uses) + } + } + } + + if len(offenders) > 0 { + t.Fatalf("maintained docs/examples must pin third-party actions to exact commit SHAs:\n%s", strings.Join(offenders, "\n")) + } +} + +func workflowActionPins(t *testing.T) map[string]workflowActionPin { + t.Helper() + + workflowRoot := filepath.Join(".github", "workflows") + entries, err := os.ReadDir(workflowRoot) + if err != nil { + t.Fatalf("read workflow directory: %v", err) + } + + pins := map[string]workflowActionPin{} + var offenders []string + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml") { + continue + } + + path := filepath.Join(workflowRoot, name) + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read workflow %s: %v", path, err) + } + + for index, line := range strings.Split(string(content), "\n") { + matches := actionRefPattern.FindStringSubmatch(line) + if matches == nil { + continue + } + + uses := strings.TrimSpace(matches[1]) + if strings.HasPrefix(uses, "./") || strings.HasPrefix(uses, "docker://") { + continue + } + + action, ref, found := strings.Cut(uses, "@") + version := "" + if len(matches) > 2 { + version = matches[2] + } + if !found || !pinnedSHAPattern.MatchString(ref) { + offenders = append(offenders, path+":"+itoa(index+1)+": "+uses) + continue + } + if !actionVersionCommentPattern.MatchString(version) { + offenders = append(offenders, path+":"+itoa(index+1)+": missing stable version comment for "+uses) + continue + } + + pin := workflowActionPin{Version: version, SHA: ref} + if existing, ok := pins[action]; ok && existing != pin { + offenders = append(offenders, path+":"+itoa(index+1)+": conflicting pin for "+action) + continue + } + pins[action] = pin + } + } + + if len(offenders) > 0 { + t.Fatalf("workflow actions must be pinned to exact commit SHAs:\n%s", strings.Join(offenders, "\n")) + } + return pins +} + +func markdownAndExampleWorkflowFiles(t *testing.T) []string { + t.Helper() + + roots := []string{"README.md", "docs", "examples"} + var paths []string + for _, root := range roots { + info, err := os.Stat(root) + if err != nil { + t.Fatalf("stat %s: %v", root, err) + } + if !info.IsDir() { + paths = append(paths, root) + continue + } + err = filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + switch filepath.Ext(path) { + case ".md", ".yml", ".yaml": + paths = append(paths, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + return paths +} + +func shouldSkipDocumentedActionPin(uses string) bool { + if strings.HasPrefix(uses, "./") || strings.HasPrefix(uses, "docker://") { + return true + } + if strings.HasPrefix(uses, "jbcom/secrets-sync@") { + return true + } + return false +} + +func publishingChecklistPins(t *testing.T) map[string]workflowActionPin { + t.Helper() + + content, err := os.ReadFile(filepath.Join("docs", "PUBLISHING_CHECKLIST.md")) + if err != nil { + t.Fatalf("read publishing checklist: %v", err) + } + + pins := map[string]workflowActionPin{} + for _, line := range strings.Split(string(content), "\n") { + matches := publishingChecklistPinPattern.FindStringSubmatch(strings.TrimSpace(line)) + if matches == nil { + continue + } + pins[matches[1]] = workflowActionPin{Version: matches[2], SHA: matches[3]} + } + return pins +} + +func itoa(value int) string { + return strconv.Itoa(value) +}