diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..6313b56c5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/docker/deploy.sh b/docker/deploy.sh index d2edbb446..9266c4929 100755 --- a/docker/deploy.sh +++ b/docker/deploy.sh @@ -436,6 +436,15 @@ persist_deploy_options() { } generate_minio_ak_sk() { + if [ -n "${MINIO_ACCESS_KEY:-}" ] && [ -n "${MINIO_SECRET_KEY:-}" ]; then + echo " Reusing existing MinIO access keys from docker/.env" + export MINIO_ACCESS_KEY + export MINIO_SECRET_KEY + update_env_var "MINIO_ACCESS_KEY" "$MINIO_ACCESS_KEY" + update_env_var "MINIO_SECRET_KEY" "$MINIO_SECRET_KEY" + return 0 + fi + echo "🔑 Generating MinIO keys..." if [ "$(uname -s | tr '[:upper:]' '[:lower:]')" = "mingw" ] || [ "$(uname -s | tr '[:upper:]' '[:lower:]')" = "msys" ]; then @@ -592,6 +601,12 @@ disable_dashboard() { update_env_var "DISABLE_CELERY_FLOWER" "true" } +sync_monitoring_env_vars() { + update_env_var "ENABLE_TELEMETRY" "$(deployment_monitoring_enabled)" + update_env_var "MONITORING_PROVIDER" "$DEPLOYMENT_MONITORING_PROVIDER" + update_env_var "MONITORING_DASHBOARD_URL" "$(deployment_monitoring_dashboard_url docker)" +} + pull_mcp_image() { if [ "$DEPLOYMENT_IMAGE_SOURCE" = "local-latest" ]; then echo "🔄 Skipping MCP image pull because image source is local-latest." @@ -813,6 +828,17 @@ deploy_core_services() { fi } +stop_unselected_data_process_service() { + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "data-process" && return 0 + + local compose_file="docker-compose${COMPOSE_FILE_SUFFIX}" + [ -f "$compose_file" ] || return 0 + + echo "data-process is not selected; stopping existing Docker container if present..." + ${docker_compose_command} -p nexent -f "$compose_file" stop nexent-data-process >/dev/null 2>&1 || true + ${docker_compose_command} -p nexent -f "$compose_file" rm -f nexent-data-process >/dev/null 2>&1 || true +} + deploy_infrastructure() { # Start infrastructure services (basic services only) echo "🔧 Starting infrastructure services..." @@ -960,6 +986,7 @@ apply_deployment_common_config() { set -a source "$SCRIPT_DIR/.env.generated" set +a + sync_monitoring_env_vars deployment_print_summary docker } @@ -1341,6 +1368,8 @@ main_deploy() { # Select deployment components, port policy and image source via shared config. apply_deployment_common_config || { echo "❌ Deployment configuration failed"; exit 1; } + deployment_persist_local_config + # Check only the ports published by the selected deployment configuration. check_deployment_ports @@ -1367,6 +1396,8 @@ main_deploy() { deploy_monitoring || { echo "❌ Monitoring deployment failed"; exit 1; } + stop_unselected_data_process_service + # Generate Elasticsearch API key generate_elasticsearch_api_key || { echo "❌ Elasticsearch API key generation failed"; exit 1; } diff --git a/k8s/helm/deploy.sh b/k8s/helm/deploy.sh index 62a485614..7a583307d 100755 --- a/k8s/helm/deploy.sh +++ b/k8s/helm/deploy.sh @@ -332,6 +332,22 @@ load_existing_supabase_secrets() { return 0 } +load_existing_minio_secrets() { + local existing_access_key + local existing_secret_key + + existing_access_key="$(get_existing_secret_value "MINIO_ACCESS_KEY")" || return 1 + existing_secret_key="$(get_existing_secret_value "MINIO_SECRET_KEY")" || return 1 + + if [ -z "$existing_access_key" ] || [ -z "$existing_secret_key" ]; then + return 1 + fi + + MINIO_ACCESS_KEY="$existing_access_key" + MINIO_SECRET_KEY="$existing_secret_key" + return 0 +} + # Generate Supabase secrets (only for full version) generate_supabase_secrets() { if [ "$DEPLOYMENT_VERSION" != "full" ]; then @@ -434,6 +450,19 @@ restart_supabase_auth_services() { done } +restart_minio_for_current_secrets() { + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "infrastructure" || return 0 + + echo "" + echo "Restarting MinIO to ensure current credentials are loaded..." + kubectl rollout restart deployment/nexent-minio -n "$NAMESPACE" 2>/dev/null || true + if kubectl rollout status deployment/nexent-minio -n "$NAMESPACE" --timeout=300s >/dev/null 2>&1; then + echo " nexent-minio is ready." + else + echo " Warning: nexent-minio did not become ready within timeout." + fi +} + render_runtime_secret_values() { { echo "nexent-common:" @@ -457,6 +486,7 @@ apply() { # Step 1: Select deployment components, port policy and image source. apply_deployment_common_config + deployment_persist_local_config # Step 2: Render generated values with image tags from selected environment update_values_yaml @@ -465,7 +495,10 @@ apply() { echo "==========================================" echo " MinIO Access Key/Secret Key Setup" echo "==========================================" - if grep -q "minio:" "$COMMON_VALUES" && grep -q "accessKey:" "$COMMON_VALUES"; then + if load_existing_minio_secrets; then + echo "Reusing existing MinIO credentials from Kubernetes secret." + echo "Access Key: $MINIO_ACCESS_KEY" + elif grep -q "minio:" "$COMMON_VALUES" && grep -q "accessKey:" "$COMMON_VALUES"; then MINIO_ACCESS_KEY=$(grep "accessKey:" "$COMMON_VALUES" | head -1 | sed 's/.*accessKey: *//' | tr -d '"' | tr -d "'" | xargs) MINIO_SECRET_KEY=$(grep "secretKey:" "$COMMON_VALUES" | head -1 | sed 's/.*secretKey: *//' | tr -d '"' | tr -d "'" | xargs) fi @@ -540,6 +573,7 @@ apply() { --set nexent-common.secrets.ssh.username="$SSH_USERNAME" \ --set nexent-common.secrets.ssh.password="$SSH_PASSWORD" + restart_minio_for_current_secrets restart_supabase_auth_services # Step 9: Wait for Elasticsearch to be ready and initialize API key diff --git a/k8s/helm/nexent/charts/nexent-data-process/values.yaml b/k8s/helm/nexent/charts/nexent-data-process/values.yaml index cdf95fbb9..189292667 100644 --- a/k8s/helm/nexent/charts/nexent-data-process/values.yaml +++ b/k8s/helm/nexent/charts/nexent-data-process/values.yaml @@ -18,4 +18,4 @@ resources: config: skipProxy: "true" pythonPath: "/opt/backend" - dockerEnvironment: "true" + dockerEnvironment: "false" diff --git a/scripts/deployment/common.sh b/scripts/deployment/common.sh index a0f18cf6f..43cb2adca 100755 --- a/scripts/deployment/common.sh +++ b/scripts/deployment/common.sh @@ -921,6 +921,56 @@ deployment_apply_image_source() { export SUPABASE_DB="${SUPABASE_DB:-supabase/postgres:15.8.1.060}" } +deployment_monitoring_enabled() { + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then + printf 'true' + else + printf 'false' + fi +} + +deployment_monitoring_dashboard_url() { + local target="${1:-docker}" + + if ! deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then + printf '' + return 0 + fi + + case "$target:$DEPLOYMENT_MONITORING_PROVIDER" in + docker:phoenix) + printf 'http://localhost:%s' "${PHOENIX_PORT:-6006}" + ;; + docker:langfuse) + printf 'http://localhost:%s' "${LANGFUSE_PORT:-3001}" + ;; + docker:grafana) + printf 'http://localhost:%s/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1' "${GRAFANA_PORT:-3002}" + ;; + docker:zipkin) + printf 'http://localhost:%s' "${ZIPKIN_PORT:-9411}" + ;; + k8s:phoenix|helm:phoenix) + printf 'http://localhost:30006' + ;; + k8s:langfuse|helm:langfuse) + printf 'http://localhost:30001' + ;; + k8s:grafana|helm:grafana) + printf 'http://localhost:30002/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1' + ;; + k8s:zipkin|helm:zipkin) + printf 'http://localhost:30011' + ;; + *:langsmith) + printf 'https://smith.langchain.com/' + ;; + *) + printf '' + ;; + esac +} + deployment_render_docker_env() { local output_file="$1" mkdir -p "$(dirname "$output_file")" @@ -937,6 +987,9 @@ deployment_render_docker_env() { printf 'SUPABASE_KONG="%s"\n' "$SUPABASE_KONG" printf 'SUPABASE_GOTRUE="%s"\n' "$SUPABASE_GOTRUE" printf 'SUPABASE_DB="%s"\n' "$SUPABASE_DB" + printf 'ENABLE_TELEMETRY="%s"\n' "$(deployment_monitoring_enabled)" + printf 'MONITORING_PROVIDER="%s"\n' "$DEPLOYMENT_MONITORING_PROVIDER" + printf 'MONITORING_DASHBOARD_URL="%s"\n' "$(deployment_monitoring_dashboard_url docker)" } > "$output_file" } @@ -1123,6 +1176,7 @@ deployment_render_helm_values() { printf ' enabled: false\n' fi printf ' provider: "%s"\n' "$DEPLOYMENT_MONITORING_PROVIDER" + printf ' dashboardUrl: "%s"\n' "$(deployment_monitoring_dashboard_url k8s)" printf 'nexent-monitoring:\n' if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then printf ' enabled: true\n'