fix(health): isolate transient dependency outages

Dockerfile

@@ -98,7 +98,7 @@ USER appuser
 
 EXPOSE 8085
 
-# Gate Coolify's rolling cutover on dependencies being ready for application traffic.
+# Gate Coolify's rolling cutover on the JVM accepting traffic; external dependencies report separately.
 HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
     CMD curl --fail --silent --show-error http://localhost:${PORT:-8085}/actuator/health/readiness || exit 1
 

src/main/java/com/williamcallahan/javachat/config/QdrantHealthIndicator.java

@@ -1,8 +1,10 @@
 package com.williamcallahan.javachat.config;
 
 import com.williamcallahan.javachat.service.ExternalServiceHealth;
+import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.boot.actuate.health.Health;
 import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.boot.actuate.health.Status;
 import org.springframework.stereotype.Component;
 
 /**
@@ -20,14 +22,19 @@ public class QdrantHealthIndicator implements HealthIndicator {
     private static final String DETAIL_KEY_NEXT_CHECK_IN = "nextCheckIn";
 
     private final ExternalServiceHealth externalServiceHealth;
+    private final ObjectProvider<QdrantIndexInitializer> indexInitializerProvider;
 
     /**
      * Creates the health indicator with a reference to the external service health monitor.
      *
      * @param externalServiceHealth the service health monitor
+     * @param indexInitializerProvider provides initialization state outside the test profile
      */
-    public QdrantHealthIndicator(ExternalServiceHealth externalServiceHealth) {
+    public QdrantHealthIndicator(
+            ExternalServiceHealth externalServiceHealth,
+            ObjectProvider<QdrantIndexInitializer> indexInitializerProvider) {
         this.externalServiceHealth = externalServiceHealth;
+        this.indexInitializerProvider = indexInitializerProvider;
     }
 
     /**
@@ -43,6 +50,14 @@ public Health health() {
         // Trigger retry checks when unhealthy and backoff has elapsed.
         externalServiceHealth.triggerRetryIfDue(ExternalServiceHealth.SERVICE_QDRANT);
 
+        QdrantIndexInitializer indexInitializer = indexInitializerProvider.getIfAvailable();
+        if (indexInitializer != null) {
+            Health initializationHealth = indexInitializer.initializationHealth();
+            if (!Status.UP.equals(initializationHealth.getStatus())) {
+                return initializationHealth;
+            }
+        }
+
         ExternalServiceHealth.HealthSnapshot healthSnapshot =
                 externalServiceHealth.getHealthSnapshot(ExternalServiceHealth.SERVICE_QDRANT);
 

src/main/java/com/williamcallahan/javachat/config/QdrantRestConnection.java

@@ -84,8 +84,8 @@ public String restBaseUrl() {
      * Returns candidate REST base URLs ordered by likelihood, for discovery scenarios.
      *
      * <p>Under TLS the canonical URL is sufficient. Without TLS the list includes the
-     * default REST port, the mapped port, and the raw configured port as fallback so
-     * that the caller can probe multiple endpoints when the exact port is uncertain.
+     * default REST port and the mapped REST port. The raw configured port is excluded
+     * when it is a gRPC endpoint because REST probes cannot establish collection state there.
      *
      * @return ordered, deduplicated list of candidate base URLs
      */
@@ -96,7 +96,6 @@ public List<String> candidateRestBaseUrls() {
         LinkedHashSet<String> candidates = new LinkedHashSet<>();
         candidates.add(buildBaseUrl(HTTP_SCHEME, QDRANT_REST_PORT));
         candidates.add(buildBaseUrl(HTTP_SCHEME, mapGrpcPortToRestPort(configuredPort)));
-        candidates.add(buildBaseUrl(HTTP_SCHEME, configuredPort));
         return List.copyOf(candidates);
     }
 

src/main/java/com/williamcallahan/javachat/config/SecurityConfig.java

@@ -29,6 +29,7 @@ public class SecurityConfig {
     private static final String HEALTH_ENDPOINT = "/actuator/health";
     private static final String LIVENESS_ENDPOINT = "/actuator/health/liveness";
     private static final String READINESS_ENDPOINT = "/actuator/health/readiness";
+    private static final String DEPENDENCIES_ENDPOINT = "/actuator/health/dependencies";
     private static final String PROMETHEUS_ENDPOINT = "/actuator/prometheus";
 
     /**
@@ -63,7 +64,11 @@ public SecurityFilterChain managementSecurityFilterChain(
         http.securityMatcher(EndpointRequest.toAnyEndpoint())
                 .cors(c -> c.configurationSource(corsConfigurationSource))
                 .authorizeHttpRequests(auth -> auth.requestMatchers(
-                                HEALTH_ENDPOINT, LIVENESS_ENDPOINT, READINESS_ENDPOINT, PROMETHEUS_ENDPOINT)
+                                HEALTH_ENDPOINT,
+                                LIVENESS_ENDPOINT,
+                                READINESS_ENDPOINT,
+                                DEPENDENCIES_ENDPOINT,
+                                PROMETHEUS_ENDPOINT)
                         .permitAll()
                         .anyRequest()
                         .denyAll())

src/main/resources/application.properties

@@ -147,7 +147,8 @@ app.cors.max-age-seconds=3600
 management.endpoints.web.exposure.include=health,prometheus
 management.endpoint.health.probes.enabled=true
 management.endpoint.health.group.liveness.include=livenessState
-management.endpoint.health.group.readiness.include=readinessState,qdrant,embeddingModelKeepAlive
+management.endpoint.health.group.readiness.include=readinessState
+management.endpoint.health.group.dependencies.include=qdrant,embeddingModelKeepAlive
 management.info.build.enabled=true
 management.info.env.enabled=true
 info.application.name=${spring.application.name}

src/test/java/com/williamcallahan/javachat/JavaChatApplicationTests.java

@@ -1,20 +1,23 @@
 package com.williamcallahan.javachat;
 
 import static org.hamcrest.Matchers.containsString;
-import static org.mockito.Mockito.when;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.doThrow;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
+import com.williamcallahan.javachat.config.QdrantHealthIndicator;
 import com.williamcallahan.javachat.service.EmbeddingClient;
 import com.williamcallahan.javachat.service.EmbeddingModelKeepAlive;
+import com.williamcallahan.javachat.service.EmbeddingServiceUnavailableException;
 import io.qdrant.client.QdrantClient;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.Answers;
 import org.springframework.ai.chat.model.ChatModel;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.Status;
 import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
@@ -29,7 +32,8 @@
             "spring.ai.openai.api-key=test",
             "spring.ai.openai.chat.api-key=test",
             "management.endpoint.health.probes.enabled=true",
-            "management.endpoint.health.group.readiness.include=readinessState,qdrant,embeddingModelKeepAlive",
+            "management.endpoint.health.group.readiness.include=readinessState",
+            "management.endpoint.health.group.dependencies.include=qdrant,embeddingModelKeepAlive",
             "spring.ai.vectorstore.qdrant.host=localhost",
             "spring.ai.vectorstore.qdrant.use-tls=false",
             "spring.ai.vectorstore.qdrant.port=8086"
@@ -50,12 +54,17 @@ class JavaChatApplicationTests {
     @MockitoBean
     QdrantClient qdrantClient;
 
-    @MockitoBean
+    @Autowired
     EmbeddingModelKeepAlive embeddingModelKeepAlive;
 
+    @Autowired
+    QdrantHealthIndicator qdrantHealthIndicator;
+
     @BeforeEach
-    void reportEmbeddingDependencyUnavailable() {
-        when(embeddingModelKeepAlive.health()).thenReturn(Health.down().build());
+    void reportExternalDependenciesUnavailable() {
+        doThrow(new EmbeddingServiceUnavailableException("provider unavailable for readiness test"))
+                .when(embeddingClient)
+                .warmUp();
     }
 
     @Test
@@ -64,12 +73,18 @@ void contextLoads() {}
     @Test
     void exposesOnlyOperationalActuatorSurfaces() throws Exception {
         mockMvc.perform(get("/actuator/health/liveness")).andExpect(status().isOk());
-        mockMvc.perform(get("/actuator/health/readiness")).andExpect(status().isServiceUnavailable());
         mockMvc.perform(get("/actuator/prometheus"))
                 .andExpect(status().isOk())
                 .andExpect(content().string(containsString("jvm_")));
-        mockMvc.perform(get("/actuator/health")).andExpect(status().isServiceUnavailable());
         mockMvc.perform(get("/actuator/metrics")).andExpect(status().isForbidden());
         mockMvc.perform(get("/actuator/info")).andExpect(status().isForbidden());
     }
+
+    @Test
+    void externalDependencyFailuresRemainObservableWithoutBlockingReadiness() throws Exception {
+        assertEquals(Status.DOWN, embeddingModelKeepAlive.health().getStatus());
+        assertEquals(Status.DOWN, qdrantHealthIndicator.health().getStatus());
+        mockMvc.perform(get("/actuator/health/readiness")).andExpect(status().isOk());
+        mockMvc.perform(get("/actuator/health/dependencies")).andExpect(status().isServiceUnavailable());
+    }
 }

src/test/java/com/williamcallahan/javachat/config/QdrantHealthIndicatorTest.java

@@ -9,6 +9,7 @@
 import com.williamcallahan.javachat.service.ExternalServiceHealth;
 import java.time.Duration;
 import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.boot.actuate.health.Health;
 import org.springframework.boot.actuate.health.Status;
 
@@ -20,7 +21,8 @@ class QdrantHealthIndicatorTest {
     @Test
     void health_triggersRetryEvaluationBeforeSnapshotRendering() {
         ExternalServiceHealth externalServiceHealth = mock(ExternalServiceHealth.class);
-        QdrantHealthIndicator qdrantHealthIndicator = new QdrantHealthIndicator(externalServiceHealth);
+        QdrantHealthIndicator qdrantHealthIndicator =
+                new QdrantHealthIndicator(externalServiceHealth, absentIndexInitializerProvider());
 
         ExternalServiceHealth.HealthSnapshot unhealthySnapshot = new ExternalServiceHealth.HealthSnapshot(
                 ExternalServiceHealth.SERVICE_QDRANT, false, "Unhealthy (checking now...)", Duration.ZERO);
@@ -32,4 +34,39 @@ void health_triggersRetryEvaluationBeforeSnapshotRendering() {
         verify(externalServiceHealth, times(1)).triggerRetryIfDue(ExternalServiceHealth.SERVICE_QDRANT);
         assertEquals(Status.DOWN, health.getStatus());
     }
+
+    @Test
+    void health_remainsDownUntilCollectionInitializationCompletes() {
+        ExternalServiceHealth externalServiceHealth = mock(ExternalServiceHealth.class);
+        QdrantIndexInitializer indexInitializer = mock(QdrantIndexInitializer.class);
+        when(indexInitializer.initializationHealth())
+                .thenReturn(
+                        Health.down().withDetail("initialization", "pending").build());
+        QdrantHealthIndicator qdrantHealthIndicator =
+                new QdrantHealthIndicator(externalServiceHealth, indexInitializerProvider(indexInitializer));
+
+        Health health = qdrantHealthIndicator.health();
+
+        assertEquals(Status.DOWN, health.getStatus());
+        assertEquals("pending", health.getDetails().get("initialization"));
+        verify(externalServiceHealth, times(1)).triggerRetryIfDue(ExternalServiceHealth.SERVICE_QDRANT);
+    }
+
+    private ObjectProvider<QdrantIndexInitializer> absentIndexInitializerProvider() {
+        return new ObjectProvider<>() {
+            @Override
+            public QdrantIndexInitializer getIfAvailable() {
+                return null;
+            }
+        };
+    }
+
+    private ObjectProvider<QdrantIndexInitializer> indexInitializerProvider(QdrantIndexInitializer indexInitializer) {
+        return new ObjectProvider<>() {
+            @Override
+            public QdrantIndexInitializer getIfAvailable() {
+                return indexInitializer;
+            }
+        };
+    }
 }