From 6c86a440c23c6da9febc00bcff33d5f4f1751f19 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 21 May 2026 18:11:46 -0400 Subject: [PATCH 1/5] Add PHP FFE system-test scaffold --- docs/understand/weblogs/end-to-end_weblog.md | 14 ++ manifests/php.yml | 6 +- tests/ffe/README.md | 28 +++ utils/build/docker/php/common/ffe.php | 185 ++++++++++++++++++ .../docker/php/common/rewrite-rules.conf | 1 + 5 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 utils/build/docker/php/common/ffe.php diff --git a/docs/understand/weblogs/end-to-end_weblog.md b/docs/understand/weblogs/end-to-end_weblog.md index 08dabf6b078..ff88e4c623e 100644 --- a/docs/understand/weblogs/end-to-end_weblog.md +++ b/docs/understand/weblogs/end-to-end_weblog.md @@ -1026,6 +1026,20 @@ The endpoint must accept a query string parameter `code`, which should be an int This endpoint is used for client-stats tests to provide a separate "resource" via the endpoint path `stats-unique` to disambiguate those tests from other stats generating tests. +### POST /ffe + +This endpoint is used by the Feature Flags & Experimentation scenario. It must +accept a JSON body with these fields: + +- `flag`: the feature flag key to evaluate. +- `variationType`: the expected variation type. +- `defaultValue`: the value to return when evaluation cannot resolve the flag. +- `targetingKey`: the evaluation subject key. +- `attributes`: flat scalar targeting attributes. + +The response must be JSON and include at least `value` and `reason`. Error +responses should also include `errorCode` and `errorMessage`. + ### GET /healthcheck Returns a JSON dict, with those values : diff --git a/manifests/php.yml b/manifests/php.yml index 28e9cb330d2..933414f0be1 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -605,9 +605,9 @@ manifest: component_version: <1.12.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) - tests/ffe/test_dynamic_evaluation.py: missing_feature - tests/ffe/test_exposures.py: missing_feature - tests/ffe/test_flag_eval_metrics.py: missing_feature + tests/ffe/test_dynamic_evaluation.py: missing_feature (PHP /ffe endpoint is scaffolded; native evaluator and Remote Config lifecycle are not wired yet) + tests/ffe/test_exposures.py: missing_feature (PHP exposure buffering and native flush lifecycle are not wired yet) + tests/ffe/test_flag_eval_metrics.py: missing_feature (PHP feature_flag.evaluations metric emission is not wired yet) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: missing_feature tests/integrations/crossed_integrations/test_kinesis.py::Test_Kinesis_PROPAGATION_VIA_MESSAGE_ATTRIBUTES: missing_feature diff --git a/tests/ffe/README.md b/tests/ffe/README.md index c67f4488c5f..bc6aa2100b7 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -16,6 +16,34 @@ This directory contains system tests for the Feature Flags & Experimentation (FF ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION --library ``` +## PHP Local Runbook + +PHP FFE system tests are scaffolded but not enabled in `manifests/php.yml` yet. Keep +them marked `missing_feature` until a local run passes against a real +tracer-backed PHP weblog. + +Build the PHP weblog images: + +```bash +./build.sh php +``` + +Run the FFE scenario for PHP: + +```bash +./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION --library php -k "Test_FFE" +``` + +The PHP weblog exposes `POST /ffe` from +`utils/build/docker/php/common/ffe.php`. The endpoint accepts the canonical +evaluation case fields used by the fixture corpus: `flag`, `variationType`, +`defaultValue`, `targetingKey`, and `attributes`. + +Until the native PHP evaluator bridge and Remote Config lifecycle are wired, the +endpoint returns the provided default value with `reason=ERROR` and +`errorCode=PROVIDER_NOT_READY`. Once the PHP client is available in the weblog, +the endpoint will call `DDTrace\FeatureFlags\Client` for the same request shape. + --- # Eval Metrics Implementation Guide diff --git a/utils/build/docker/php/common/ffe.php b/utils/build/docker/php/common/ffe.php new file mode 100644 index 00000000000..1c1415e49ec --- /dev/null +++ b/utils/build/docker/php/common/ffe.php @@ -0,0 +1,185 @@ + null, + 'reason' => 'ERROR', + 'variant' => null, + 'errorCode' => $errorCode, + 'errorMessage' => $errorMessage, + 'providerState' => array('ready' => false), + )); +} + +function dd_ffe_read_payload() +{ + $rawBody = file_get_contents('php://input'); + if ($rawBody === false || $rawBody === '') { + return array(); + } + + $payload = json_decode($rawBody, true); + if (json_last_error() !== JSON_ERROR_NONE || !is_array($payload)) { + dd_ffe_error_response(400, 'INVALID_REQUEST', 'Expected a JSON object request body.'); + exit; + } + + return $payload; +} + +function dd_ffe_normalized_variation_type($variationType) +{ + return strtoupper(str_replace('-', '_', (string) $variationType)); +} + +function dd_ffe_normalize_default_value($defaultValue, $variationType) +{ + switch (dd_ffe_normalized_variation_type($variationType)) { + case 'BOOLEAN': + return is_bool($defaultValue) ? $defaultValue : (bool) $defaultValue; + case 'STRING': + return is_string($defaultValue) ? $defaultValue : (string) $defaultValue; + case 'INTEGER': + return is_int($defaultValue) ? $defaultValue : (int) $defaultValue; + case 'NUMERIC': + case 'FLOAT': + case 'DOUBLE': + return is_int($defaultValue) || is_float($defaultValue) ? $defaultValue : (float) $defaultValue; + case 'JSON': + case 'OBJECT': + return is_array($defaultValue) ? $defaultValue : array(); + default: + return $defaultValue; + } +} + +function dd_ffe_scalar_attributes(array $attributes) +{ + $normalized = array(); + foreach ($attributes as $key => $value) { + if (is_bool($value) || is_int($value) || is_float($value) || is_string($value)) { + $normalized[(string) $key] = $value; + } + } + + return $normalized; +} + +function dd_ffe_evaluate_with_client($flagKey, $variationType, $defaultValue, $targetingKey, array $attributes) +{ + if (!class_exists('\\DDTrace\\FeatureFlags\\Client')) { + return null; + } + + $client = \DDTrace\FeatureFlags\Client::create(); + $context = array( + 'targetingKey' => $targetingKey, + 'attributes' => $attributes, + ); + + switch (dd_ffe_normalized_variation_type($variationType)) { + case 'BOOLEAN': + return $client->getBooleanDetails($flagKey, $defaultValue, $context); + case 'STRING': + return $client->getStringDetails($flagKey, $defaultValue, $context); + case 'INTEGER': + return $client->getIntegerDetails($flagKey, $defaultValue, $context); + case 'NUMERIC': + case 'FLOAT': + case 'DOUBLE': + return $client->getFloatDetails($flagKey, $defaultValue, $context); + case 'JSON': + case 'OBJECT': + return $client->getObjectDetails($flagKey, is_array($defaultValue) ? $defaultValue : array(), $context); + default: + throw new InvalidArgumentException('Unsupported variationType: ' . (string) $variationType); + } +} + +function dd_ffe_details_payload($details) +{ + $payload = array( + 'value' => $details->getValue(), + 'reason' => $details->getReason(), + 'variant' => $details->getVariant(), + 'errorCode' => $details->getErrorCode(), + 'errorMessage' => $details->getErrorMessage(), + 'flagMetadata' => $details->getFlagMetadata(), + 'exposureData' => $details->getExposureData(), + 'providerState' => $details->getProviderState(), + ); + + if (method_exists($details, 'getValueType')) { + $payload['valueType'] = $details->getValueType(); + } + + return $payload; +} + +$payload = dd_ffe_read_payload(); + +if (!isset($payload['flag']) || !is_string($payload['flag']) || $payload['flag'] === '') { + dd_ffe_error_response(400, 'INVALID_REQUEST', 'Expected non-empty string field: flag.'); + exit; +} + +if (!array_key_exists('variationType', $payload) || !is_string($payload['variationType'])) { + dd_ffe_error_response(400, 'INVALID_REQUEST', 'Expected string field: variationType.'); + exit; +} + +if (!array_key_exists('defaultValue', $payload)) { + dd_ffe_error_response(400, 'INVALID_REQUEST', 'Expected field: defaultValue.'); + exit; +} + +$flagKey = $payload['flag']; +$variationType = $payload['variationType']; +$defaultValue = dd_ffe_normalize_default_value($payload['defaultValue'], $variationType); +$targetingKey = isset($payload['targetingKey']) && $payload['targetingKey'] !== null + ? (string) $payload['targetingKey'] + : null; +$attributes = isset($payload['attributes']) && is_array($payload['attributes']) + ? dd_ffe_scalar_attributes($payload['attributes']) + : array(); + +try { + $details = dd_ffe_evaluate_with_client($flagKey, $variationType, $defaultValue, $targetingKey, $attributes); + if ($details !== null) { + dd_ffe_json_response(200, dd_ffe_details_payload($details)); + return; + } +} catch (Throwable $exception) { + dd_ffe_json_response(200, array( + 'value' => $defaultValue, + 'reason' => 'ERROR', + 'variant' => null, + 'errorCode' => 'PROVIDER_NOT_READY', + 'errorMessage' => $exception->getMessage(), + 'providerState' => array( + 'ready' => false, + 'productionRuntime' => false, + ), + )); + return; +} + +dd_ffe_json_response(200, array( + 'value' => $defaultValue, + 'reason' => 'ERROR', + 'variant' => null, + 'errorCode' => 'PROVIDER_NOT_READY', + 'errorMessage' => 'Datadog-backed PHP feature flag evaluation is not wired in this weblog yet.', + 'providerState' => array( + 'ready' => false, + 'productionRuntime' => false, + ), +)); diff --git a/utils/build/docker/php/common/rewrite-rules.conf b/utils/build/docker/php/common/rewrite-rules.conf index 1191ca1fc4c..e55c8981dde 100644 --- a/utils/build/docker/php/common/rewrite-rules.conf +++ b/utils/build/docker/php/common/rewrite-rules.conf @@ -34,6 +34,7 @@ RewriteRule "^/trace/mongo$" "/trace_mongo/" RewriteRule "^/e2e_otel_span$" "/e2e_otel_span/" RewriteRule "^/e2e_single_span$" "/e2e_single_span/" RewriteRule "^/crashme$" "/crashme/" +RewriteRule "^/ffe$" "/ffe/" RewriteRule "^/exceptionreplay/(.*)$" "/debugger/exceptionreplay/$1" [QSA] RewriteRule "^/llm$" "/llm/" RewriteRule "^/stats-unique$" "/stats-unique/" From b1a11052309a025e994853c7e820d641154f74e8 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 22 May 2026 14:56:57 -0400 Subject: [PATCH 2/5] Enable PHP FFE parametric system tests --- manifests/php.yml | 6 +- tests/ffe/README.md | 43 ++++-- .../test_ffe/test_dynamic_evaluation.py | 2 +- utils/build/docker/php/common/ffe.php | 21 ++- utils/build/docker/php/parametric/server.php | 134 ++++++++++++++++++ 5 files changed, 186 insertions(+), 20 deletions(-) diff --git a/manifests/php.yml b/manifests/php.yml index 933414f0be1..e4c3d175ef1 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -605,7 +605,7 @@ manifest: component_version: <1.12.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) - tests/ffe/test_dynamic_evaluation.py: missing_feature (PHP /ffe endpoint is scaffolded; native evaluator and Remote Config lifecycle are not wired yet) + tests/ffe/test_dynamic_evaluation.py: missing_feature (PHP M1 dynamic evaluation is locally validated through the PHP 8.2 parametric path; weblog activation remains deferred) tests/ffe/test_exposures.py: missing_feature (PHP exposure buffering and native flush lifecycle are not wired yet) tests/ffe/test_flag_eval_metrics.py: missing_feature (PHP feature_flag.evaluations metric emission is not wired yet) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) @@ -730,7 +730,7 @@ manifest: tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV1_ServiceTargets::test_not_match_service_target: missing_feature tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2: '>=1.16.0' tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2::test_tracing_client_tracing_tags: missing_feature - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.20.0-dev tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_invalid: - declaration: missing_feature (Need to remove b3=b3multi alias) component_version: <1.16.0 @@ -865,7 +865,7 @@ manifest: tests/parametric/test_parametric_endpoints.py::Test_Parametric_DDSpan_Start: v1.13.0+4663b2fa7c20c6920f347d059b57dc2a419cb7f7 tests/parametric/test_parametric_endpoints.py::Test_Parametric_DDTrace_Baggage: missing_feature (baggage is not supported) tests/parametric/test_parametric_endpoints.py::Test_Parametric_DDTrace_Current_Span: bug (APMAPI-778) # current span endpoint should return span and trace id of zero if no span is "active" - tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start: missing_feature + tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start: v1.20.0-dev tests/parametric/test_parametric_endpoints.py::Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported) tests/parametric/test_parametric_endpoints.py::Test_Parametric_Otel_Current_Span: bug (APMAPI-778) # otel current span endpoint should return a span and trace id of zero if no span is "active" tests/parametric/test_parametric_endpoints.py::Test_Parametric_Write_Log: missing_feature diff --git a/tests/ffe/README.md b/tests/ffe/README.md index bc6aa2100b7..8675ba8a059 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -18,31 +18,44 @@ This directory contains system tests for the Feature Flags & Experimentation (FF ## PHP Local Runbook -PHP FFE system tests are scaffolded but not enabled in `manifests/php.yml` yet. Keep -them marked `missing_feature` until a local run passes against a real -tracer-backed PHP weblog. +PHP Milestone 1 is evaluation-only. Dynamic evaluation is enabled in +`manifests/php.yml` for the locally validated PHP 8.2 parametric path. Weblog +FFE activation remains deferred. Exposure and flag-evaluation metric tests +remain `missing_feature`. -Build the PHP weblog images: +The PHP parametric container consumes a custom tracer package from `binaries/`. +Build the package from `dd-trace-php` with Bob's debug-artifact helper: ```bash -./build.sh php +cd /path/to/dd-trace-php +./tooling/bin/build-debug-artifact gnu-aarch64-8.2-nts /path/to/system-tests/binaries ``` -Run the FFE scenario for PHP: +Run the validated dynamic evaluation tests for PHP: ```bash -./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION --library php -k "Test_FFE" +TEST_LIBRARY=php \ + ./run.sh PARAMETRIC tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation \ + -F tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation -vv ``` -The PHP weblog exposes `POST /ffe` from -`utils/build/docker/php/common/ffe.php`. The endpoint accepts the canonical -evaluation case fields used by the fixture corpus: `flag`, `variationType`, -`defaultValue`, `targetingKey`, and `attributes`. +Run the FFE parametric endpoint smoke: -Until the native PHP evaluator bridge and Remote Config lifecycle are wired, the -endpoint returns the provided default value with `reason=ERROR` and -`errorCode=PROVIDER_NOT_READY`. Once the PHP client is available in the weblog, -the endpoint will call `DDTrace\FeatureFlags\Client` for the same request shape. +```bash +TEST_LIBRARY=php \ + ./run.sh PARAMETRIC tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start \ + -F tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start -vv +``` + +The PHP parametric server exposes `POST /ffe/start` and `POST /ffe/evaluate` +from `utils/build/docker/php/parametric/server.php`. The evaluation endpoint +accepts the canonical evaluation case fields used by the fixture corpus: +`flag`, `variationType`, `defaultValue`, `targetingKey`, and `attributes`. + +The endpoint calls `DDTrace\FeatureFlags\Client`, which reads only from the +tracer Remote Config lifecycle and the native `libdatadog` evaluator. M1 +transitional warnings are suppressed inside the endpoint so responses stay valid +JSON for system-tests. --- diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index fe57f6075a5..849fa9bf399 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -154,7 +154,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI assert actual_value == expected_result, ( f"Test case {i} in {test_case_file} failed: " f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={actual_value}" + f"expected={expected_result}, actual={actual_value}, result={result}" ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) diff --git a/utils/build/docker/php/common/ffe.php b/utils/build/docker/php/common/ffe.php index 1c1415e49ec..a55e2fb8e5a 100644 --- a/utils/build/docker/php/common/ffe.php +++ b/utils/build/docker/php/common/ffe.php @@ -104,6 +104,25 @@ function dd_ffe_evaluate_with_client($flagKey, $variationType, $defaultValue, $t } } +function dd_ffe_warning_handler($severity, $message) +{ + if ($severity === E_USER_WARNING && strpos($message, 'Datadog-backed PHP feature flag evaluation') !== false) { + return true; + } + + return false; +} + +function dd_ffe_evaluate($flagKey, $variationType, $defaultValue, $targetingKey, array $attributes) +{ + set_error_handler('dd_ffe_warning_handler'); + try { + return dd_ffe_evaluate_with_client($flagKey, $variationType, $defaultValue, $targetingKey, $attributes); + } finally { + restore_error_handler(); + } +} + function dd_ffe_details_payload($details) { $payload = array( @@ -152,7 +171,7 @@ function dd_ffe_details_payload($details) : array(); try { - $details = dd_ffe_evaluate_with_client($flagKey, $variationType, $defaultValue, $targetingKey, $attributes); + $details = dd_ffe_evaluate($flagKey, $variationType, $defaultValue, $targetingKey, $attributes); if ($details !== null) { dd_ffe_json_response(200, dd_ffe_details_payload($details)); return; diff --git a/utils/build/docker/php/parametric/server.php b/utils/build/docker/php/parametric/server.php index ed38a766aaa..7e124b7ef54 100644 --- a/utils/build/docker/php/parametric/server.php +++ b/utils/build/docker/php/parametric/server.php @@ -51,6 +51,87 @@ function arg($req, $arg) { return ($buffer[$req] ??= json_decode($req->getBody()->buffer(), true))[$arg] ?? null; } +function ffeNormalizedVariationType($variationType) { + return strtoupper(str_replace('-', '_', (string) $variationType)); +} + +function ffeNormalizeDefaultValue($defaultValue, $variationType) { + switch (ffeNormalizedVariationType($variationType)) { + case 'BOOLEAN': + return is_bool($defaultValue) ? $defaultValue : (bool) $defaultValue; + case 'STRING': + return is_string($defaultValue) ? $defaultValue : (string) $defaultValue; + case 'INTEGER': + return is_int($defaultValue) ? $defaultValue : (int) $defaultValue; + case 'NUMERIC': + case 'FLOAT': + case 'DOUBLE': + return is_int($defaultValue) || is_float($defaultValue) ? $defaultValue : (float) $defaultValue; + case 'JSON': + case 'OBJECT': + return is_array($defaultValue) ? $defaultValue : []; + default: + return $defaultValue; + } +} + +function ffeScalarAttributes($attributes) { + $normalized = []; + if (!is_array($attributes)) { + return $normalized; + } + + foreach ($attributes as $key => $value) { + if (is_bool($value) || is_int($value) || is_float($value) || is_string($value)) { + $normalized[(string) $key] = $value; + } + } + + return $normalized; +} + +function ffeWarningHandler($severity, $message) { + if ($severity === E_USER_WARNING && strpos($message, 'Datadog-backed PHP feature flag evaluation') !== false) { + return true; + } + + return false; +} + +function ffeEvaluate($client, $flagKey, $variationType, $defaultValue, $targetingKey, array $attributes) { + $context = [ + 'targetingKey' => $targetingKey, + 'attributes' => $attributes, + ]; + + switch (ffeNormalizedVariationType($variationType)) { + case 'BOOLEAN': + return $client->getBooleanDetails($flagKey, $defaultValue, $context); + case 'STRING': + return $client->getStringDetails($flagKey, $defaultValue, $context); + case 'INTEGER': + return $client->getIntegerDetails($flagKey, $defaultValue, $context); + case 'NUMERIC': + case 'FLOAT': + case 'DOUBLE': + return $client->getFloatDetails($flagKey, $defaultValue, $context); + case 'JSON': + case 'OBJECT': + return $client->getObjectDetails($flagKey, $defaultValue, $context); + default: + throw new InvalidArgumentException('Unsupported variationType: ' . (string) $variationType); + } +} + +function ffeResponseValue($value, $variationType) { + $type = ffeNormalizedVariationType($variationType); + if (($type === 'JSON' || $type === 'OBJECT') && $value === []) { + return (object) []; + } + + return $value; +} + // Source: https://magp.ie/2015/09/30/convert-large-integer-to-hexadecimal-without-php-math-extension/ function convertBase16ToBase10($numString) { @@ -105,6 +186,8 @@ function remappedSpanKind($spanKind) { $closed_spans = $spans = []; /** @var Span[] $otelSpans */ $otelSpans = []; +/** @var ?\DDTrace\FeatureFlags\Client $ffeClient */ +$ffeClient = null; /** @var ScopeInterface[] $scopes */ $scopes = []; /** @var ?\DDTrace\SpanData $span */ @@ -487,6 +570,57 @@ function remappedSpanKind($spanKind) { return jsonResponse([]); })); +$router->addRoute('POST', '/ffe/start', new ClosureRequestHandler(function () use (&$ffeClient) { + if (!class_exists('\\DDTrace\\FeatureFlags\\Client')) { + return new Response(status: 500, headers: ['content-type' => 'application/json'], body: json_encode([ + 'error' => 'DDTrace\\FeatureFlags\\Client is not available', + ])); + } + + set_error_handler('ffeWarningHandler'); + try { + $ffeClient = \DDTrace\FeatureFlags\Client::create(); + } finally { + restore_error_handler(); + } + + return jsonResponse([]); +})); +$router->addRoute('POST', '/ffe/evaluate', new ClosureRequestHandler(function (Request $req) use (&$ffeClient) { + if ($ffeClient === null) { + return new Response(status: 500, headers: ['content-type' => 'application/json'], body: json_encode([ + 'error' => 'FFE client is not initialized', + ])); + } + + $flagKey = arg($req, 'flag'); + $variationType = arg($req, 'variationType'); + $defaultValue = ffeNormalizeDefaultValue(arg($req, 'defaultValue'), $variationType); + $targetingKey = arg($req, 'targetingKey'); + $attributes = ffeScalarAttributes(arg($req, 'attributes') ?? []); + + set_error_handler('ffeWarningHandler'); + try { + $details = ffeEvaluate($ffeClient, $flagKey, $variationType, $defaultValue, $targetingKey, $attributes); + + return jsonResponse([ + 'value' => ffeResponseValue($details->getValue(), $variationType), + 'reason' => $details->getReason(), + 'variant' => $details->getVariant(), + 'errorCode' => $details->getErrorCode(), + 'errorMessage' => $details->getErrorMessage(), + ]); + } catch (Throwable $exception) { + return jsonResponse([ + 'value' => $defaultValue, + 'reason' => 'ERROR', + 'errorCode' => 'PROVIDER_NOT_READY', + 'errorMessage' => $exception->getMessage(), + ]); + } finally { + restore_error_handler(); + } +})); $router->addRoute('POST', '/trace/otel/record_exception', new ClosureRequestHandler(function (Request $req) use (&$otelSpans) { $spanId = arg($req, 'span_id'); $message = arg($req, 'message'); From fe659eb433526db948f8da877c73a0050e83956c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 22 May 2026 15:26:13 -0400 Subject: [PATCH 3/5] docs: document PHP FFE validation --- tests/ffe/README.md | 10 +++++ tests/parametric/test_ffe/AGENTS.md | 60 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/parametric/test_ffe/AGENTS.md diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 8675ba8a059..0a169376611 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -31,6 +31,16 @@ cd /path/to/dd-trace-php ./tooling/bin/build-debug-artifact gnu-aarch64-8.2-nts /path/to/system-tests/binaries ``` +On macOS with Colima, run the same command with the Docker socket exported: + +```bash +DOCKER_HOST=unix://$HOME/.colima/default/docker.sock \ + ./tooling/bin/build-debug-artifact gnu-aarch64-8.2-nts /path/to/system-tests/binaries +``` + +See `tests/parametric/test_ffe/AGENTS.md` for the focused PHP M1 validation +commands and FFE-specific invariants. + Run the validated dynamic evaluation tests for PHP: ```bash diff --git a/tests/parametric/test_ffe/AGENTS.md b/tests/parametric/test_ffe/AGENTS.md new file mode 100644 index 00000000000..01b79b44bdb --- /dev/null +++ b/tests/parametric/test_ffe/AGENTS.md @@ -0,0 +1,60 @@ +# FFE Parametric Test Notes + +Follow the repository root `AGENTS.md` first. These notes are specific to +Feature Flags and Experimentation parametric tests. + +## PHP macOS Validation + +PHP M1 validation uses the parametric app, not the weblog. Build a local +`dd-trace-php` artifact into this repo before running PHP tests: + +```bash +DD_TRACE_PHP=/Users/leo.romanovsky/go/src/github.com/DataDog/dd-trace-php-ffe-runtime-first +SYSTEM_TESTS=/Users/leo.romanovsky/go/src/github.com/DataDog/system-tests-pr-g-php-ffe-scaffold + +cd "$DD_TRACE_PHP" +DOCKER_HOST=unix://$HOME/.colima/default/docker.sock \ +./tooling/bin/build-debug-artifact \ + gnu-aarch64-8.2-nts \ + "$SYSTEM_TESTS/binaries" +``` + +Use `gnu-aarch64-8.2-nts` for the PHP 8.2 parametric path on Apple Silicon. +Docker Desktop users can usually omit `DOCKER_HOST`; Colima users normally need +the socket path above. + +Run the focused PHP validator: + +```bash +cd "$SYSTEM_TESTS" +TEST_LIBRARY=php \ + ./run.sh PARAMETRIC \ + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation \ + -F tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation \ + -vv +``` + +Run the endpoint smoke: + +```bash +TEST_LIBRARY=php \ + ./run.sh PARAMETRIC \ + tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start \ + -F tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start \ + -vv +``` + +## Test Invariants + +- PHP evaluation tests must use the live Remote Config flow through + `_set_and_wait_ffe_rc`; do not bypass RC with a PHP fixture loader. +- The PHP endpoint must call `DDTrace\FeatureFlags\Client`, which is backed by + the tracer Remote Config lifecycle and the native `libdatadog` evaluator. +- Preserve empty targeting keys. `targetingKey: ""` is valid and must not be + collapsed to missing or `null`. +- Preserve empty JSON object responses as `{}`, not `[]`. +- Reason assertions are intentionally narrow for M1. Keep value assertions + canonical and avoid broadening reason checks until libdatadog/system-test + reason semantics are settled. +- Do not enable PHP weblog FFE, exposures, or evaluation metrics in this M1 + parametric change. Those are later milestones. From 4a955e525fda79bc6a9723d3be30a82434dec824 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 22 May 2026 18:44:53 -0400 Subject: [PATCH 4/5] Add PHP FFE telemetry parametric validation --- manifests/php.yml | 2 + .../test_ffe/test_dynamic_evaluation.py | 163 ++++++++++++++++++ utils/build/docker/php/parametric/server.php | 31 ++++ .../_test_clients/_test_client_parametric.py | 8 + 4 files changed, 204 insertions(+) diff --git a/manifests/php.yml b/manifests/php.yml index e4c3d175ef1..6b395d785ac 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -731,6 +731,8 @@ manifest: tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2: '>=1.16.0' tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2::test_tracing_client_tracing_tags: missing_feature tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.20.0-dev + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Parametric_Evaluation_Metrics: v1.20.0-dev + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Parametric_Exposures: v1.20.0-dev tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_invalid: - declaration: missing_feature (Need to remove b3=b3multi alias) component_version: <1.16.0 diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 849fa9bf399..92a197680e2 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -1,5 +1,6 @@ """Test FFE (Feature Flags & Experimentation) functionality via parametric tests.""" +import base64 import json import pytest from pathlib import Path @@ -51,6 +52,11 @@ def _get_test_case_files() -> list[str]: "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", } +OTLP_METRICS_ENVVARS = { + **DEFAULT_ENVVARS, + "DD_METRICS_OTEL_ENABLED": "true", +} + def _set_and_wait_ffe_rc( test_agent: TestAgentAPI, ufc_data: dict[str, Any], config_id: str | None = None @@ -79,6 +85,80 @@ def _set_and_wait_ffe_rc( return test_agent.wait_for_rc_apply_state(RC_PRODUCT, state=RemoteConfigApplyState.ACKNOWLEDGED, clear=True) +def _make_logging_ufc_fixture(flag_key: str, *, do_log: bool = True) -> dict[str, Any]: + return { + "createdAt": "2024-04-17T19:40:53.716Z", + "format": "SERVER", + "environment": {"name": "Test"}, + "flags": { + flag_key: { + "key": flag_key, + "enabled": True, + "variationType": "STRING", + "variations": { + "on": {"key": "on", "value": "on-value"}, + "off": {"key": "off", "value": "off-value"}, + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [{"variationKey": "on", "shards": []}], + "doLog": do_log, + } + ], + } + }, + } + + +def _decode_agent_json_body(encoded_body: str) -> dict[str, Any]: + decoded = base64.b64decode(encoded_body) + return json.loads(decoded.decode("utf-8")) + + +def _find_exposure(test_agent: TestAgentAPI, flag_key: str, targeting_key: str) -> dict[str, Any] | None: + for request in test_agent.requests(): + if not request["url"].endswith("/evp_proxy/v2/api/v2/exposures"): + continue + + payload = _decode_agent_json_body(request["body"]) + for exposure in payload.get("exposures", []): + if ( + exposure.get("flag", {}).get("key") == flag_key + and exposure.get("subject", {}).get("id") == targeting_key + ): + return exposure + + return None + + +def _find_metric_data_point(metrics: list[dict[str, Any]], attributes: dict[str, str]) -> dict[str, Any] | None: + for export in metrics: + for resource_metric in export.get("resource_metrics", []): + for scope_metric in resource_metric.get("scope_metrics", []): + for metric in scope_metric.get("metrics", []): + if metric.get("name") != "feature_flag.evaluations": + continue + + for data_point in metric.get("sum", {}).get("data_points", []): + data_point_attributes = { + item["key"]: item["value"]["string_value"] for item in data_point.get("attributes", []) + } + if attributes == data_point_attributes: + return data_point + + return None + + +def _data_point_value(data_point: dict[str, Any]) -> int | float | None: + if "as_int" in data_point: + return int(data_point["as_int"]) + if "as_double" in data_point: + return data_point["as_double"] + return None + + @scenarios.parametric @features.feature_flags_dynamic_evaluation class Test_Feature_Flag_Dynamic_Evaluation: @@ -186,3 +266,86 @@ def test_ffe_of7_empty_targeting_key(self, test_agent: TestAgentAPI, test_librar assert result.get("value") == "on-value", ( f"OF.7 failed: empty targeting key should return 'on-value', got '{result.get('value')}'" ) + + +@scenarios.parametric +@features.feature_flags_exposures +class Test_Feature_Flag_Parametric_Exposures: + """Test PHP FFE exposure emission through the parametric app.""" + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_php_ffe_exposure_event(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + if context.library.name != "php": + pytest.skip("Parametric FFE exposure validation is currently PHP-specific") + + flag_key = "php-parametric-exposure-flag" + targeting_key = "php-parametric-user" + + _set_and_wait_ffe_rc(test_agent, _make_logging_ufc_fixture(flag_key), "php-parametric-exposure") + + success = test_library.ffe_start() + assert success, "Failed to start FFE provider" + + result = test_library.ffe_evaluate( + flag=flag_key, + variation_type="STRING", + default_value="default", + targeting_key=targeting_key, + attributes={"plan": "pro"}, + ) + + assert result.get("value") == "on-value" + assert result.get("variant") == "on" + + flush_result = test_library.ffe_flush() + assert flush_result.get("flushed"), f"Expected FFE exposure flush, got {flush_result}" + + exposure = _find_exposure(test_agent, flag_key, targeting_key) + assert exposure is not None, f"Expected exposure event for flag '{flag_key}' and subject '{targeting_key}'" + assert exposure.get("allocation", {}).get("key") == "default-allocation" + assert exposure.get("variant", {}).get("key") == "on" + assert exposure.get("subject", {}).get("attributes", {}).get("plan") == "pro" + + +@scenarios.parametric +@features.feature_flags_eval_metrics +class Test_Feature_Flag_Parametric_Evaluation_Metrics: + """Test PHP FFE evaluation metric emission through the parametric app.""" + + @parametrize("library_env", [{**OTLP_METRICS_ENVVARS}]) + def test_php_ffe_evaluation_metric(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + if context.library.name != "php": + pytest.skip("Parametric FFE evaluation metric validation is currently PHP-specific") + + flag_key = "php-parametric-metric-flag" + + _set_and_wait_ffe_rc(test_agent, _make_logging_ufc_fixture(flag_key), "php-parametric-metric") + + success = test_library.ffe_start() + assert success, "Failed to start FFE provider" + + for _ in range(3): + result = test_library.ffe_evaluate( + flag=flag_key, + variation_type="STRING", + default_value="default", + targeting_key="php-parametric-user", + attributes={}, + ) + assert result.get("value") == "on-value" + assert result.get("variant") == "on" + + flush_result = test_library.ffe_flush() + assert flush_result.get("flushed"), f"Expected FFE metric flush, got {flush_result}" + + data_point = _find_metric_data_point( + test_agent.wait_for_num_otlp_metrics(num=1), + { + "feature_flag.key": flag_key, + "feature_flag.result.variant": "on", + "feature_flag.result.reason": "static", + "feature_flag.result.allocation_key": "default-allocation", + }, + ) + assert data_point is not None, f"Expected feature_flag.evaluations metric for flag '{flag_key}'" + assert _data_point_value(data_point) == 3 diff --git a/utils/build/docker/php/parametric/server.php b/utils/build/docker/php/parametric/server.php index 7e124b7ef54..30868988e89 100644 --- a/utils/build/docker/php/parametric/server.php +++ b/utils/build/docker/php/parametric/server.php @@ -132,6 +132,32 @@ function ffeResponseValue($value, $variationType) { return $value; } +function ffeFlushDefaultWriters() { + $flushed = []; + foreach ([ + '\\DDTrace\\FeatureFlags\\Internal\\Exposure\\ExposureHook', + '\\DDTrace\\FeatureFlags\\Internal\\Metric\\EvaluationMetricHook', + ] as $class) { + if (!class_exists($class)) { + continue; + } + + $reflection = new ReflectionClass($class); + if (!$reflection->hasProperty('defaultWriter')) { + continue; + } + + $property = $reflection->getProperty('defaultWriter'); + $property->setAccessible(true); + $writer = $property->getValue(); + if (is_object($writer) && method_exists($writer, 'flush')) { + $flushed[$class] = (bool) $writer->flush(); + } + } + + return $flushed; +} + // Source: https://magp.ie/2015/09/30/convert-large-integer-to-hexadecimal-without-php-math-extension/ function convertBase16ToBase10($numString) { @@ -621,6 +647,11 @@ function remappedSpanKind($spanKind) { restore_error_handler(); } })); +$router->addRoute('POST', '/ffe/flush', new ClosureRequestHandler(function () { + return jsonResponse([ + 'flushed' => ffeFlushDefaultWriters(), + ]); +})); $router->addRoute('POST', '/trace/otel/record_exception', new ClosureRequestHandler(function (Request $req) use (&$otelSpans) { $spanId = arg($req, 'span_id'); $message = arg($req, 'message'); diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index 816c0dab236..92ac37ce4ea 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -777,6 +777,10 @@ def ffe_evaluate( ) return resp.json() + def ffe_flush(self) -> dict: + """Flush FFE telemetry buffered by the parametric app.""" + return self._session.post(self._url("/ffe/flush"), json={}).json() + def otel_get_meter( self, name: str, version: str | None = None, schema_url: str | None = None, attributes: dict | None = None ) -> None: @@ -1181,6 +1185,10 @@ def ffe_evaluate( attributes=attributes, ) + def ffe_flush(self) -> dict: + """Flush FFE telemetry buffered by the parametric app.""" + return self._client.ffe_flush() + def llmobs_trace( self, trace_structure_request: SpanRequest | LlmObsAnnotationContextRequest, *, raise_on_error: bool = True ) -> dict | str | None: From ccc1276f869bdd7b1501816dd6e5b5bdc604cff9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 23 May 2026 20:05:21 -0400 Subject: [PATCH 5/5] parametric: inject OTEL_EXPORTER_OTLP_METRICS_ENDPOINT into SUT containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the existing DD_AGENT_HOST / DD_TRACE_AGENT_URL injection so OTel-aware exporters in the system-under-test can reach the test agent's OTLP/HTTP listener without each tracer needing to special-case the parametric framework. Concretely: the PHP FFE OTLP-metrics path (DataDog/dd-trace-php#3911) resolves its endpoint per the OpenTelemetry spec — OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, then OTEL_EXPORTER_OTLP_ENDPOINT, then localhost:4318. Without this injection, the parametric metric test had no way to reach the test agent. --- .../docker_fixtures/_test_clients/_test_client_parametric.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index 92ac37ce4ea..92bf45d2f20 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -57,6 +57,11 @@ def get_apm_library( "DD_TRACE_AGENT_PORT": str(test_agent.container_port), "APM_TEST_CLIENT_SERVER_PORT": str(container_port), "DD_TRACE_OTEL_ENABLED": "true", + # Point any OTel-aware exporters in the SUT at the test agent's + # OTLP/HTTP listener. Mirrors the DD_AGENT_HOST injection above + # so OTel-enabled tracers don't have to special-case the + # parametric framework. + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://{test_agent.container_name}:4318/v1/metrics", } )