From a136a3433ca2716461dbacb7702d13e29b672d5c Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 12 Jun 2026 15:33:55 +0200 Subject: [PATCH] =?UTF-8?q?feat(=F0=9F=A4=96):=20correct=20per-buffer=20YU?= =?UTF-8?q?V=20conversion=20for=20Android=20camera=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three improvements to the Android camera-frame path, found while reviewing the VisionCamera integration: 1. Expose GPUExternalTexture.yuvToRgbMatrix (non-spec extension). Dawn samples Android external-format (opaque YCbCr) buffers through a Vulkan conversion hard-coded to RGB_IDENTITY (SamplerVk.cpp, see crbug.com/497675620), so shaders receive raw [Y, Cb, Cr] and previously had to guess the conversion. The example hard-coded BT.709 narrow-range, but Android camera streams are usually BT.601, and range varies by device, which skews colors. The driver reports the correct model and range per buffer (suggestedYcbcrModel/Range); Dawn captures them at import time and we now read them back via the shared memory's AHardwareBuffer properties and derive the exact 3x4 conversion matrix. On iOS and for RGBA surfaces the matrix is the identity passthrough, so shaders can apply it unconditionally. The VisionCamera example now uploads the matrix as uniforms instead of hard-coding coefficients. 2. Fail fast on CPU-only AHardwareBuffers. Dawn only grants TextureBinding when the AHB was allocated with AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE; without it, import fails deep inside Dawn with an opaque validation error. wrapNativeBuffer now checks the usage bits and throws an actionable error (pointing at vision-camera's pixelFormat: 'native'). 3. Remove the unreachable "defined biplanar format" branch on Android. Dawn maps every YUV AHB format (Y8Cb8Cr8_420, P010, vendor formats) to OpaqueYCbCrAndroid (AHBFunctions.cpp), so the R8BG8Biplanar420Unorm plane-splitting path and its BT.709 matrix could never run. Also folds the Android both-axes flip in the example into the rotation passed to importExternalTexture (a double mirror is a 180° rotation), removing the per-fragment flip branches from the WGSL prelude. Co-Authored-By: Claude Fable 5 --- .../example/src/VisionCamera/VisionCamera.tsx | 64 +++++-- apps/example/src/VisionCamera/blurShaders.ts | 14 +- apps/example/src/VisionCamera/shaders.ts | 65 +++---- .../android/cpp/AndroidPlatformContext.h | 33 +++- .../cpp/rnwgpu/api/GPUExternalTexture.cpp | 179 ++++++++++-------- .../cpp/rnwgpu/api/GPUExternalTexture.h | 23 ++- packages/webgpu/src/index.tsx | 9 + 7 files changed, 235 insertions(+), 152 deletions(-) diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index c2571c4f8..e35f11863 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -167,6 +167,8 @@ const CameraView = () => { // This is what lets us call the interop factory off RNWebGPU, where the native // platform context already lives, instead of off `device`. const rnwgpu = RNWebGPU; + // Captured as a plain bool so the worklet doesn't reach for Platform. + const isAndroid = Platform.OS === "android"; const devices = useCameraDevices(); // Pick back camera if available, otherwise front, otherwise anything. The // iOS simulator returns an empty list since there are no cameras, in which @@ -254,7 +256,8 @@ const CameraView = () => { minFilter: "linear", }); const uniformBuffer = device.createBuffer({ - size: 32, + // vec4f params + vec4u modes + 3x vec4f yuvToRgbMatrix rows. + size: 80, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); @@ -282,7 +285,8 @@ const CameraView = () => { primitive: { topology: "triangle-list" }, }); const prepassUniformBuffer = device.createBuffer({ - size: 16, + // 2x vec2f sizes + 3x vec4f yuvToRgbMatrix rows. + size: 64, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); @@ -469,8 +473,11 @@ const CameraView = () => { // Orientation. The sensor buffer is landscape; frame.orientation is // the rotation needed to bring it upright. We hand that to Dawn via // importExternalTexture's `rotation`, which de-rotates the frame into - // the portrait canvas. The residual vertical flip (Android buffer - // Y-origin) and YUV->RGB are corrected in the shader (CAMERA_PRELUDE). + // the portrait canvas. On Android the buffer additionally arrives + // mirrored on both axes relative to the canvas (buffer Y-origin), + // which is the same as an extra 180° rotation, folded in below. + // YUV->RGB is applied in the shader via the + // GPUExternalTexture.yuvToRgbMatrix uniform (see CAMERA_PRELUDE). let rotationDeg: 0 | 90 | 180 | 270 = 0; if (frame.orientation === "right") { rotationDeg = 90; @@ -479,6 +486,9 @@ const CameraView = () => { } else if (frame.orientation === "left") { rotationDeg = 270; } + if (isAndroid) { + rotationDeg = ((rotationDeg + 180) % 360) as 0 | 90 | 180 | 270; + } // A 90/270 rotation swaps the displayed width & height, so cover-fit // uses the post-rotation dimensions. const rotated = rotationDeg === 90 || rotationDeg === 270; @@ -496,21 +506,6 @@ const CameraView = () => { } else { sy = frameAR / canvasAR; } - // 32-byte uniform: vec4f params + vec4u modes. Built on a single - // ArrayBuffer so the f32/u32 halves go up in one writeBuffer call. - const uniformData = new ArrayBuffer(32); - const uniformF32 = new Float32Array(uniformData); - const uniformU32 = new Uint32Array(uniformData); - uniformF32[0] = sx; - uniformF32[1] = sy; - uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0; - uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0; - uniformU32[4] = modes.effect; - uniformU32[5] = modes.tint; - uniformU32[6] = modes.vignette; - uniformU32[7] = blurMode; - device.queue.writeBuffer(uniformBuffer, 0, uniformData); - let externalTex; try { externalTex = device.importExternalTexture({ @@ -525,6 +520,29 @@ const CameraView = () => { ); throw e; } + // Per-buffer YUV->RGB conversion for the shader: the driver-derived + // matrix on the Android opaque-YCbCr path, an identity passthrough + // on iOS / RGBA surfaces (see CAMERA_PRELUDE). + const yuvMatrix = externalTex.yuvToRgbMatrix; + + // 80-byte uniform: vec4f params + vec4u modes + the three vec4f + // yuvToRgbMatrix rows. Built on a single ArrayBuffer so the f32/u32 + // halves go up in one writeBuffer call. + const uniformData = new ArrayBuffer(80); + const uniformF32 = new Float32Array(uniformData); + const uniformU32 = new Uint32Array(uniformData); + uniformF32[0] = sx; + uniformF32[1] = sy; + uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0; + uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0; + uniformU32[4] = modes.effect; + uniformU32[5] = modes.tint; + uniformU32[6] = modes.vignette; + uniformU32[7] = blurMode; + for (let i = 0; i < 12; i++) { + uniformF32[8 + i] = yuvMatrix[i] ?? 0; + } + device.queue.writeBuffer(uniformBuffer, 0, uniformData); const bindGroup = device.createBindGroup({ layout: pipeline.getBindGroupLayout(0), entries: [ @@ -542,7 +560,13 @@ const CameraView = () => { device.queue.writeBuffer( prepassUniformBuffer, 0, - new Float32Array([dispW, dispH, canvasWidth, canvasHeight]), + new Float32Array([ + dispW, + dispH, + canvasWidth, + canvasHeight, + ...yuvMatrix, + ]), ); const prepassBindGroup = device.createBindGroup({ layout: prepassPipeline.getBindGroupLayout(0), diff --git a/apps/example/src/VisionCamera/blurShaders.ts b/apps/example/src/VisionCamera/blurShaders.ts index eddbfb8d7..803fcde94 100644 --- a/apps/example/src/VisionCamera/blurShaders.ts +++ b/apps/example/src/VisionCamera/blurShaders.ts @@ -21,6 +21,10 @@ export const PREPASS_SHADER = /* wgsl */ ` struct PrepassUniforms { texSize: vec2f, canvasSize: vec2f, + // The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE). + yuv0: vec4f, + yuv1: vec4f, + yuv2: vec4f, }; @group(0) @binding(0) var srcTex: texture_external; @@ -63,14 +67,14 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { scale = vec2f(1.0, texAR / canvasAR); } let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * scale; - // cameraCoord (vertical flip) + cameraDecode (YUV->RGB) come from - // CAMERA_PRELUDE, prepended when this module is compiled. They are no-ops on - // iOS and handle the Android opaque-YUV case. + // cameraDecode (from CAMERA_PRELUDE, prepended when this module is + // compiled) applies GPUExternalTexture.yuvToRgbMatrix: the real YUV->RGB + // conversion on the Android opaque-YUV path, an identity passthrough on iOS. let c = cameraDecode(textureSampleBaseClampToEdge( srcTex, srcSampler, - cameraCoord(clamp(uv, vec2f(0.0), vec2f(1.0))), - )); + clamp(uv, vec2f(0.0), vec2f(1.0)), + ), u.yuv0, u.yuv1, u.yuv2); return vec4f(c.rgb, 1.0); } `; diff --git a/apps/example/src/VisionCamera/shaders.ts b/apps/example/src/VisionCamera/shaders.ts index e93520107..3d5cade38 100644 --- a/apps/example/src/VisionCamera/shaders.ts +++ b/apps/example/src/VisionCamera/shaders.ts @@ -1,10 +1,8 @@ -import { Platform } from "react-native"; - // Main camera-effects shader: samples the imported external texture (camera // frame) with cover-fit + optional chromatic aberration / pixelate, optionally // mixes in the pre-blurred backdrop, then applies effect / tint / vignette. -// `cameraCoord` / `cameraDecode` come from CAMERA_PRELUDE, which is prepended -// at shader-module creation time. +// `cameraDecode` comes from CAMERA_PRELUDE, which is prepended at +// shader-module creation time. export const SHADER = /* wgsl */ ` struct VsOut { @builtin(position) position: vec4f, @@ -22,6 +20,10 @@ struct Uniforms { // w: blurMode (0 off, 1 strong - blurred everywhere (prepass bakes // cover-fit), 2 overlay - blurred backdrop + sharp card) modes: vec4u, + // The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE). + yuv0: vec4f, + yuv1: vec4f, + yuv2: vec4f, }; @group(0) @binding(0) var srcTex: texture_external; @@ -56,7 +58,8 @@ fn snap(uv: vec2f, block: f32) -> vec2f { fn sampleExternal(uv: vec2f, block: f32) -> vec4f { return cameraDecode( - textureSampleBaseClampToEdge(srcTex, srcSampler, cameraCoord(snap(uv, block))), + textureSampleBaseClampToEdge(srcTex, srcSampler, snap(uv, block)), + u.yuv0, u.yuv1, u.yuv2, ); } @@ -177,43 +180,21 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { // YCbCr conversion that is hard-coded to RGB_IDENTITY (Dawn's // SamplerVk.cpp::GetYCbCrForTextureView; see crbug.com/497675620), so the // external sample comes back as raw [Y, Cb, Cr] on *every* device — this is by -// design, not a driver quirk — and we do the BT.709 YUV->RGB ourselves below. -// (Dawn's own SharedTextureMemoryOpaqueYCbCrAndroidForExternalTexture -// .NoopSampleY8Cb8Cr8AHB test asserts the same raw passthrough.) The frame also -// comes out mirrored on both axes relative to the canvas (Android buffer -// origin), so we flip X and Y. iOS goes through the native two-plane path, -// which already converts and orients, so this correction is Android-only. The -// prelude is prepended to every shader module that samples the camera (main -// pass + blur prepass). +// design, not a driver quirk. The correct conversion depends on the buffer: +// react-native-webgpu derives it from the driver's suggested YCbCr model and +// range (BT.601/709/2020, full/narrow) and exposes it as +// GPUExternalTexture.yuvToRgbMatrix; the worklet uploads its three rows as +// vec4f uniforms and cameraDecode applies them. On iOS (native two-plane path, +// already converted by Dawn) and for RGBA surfaces the matrix is the identity +// passthrough, so the decode is safe to apply unconditionally. The prelude is +// prepended to every shader module that samples the camera (main pass + blur +// prepass). export const CAMERA_PRELUDE = /* wgsl */ ` -const CAMERA_IS_YUV: bool = ${Platform.OS === "android"}; -const CAMERA_FLIP_X: bool = ${Platform.OS === "android"}; -const CAMERA_FLIP_Y: bool = ${Platform.OS === "android"}; - -fn cameraCoord(uv: vec2f) -> vec2f { - var c = uv; - if (CAMERA_FLIP_X) { - c.x = 1.0 - c.x; - } - if (CAMERA_FLIP_Y) { - c.y = 1.0 - c.y; - } - return c; -} - -// BT.709 limited-range YUV -> RGB. On the Android opaque path the sampled -// channels are always raw [Y, Cb, Cr] (Dawn forces an RGB_IDENTITY Vulkan -// conversion); a no-op passthrough on every other platform. -fn cameraDecode(c: vec4f) -> vec4f { - if (!CAMERA_IS_YUV) { - return c; - } - let y = c.r - 0.0627451; - let cb = c.g - 0.5; - let cr = c.b - 0.5; - let r = 1.164384 * y + 1.792741 * cr; - let g = 1.164384 * y - 0.213249 * cb - 0.532909 * cr; - let b = 1.164384 * y + 2.112402 * cb; - return vec4f(clamp(vec3f(r, g, b), vec3f(0.0), vec3f(1.0)), 1.0); +// Apply a 3x4 row-major [c.r, c.g, c.b, 1] -> R'G'B' matrix +// (GPUExternalTexture.yuvToRgbMatrix, see above). +fn cameraDecode(c: vec4f, m0: vec4f, m1: vec4f, m2: vec4f) -> vec4f { + let v = vec4f(c.rgb, 1.0); + let rgb = vec3f(dot(m0, v), dot(m1, v), dot(m2, v)); + return vec4f(clamp(rgb, vec3f(0.0), vec3f(1.0)), 1.0); } `; diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index 080b1bb5e..b20078669 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -241,8 +241,8 @@ class AndroidPlatformContext : public PlatformContext { const uint32_t stridePixels = actualDesc.stride; void *vaddr = nullptr; - int rc = AHardwareBuffer_lock(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, - -1, nullptr, &vaddr); + int rc = AHardwareBuffer_lock( + buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1, nullptr, &vaddr); if (rc != 0 || !vaddr) { AHardwareBuffer_release(buffer); throw std::runtime_error( @@ -302,6 +302,21 @@ class AndroidPlatformContext : public PlatformContext { AHardwareBuffer_Desc desc = {}; AHardwareBuffer_describe(buffer, &desc); + // Dawn derives the importable WebGPU usages from the AHB's usage bits; + // without GPU_SAMPLED_IMAGE the imported texture never gets + // TextureBinding and import fails deep inside Dawn with an opaque + // validation error. Surface the real cause here instead. (CameraX's + // default ImageReaders allocate CPU-only buffers; with + // react-native-vision-camera, use pixelFormat: 'native' which allocates + // GPU-sampleable buffers.) + if ((desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) == 0) { + throw std::runtime_error( + "wrapNativeBuffer: this AHardwareBuffer was allocated without " + "AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, so the GPU cannot sample " + "it. Camera/CPU pipelines must allocate frames with GPU usage (for " + "react-native-vision-camera, use pixelFormat: 'native')."); + } + AHardwareBuffer_acquire(buffer); VideoFrameHandle handle; @@ -312,13 +327,13 @@ class AndroidPlatformContext : public PlatformContext { // Dawn's OpaqueYCbCrAndroidForExternalTexture path. Single-plane RGBA AHBs // take the plain BGRA8 path (sampled as a regular 2D texture). switch (desc.format) { - case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420: - case AHARDWAREBUFFER_FORMAT_YCbCr_P010: - handle.pixelFormat = VideoPixelFormat::NV12; - break; - default: - handle.pixelFormat = VideoPixelFormat::BGRA8; - break; + case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420: + case AHARDWAREBUFFER_FORMAT_YCbCr_P010: + handle.pixelFormat = VideoPixelFormat::NV12; + break; + default: + handle.pixelFormat = VideoPixelFormat::BGRA8; + break; } handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); }; return handle; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp index 255cd6f80..44b2ea4b0 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp @@ -1,5 +1,6 @@ #include "GPUExternalTexture.h" +#include #include #include #include @@ -56,44 +57,76 @@ static const float kIdentityTransferParams[7] = { 0.0f, // F }; -// BT.709 limited-range YUV -> R'G'B' as a 3x4 row-major matrix mapping -// [Y, Cb, Cr, 1] to gamma-encoded R'G'B' (NOT linear; the sRGB decode in -// srcTransferFunctionParameters linearizes afterwards). Same values the Apple -// NV12 path computes from the CVPixelBuffer; used for Android buffers that -// arrive as a *defined* biplanar format (where we split the planes and convert -// ourselves) rather than an opaque external-format AHB. Camera streams are -// limited-range BT.709 in the overwhelming majority of cases; full-range / -// BT.601 would need different coefficients (refine from the buffer's suggested -// range if it matters). -[[maybe_unused]] static const float kBT709LimitedToRgb[12] = { - 1.164383f, 0.000000f, 1.792741f, -0.972945f, // - 1.164383f, -0.213249f, -0.532909f, 0.301517f, // - 1.164383f, 2.112402f, 0.000000f, -1.133402f, // +// Identity passthrough for GPUExternalTexture.yuvToRgbMatrix: the sampled +// texel is already RGB (Apple's biplanar path where Dawn converts in the +// sampler transform, RGBA surfaces, or an Android AHB whose driver reports +// RGB_IDENTITY). A 3x4 row-major matrix mapping [c.r, c.g, c.b, 1] to itself. +static constexpr std::array kYuvPassthroughMatrix = { + 1.0f, 0.0f, 0.0f, 0.0f, // + 0.0f, 1.0f, 0.0f, 0.0f, // + 0.0f, 0.0f, 1.0f, 0.0f, // }; -// True for the multi-planar Y + CbCr formats whose planes we can view as -// Plane0Only (luma) / Plane1Only (chroma) and convert with an explicit matrix. -// Excludes OpaqueYCbCrAndroid (external format, no plane views) and triplanar -// formats (would need a third plane). Only referenced on Android. -[[maybe_unused]] static bool isBiplanarYuvFormat(wgpu::TextureFormat format) { - switch (format) { - case wgpu::TextureFormat::R8BG8Biplanar420Unorm: - case wgpu::TextureFormat::R8BG8Biplanar422Unorm: - case wgpu::TextureFormat::R8BG8Biplanar444Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar420Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar422Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar444Unorm: - return true; +// Build a 3x4 row-major matrix mapping the *sampled* gamma-encoded +// [Y, Cb, Cr, 1] (each normalized to [0,1]) to gamma-encoded R'G'B', from the +// Vulkan-suggested YCbCr model + range of an Android external-format buffer. +// Needed because Dawn's opaque-YCbCr sampling path hard-codes an RGB_IDENTITY +// conversion (SamplerVk.cpp::GetYCbCrForTextureView, crbug.com/497675620), so +// the shader receives raw Y/Cb/Cr and must convert itself. +// The VkSamplerYcbcrModelConversion / VkSamplerYcbcrRange values are inlined +// to avoid a Vulkan header dependency in this cross-platform file. +[[maybe_unused]] static std::array +makeYuvToRgbMatrix(uint32_t vkModel, uint32_t vkRange) { + constexpr uint32_t kModelRgbIdentity = 0; // VK_..._RGB_IDENTITY + constexpr uint32_t kModel709 = 2; // VK_..._YCBCR_709 + constexpr uint32_t kModel601 = 3; // VK_..._YCBCR_601 + constexpr uint32_t kModel2020 = 4; // VK_..._YCBCR_2020 + constexpr uint32_t kRangeItuNarrow = 1; // VK_SAMPLER_YCBCR_RANGE_ITU_NARROW + + if (vkModel == kModelRgbIdentity) { + // The buffer content is RGB; nothing to convert. + return kYuvPassthroughMatrix; + } + float kr; + float kb; + switch (vkModel) { + case kModel709: + kr = 0.2126f; + kb = 0.0722f; + break; + case kModel2020: + kr = 0.2627f; + kb = 0.0593f; + break; + case kModel601: default: - return false; + // YCBCR_IDENTITY and unknown models: Android camera streams are BT.601 in + // practice, so that is the sane default. + kr = 0.299f; + kb = 0.114f; + break; } + const float kg = 1.0f - kr - kb; + const bool narrow = vkRange == kRangeItuNarrow; + const float yScale = narrow ? 255.0f / 219.0f : 1.0f; + const float cScale = narrow ? 255.0f / 224.0f : 1.0f; + const float yOffset = narrow ? 16.0f / 255.0f : 0.0f; + const float cOffset = 128.0f / 255.0f; + const float crR = 2.0f * (1.0f - kr) * cScale; + const float cbG = -2.0f * kb * (1.0f - kb) / kg * cScale; + const float crG = -2.0f * kr * (1.0f - kr) / kg * cScale; + const float cbB = 2.0f * (1.0f - kb) * cScale; + return { + yScale, 0.0f, crR, -yScale * yOffset - crR * cOffset, // + yScale, cbG, crG, -yScale * yOffset - (cbG + crG) * cOffset, // + yScale, cbB, 0.0f, -yScale * yOffset - cbB * cOffset, // + }; } // Map a rotation in degrees (0 / 90 / 180 / 270) to Dawn's enum. Anything that // isn't a clean multiple of 90 snaps to the nearest quadrant; Dawn only // supports those four steps for external textures. -static wgpu::ExternalTextureRotation -toExternalTextureRotation(double degrees) { +static wgpu::ExternalTextureRotation toExternalTextureRotation(double degrees) { int quadrant = static_cast(std::lround(degrees / 90.0)) & 3; switch (quadrant) { case 1: @@ -216,7 +249,11 @@ std::shared_ptr GPUExternalTexture::Create( return std::make_shared( std::move(external), std::move(memory), std::move(texture), - std::move(descriptor->source), std::move(label)); + std::move(descriptor->source), std::move(label), + // Dawn's Metal path applies the YUV->RGB conversion (from + // frame.yuvToRgbMatrix above) inside the sampling transform, so the + // sampled texel is already RGB. + kYuvPassthroughMatrix); #elif defined(__ANDROID__) // 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this // yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid @@ -257,32 +294,39 @@ std::shared_ptr GPUExternalTexture::Create( "GPUExternalTexture::Create(): BeginAccess failed"); } - // 4. Build the ExternalTextureDescriptor. There are two cases depending on - // how Dawn imported the AHB (see SharedTextureMemoryVk.cpp): - // - // a. *External* format (camera buffers whose layout has no Vulkan - // equivalent) -> OpaqueYCbCrAndroid, a single opaque plane. Sampling - // routes through a Vulkan SamplerYcbcrConversion whose model Dawn - // copies verbatim from the AHB's suggestedYcbcrModel. We pass a single - // plane + identity transfer and let that conversion (if any) run. NOTE: - // when the driver reports RGB_IDENTITY the sample comes back as raw - // Y/Cb/Cr; there is no public hook to override the model on this path. - // - // b. *Defined* biplanar format (e.g. R8BG8Biplanar420Unorm, exposed by the - // dawn-multi-planar-formats feature) -> we split Plane0Only (luma) / - // Plane1Only (chroma) and hand Dawn an explicit BT.709 matrix + sRGB - // transfer, exactly like the iOS NV12 path. This makes numPlanes == 2 - // so the matrix is actually applied (the single-plane branch in Dawn's - // Tint transform ignores yuvToRgbConversionMatrix). - // - // Either way we must pass non-null gamut/transfer arrays: - // ComputeExternalTextureParams dereferences them unconditionally - // (kIdentityTransferParams is defined at file scope). - const bool isBiplanar = frame.pixelFormat == VideoPixelFormat::NV12 && - isBiplanarYuvFormat(texture.GetFormat()); + // 4. Resolve the YUV->RGB conversion the *shader* must apply. Dawn imports + // every YUV / implementation-defined AHB format as OpaqueYCbCrAndroid + // (AHBFunctions.cpp::FormatFromAHardwareBufferFormat) and samples it + // through a Vulkan SamplerYcbcrConversion that is hard-coded to + // RGB_IDENTITY (SamplerVk.cpp::GetYCbCrForTextureView, + // crbug.com/497675620), so the sample always comes back as raw + // [Y, Cb, Cr]. The driver's *suggested* model + range are captured at + // import time though; read them back from the shared memory's properties + // and derive the correct conversion matrix, exposed to JS as + // GPUExternalTexture.yuvToRgbMatrix. Defined color formats (e.g. an RGBA + // AHB -> RGBA8Unorm) sample as RGB directly and get the passthrough. + std::array yuvToRgbMatrix = kYuvPassthroughMatrix; + if (texture.GetFormat() == wgpu::TextureFormat::OpaqueYCbCrAndroid) { + wgpu::SharedTextureMemoryAHardwareBufferProperties ahbProps{}; + wgpu::SharedTextureMemoryProperties props{}; + props.nextInChain = &ahbProps; + if (memory.GetProperties(&props)) { + yuvToRgbMatrix = makeYuvToRgbMatrix(ahbProps.yCbCrInfo.vkYCbCrModel, + ahbProps.yCbCrInfo.vkYCbCrRange); + } else { + // Fall back to the Android camera norm rather than passthrough. + yuvToRgbMatrix = makeYuvToRgbMatrix(/* YCBCR_601 */ 3, + /* ITU_NARROW */ 1); + } + } - wgpu::TextureView plane0; - wgpu::TextureView plane1; + // 5. Build the ExternalTextureDescriptor: a single (opaque or RGB) plane + // with identity transfer. The YUV->RGB conversion cannot run inside + // Dawn's external-texture transform here: the single-plane branch of the + // Tint transform ignores yuvToRgbConversionMatrix, which is why the + // matrix is surfaced to the shader instead. The gamut/transfer arrays + // must still be non-null: ComputeExternalTextureParams dereferences them + // unconditionally. wgpu::ExternalTextureDescriptor extDesc{}; if (!label.empty()) { extDesc.label = wgpu::StringView(label.c_str(), label.size()); @@ -291,24 +335,10 @@ std::shared_ptr GPUExternalTexture::Create( extDesc.cropSize = {frame.width, frame.height}; extDesc.apparentSize = {frame.width, frame.height}; extDesc.gamutConversionMatrix = kIdentityGamutMatrix; - if (isBiplanar) { - wgpu::TextureViewDescriptor v0{}; - v0.aspect = wgpu::TextureAspect::Plane0Only; - plane0 = texture.CreateView(&v0); - wgpu::TextureViewDescriptor v1{}; - v1.aspect = wgpu::TextureAspect::Plane1Only; - plane1 = texture.CreateView(&v1); - extDesc.plane0 = plane0; - extDesc.plane1 = plane1; - extDesc.yuvToRgbConversionMatrix = kBT709LimitedToRgb; - extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; - extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; - } else { - plane0 = texture.CreateView(); - extDesc.plane0 = plane0; - extDesc.srcTransferFunctionParameters = kIdentityTransferParams; - extDesc.dstTransferFunctionParameters = kIdentityTransferParams; - } + wgpu::TextureView plane0 = texture.CreateView(); + extDesc.plane0 = plane0; + extDesc.srcTransferFunctionParameters = kIdentityTransferParams; + extDesc.dstTransferFunctionParameters = kIdentityTransferParams; extDesc.mirrored = descriptor->mirrored.value_or(false); extDesc.rotation = toExternalTextureRotation(descriptor->rotation.value_or(0)); @@ -324,7 +354,8 @@ std::shared_ptr GPUExternalTexture::Create( return std::make_shared( std::move(external), std::move(memory), std::move(texture), - std::move(descriptor->source), std::move(label)); + std::move(descriptor->source), std::move(label), + std::move(yuvToRgbMatrix)); #else throw std::runtime_error( "GPUExternalTexture::Create(): not yet implemented on this " diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h index 217ab45d9..917909eb9 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h @@ -1,8 +1,10 @@ #pragma once +#include #include #include #include +#include #include "Unions.h" @@ -36,10 +38,12 @@ class GPUExternalTexture : public NativeObject { // the producer (e.g. AVPlayer) can reclaim the IOSurface. GPUExternalTexture(wgpu::ExternalTexture instance, wgpu::SharedTextureMemory memory, wgpu::Texture texture, - std::shared_ptr source, std::string label) + std::shared_ptr source, std::string label, + std::array yuvToRgbMatrix) : NativeObject(CLASS_NAME), _instance(std::move(instance)), _memory(std::move(memory)), _texture(std::move(texture)), - _source(std::move(source)), _label(std::move(label)) {} + _source(std::move(source)), _label(std::move(label)), + _yuvToRgbMatrix(yuvToRgbMatrix) {} ~GPUExternalTexture() override { destroy(); } @@ -69,11 +73,25 @@ class GPUExternalTexture : public NativeObject { _instance.SetLabel(_label.c_str()); } + // Non-spec extension. A 3x4 row-major matrix mapping the *sampled* texel + // [c.r, c.g, c.b, 1] to gamma-encoded R'G'B'. When Dawn's sampler already + // produces RGB (Apple's biplanar path, RGBA surfaces), this is the identity + // passthrough; on the Android opaque-YCbCr path the sample comes back as raw + // [Y, Cb, Cr] (Dawn hard-codes an RGB_IDENTITY Vulkan conversion, see + // crbug.com/497675620) and this matrix is derived from the driver's + // suggested YCbCr model + range for the buffer. Shaders can therefore apply + // it unconditionally after textureSampleBaseClampToEdge. + std::vector getYuvToRgbMatrix() { + return std::vector(_yuvToRgbMatrix.begin(), _yuvToRgbMatrix.end()); + } + static void definePrototype(jsi::Runtime &runtime, jsi::Object &prototype) { installGetter(runtime, prototype, "__brand", &GPUExternalTexture::getBrand); installGetterSetter(runtime, prototype, "label", &GPUExternalTexture::getLabel, &GPUExternalTexture::setLabel); + installGetter(runtime, prototype, "yuvToRgbMatrix", + &GPUExternalTexture::getYuvToRgbMatrix); installMethod(runtime, prototype, "destroy", &GPUExternalTexture::destroy); } @@ -85,6 +103,7 @@ class GPUExternalTexture : public NativeObject { wgpu::Texture _texture; std::shared_ptr _source; std::string _label; + std::array _yuvToRgbMatrix; }; } // namespace rnwgpu diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 58728ad32..80c1df5e6 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -93,6 +93,15 @@ declare global { // pool (e.g. a camera/video player) and pile up GPU resources. interface GPUExternalTexture { destroy(): void; + // Non-spec extension: a 3x4 row-major matrix (12 numbers) mapping the + // sampled texel [r, g, b, 1] to gamma-encoded R'G'B'. On the Android + // opaque-YCbCr camera path, textureSampleBaseClampToEdge returns raw + // [Y, Cb, Cr] (Dawn hard-codes an RGB_IDENTITY Vulkan conversion); this + // matrix is derived per-buffer from the driver's suggested YCbCr model and + // range (BT.601/709/2020, full/narrow). Everywhere else (iOS, RGBA + // surfaces) it is the identity passthrough, so shaders can apply it + // unconditionally. Upload it as a uniform and multiply after sampling. + readonly yuvToRgbMatrix: number[]; } // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes)