diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index c2571c4f8..e35f11863 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -167,6 +167,8 @@ const CameraView = () => { // This is what lets us call the interop factory off RNWebGPU, where the native // platform context already lives, instead of off `device`. const rnwgpu = RNWebGPU; + // Captured as a plain bool so the worklet doesn't reach for Platform. + const isAndroid = Platform.OS === "android"; const devices = useCameraDevices(); // Pick back camera if available, otherwise front, otherwise anything. The // iOS simulator returns an empty list since there are no cameras, in which @@ -254,7 +256,8 @@ const CameraView = () => { minFilter: "linear", }); const uniformBuffer = device.createBuffer({ - size: 32, + // vec4f params + vec4u modes + 3x vec4f yuvToRgbMatrix rows. + size: 80, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); @@ -282,7 +285,8 @@ const CameraView = () => { primitive: { topology: "triangle-list" }, }); const prepassUniformBuffer = device.createBuffer({ - size: 16, + // 2x vec2f sizes + 3x vec4f yuvToRgbMatrix rows. + size: 64, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); @@ -469,8 +473,11 @@ const CameraView = () => { // Orientation. The sensor buffer is landscape; frame.orientation is // the rotation needed to bring it upright. We hand that to Dawn via // importExternalTexture's `rotation`, which de-rotates the frame into - // the portrait canvas. The residual vertical flip (Android buffer - // Y-origin) and YUV->RGB are corrected in the shader (CAMERA_PRELUDE). + // the portrait canvas. On Android the buffer additionally arrives + // mirrored on both axes relative to the canvas (buffer Y-origin), + // which is the same as an extra 180° rotation, folded in below. + // YUV->RGB is applied in the shader via the + // GPUExternalTexture.yuvToRgbMatrix uniform (see CAMERA_PRELUDE). let rotationDeg: 0 | 90 | 180 | 270 = 0; if (frame.orientation === "right") { rotationDeg = 90; @@ -479,6 +486,9 @@ const CameraView = () => { } else if (frame.orientation === "left") { rotationDeg = 270; } + if (isAndroid) { + rotationDeg = ((rotationDeg + 180) % 360) as 0 | 90 | 180 | 270; + } // A 90/270 rotation swaps the displayed width & height, so cover-fit // uses the post-rotation dimensions. const rotated = rotationDeg === 90 || rotationDeg === 270; @@ -496,21 +506,6 @@ const CameraView = () => { } else { sy = frameAR / canvasAR; } - // 32-byte uniform: vec4f params + vec4u modes. Built on a single - // ArrayBuffer so the f32/u32 halves go up in one writeBuffer call. - const uniformData = new ArrayBuffer(32); - const uniformF32 = new Float32Array(uniformData); - const uniformU32 = new Uint32Array(uniformData); - uniformF32[0] = sx; - uniformF32[1] = sy; - uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0; - uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0; - uniformU32[4] = modes.effect; - uniformU32[5] = modes.tint; - uniformU32[6] = modes.vignette; - uniformU32[7] = blurMode; - device.queue.writeBuffer(uniformBuffer, 0, uniformData); - let externalTex; try { externalTex = device.importExternalTexture({ @@ -525,6 +520,29 @@ const CameraView = () => { ); throw e; } + // Per-buffer YUV->RGB conversion for the shader: the driver-derived + // matrix on the Android opaque-YCbCr path, an identity passthrough + // on iOS / RGBA surfaces (see CAMERA_PRELUDE). + const yuvMatrix = externalTex.yuvToRgbMatrix; + + // 80-byte uniform: vec4f params + vec4u modes + the three vec4f + // yuvToRgbMatrix rows. Built on a single ArrayBuffer so the f32/u32 + // halves go up in one writeBuffer call. + const uniformData = new ArrayBuffer(80); + const uniformF32 = new Float32Array(uniformData); + const uniformU32 = new Uint32Array(uniformData); + uniformF32[0] = sx; + uniformF32[1] = sy; + uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0; + uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0; + uniformU32[4] = modes.effect; + uniformU32[5] = modes.tint; + uniformU32[6] = modes.vignette; + uniformU32[7] = blurMode; + for (let i = 0; i < 12; i++) { + uniformF32[8 + i] = yuvMatrix[i] ?? 0; + } + device.queue.writeBuffer(uniformBuffer, 0, uniformData); const bindGroup = device.createBindGroup({ layout: pipeline.getBindGroupLayout(0), entries: [ @@ -542,7 +560,13 @@ const CameraView = () => { device.queue.writeBuffer( prepassUniformBuffer, 0, - new Float32Array([dispW, dispH, canvasWidth, canvasHeight]), + new Float32Array([ + dispW, + dispH, + canvasWidth, + canvasHeight, + ...yuvMatrix, + ]), ); const prepassBindGroup = device.createBindGroup({ layout: prepassPipeline.getBindGroupLayout(0), diff --git a/apps/example/src/VisionCamera/blurShaders.ts b/apps/example/src/VisionCamera/blurShaders.ts index eddbfb8d7..803fcde94 100644 --- a/apps/example/src/VisionCamera/blurShaders.ts +++ b/apps/example/src/VisionCamera/blurShaders.ts @@ -21,6 +21,10 @@ export const PREPASS_SHADER = /* wgsl */ ` struct PrepassUniforms { texSize: vec2f, canvasSize: vec2f, + // The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE). + yuv0: vec4f, + yuv1: vec4f, + yuv2: vec4f, }; @group(0) @binding(0) var srcTex: texture_external; @@ -63,14 +67,14 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { scale = vec2f(1.0, texAR / canvasAR); } let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * scale; - // cameraCoord (vertical flip) + cameraDecode (YUV->RGB) come from - // CAMERA_PRELUDE, prepended when this module is compiled. They are no-ops on - // iOS and handle the Android opaque-YUV case. + // cameraDecode (from CAMERA_PRELUDE, prepended when this module is + // compiled) applies GPUExternalTexture.yuvToRgbMatrix: the real YUV->RGB + // conversion on the Android opaque-YUV path, an identity passthrough on iOS. let c = cameraDecode(textureSampleBaseClampToEdge( srcTex, srcSampler, - cameraCoord(clamp(uv, vec2f(0.0), vec2f(1.0))), - )); + clamp(uv, vec2f(0.0), vec2f(1.0)), + ), u.yuv0, u.yuv1, u.yuv2); return vec4f(c.rgb, 1.0); } `; diff --git a/apps/example/src/VisionCamera/shaders.ts b/apps/example/src/VisionCamera/shaders.ts index e93520107..3d5cade38 100644 --- a/apps/example/src/VisionCamera/shaders.ts +++ b/apps/example/src/VisionCamera/shaders.ts @@ -1,10 +1,8 @@ -import { Platform } from "react-native"; - // Main camera-effects shader: samples the imported external texture (camera // frame) with cover-fit + optional chromatic aberration / pixelate, optionally // mixes in the pre-blurred backdrop, then applies effect / tint / vignette. -// `cameraCoord` / `cameraDecode` come from CAMERA_PRELUDE, which is prepended -// at shader-module creation time. +// `cameraDecode` comes from CAMERA_PRELUDE, which is prepended at +// shader-module creation time. export const SHADER = /* wgsl */ ` struct VsOut { @builtin(position) position: vec4f, @@ -22,6 +20,10 @@ struct Uniforms { // w: blurMode (0 off, 1 strong - blurred everywhere (prepass bakes // cover-fit), 2 overlay - blurred backdrop + sharp card) modes: vec4u, + // The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE). + yuv0: vec4f, + yuv1: vec4f, + yuv2: vec4f, }; @group(0) @binding(0) var srcTex: texture_external; @@ -56,7 +58,8 @@ fn snap(uv: vec2f, block: f32) -> vec2f { fn sampleExternal(uv: vec2f, block: f32) -> vec4f { return cameraDecode( - textureSampleBaseClampToEdge(srcTex, srcSampler, cameraCoord(snap(uv, block))), + textureSampleBaseClampToEdge(srcTex, srcSampler, snap(uv, block)), + u.yuv0, u.yuv1, u.yuv2, ); } @@ -177,43 +180,21 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { // YCbCr conversion that is hard-coded to RGB_IDENTITY (Dawn's // SamplerVk.cpp::GetYCbCrForTextureView; see crbug.com/497675620), so the // external sample comes back as raw [Y, Cb, Cr] on *every* device — this is by -// design, not a driver quirk — and we do the BT.709 YUV->RGB ourselves below. -// (Dawn's own SharedTextureMemoryOpaqueYCbCrAndroidForExternalTexture -// .NoopSampleY8Cb8Cr8AHB test asserts the same raw passthrough.) The frame also -// comes out mirrored on both axes relative to the canvas (Android buffer -// origin), so we flip X and Y. iOS goes through the native two-plane path, -// which already converts and orients, so this correction is Android-only. The -// prelude is prepended to every shader module that samples the camera (main -// pass + blur prepass). +// design, not a driver quirk. The correct conversion depends on the buffer: +// react-native-webgpu derives it from the driver's suggested YCbCr model and +// range (BT.601/709/2020, full/narrow) and exposes it as +// GPUExternalTexture.yuvToRgbMatrix; the worklet uploads its three rows as +// vec4f uniforms and cameraDecode applies them. On iOS (native two-plane path, +// already converted by Dawn) and for RGBA surfaces the matrix is the identity +// passthrough, so the decode is safe to apply unconditionally. The prelude is +// prepended to every shader module that samples the camera (main pass + blur +// prepass). export const CAMERA_PRELUDE = /* wgsl */ ` -const CAMERA_IS_YUV: bool = ${Platform.OS === "android"}; -const CAMERA_FLIP_X: bool = ${Platform.OS === "android"}; -const CAMERA_FLIP_Y: bool = ${Platform.OS === "android"}; - -fn cameraCoord(uv: vec2f) -> vec2f { - var c = uv; - if (CAMERA_FLIP_X) { - c.x = 1.0 - c.x; - } - if (CAMERA_FLIP_Y) { - c.y = 1.0 - c.y; - } - return c; -} - -// BT.709 limited-range YUV -> RGB. On the Android opaque path the sampled -// channels are always raw [Y, Cb, Cr] (Dawn forces an RGB_IDENTITY Vulkan -// conversion); a no-op passthrough on every other platform. -fn cameraDecode(c: vec4f) -> vec4f { - if (!CAMERA_IS_YUV) { - return c; - } - let y = c.r - 0.0627451; - let cb = c.g - 0.5; - let cr = c.b - 0.5; - let r = 1.164384 * y + 1.792741 * cr; - let g = 1.164384 * y - 0.213249 * cb - 0.532909 * cr; - let b = 1.164384 * y + 2.112402 * cb; - return vec4f(clamp(vec3f(r, g, b), vec3f(0.0), vec3f(1.0)), 1.0); +// Apply a 3x4 row-major [c.r, c.g, c.b, 1] -> R'G'B' matrix +// (GPUExternalTexture.yuvToRgbMatrix, see above). +fn cameraDecode(c: vec4f, m0: vec4f, m1: vec4f, m2: vec4f) -> vec4f { + let v = vec4f(c.rgb, 1.0); + let rgb = vec3f(dot(m0, v), dot(m1, v), dot(m2, v)); + return vec4f(clamp(rgb, vec3f(0.0), vec3f(1.0)), 1.0); } `; diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index 080b1bb5e..b20078669 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -241,8 +241,8 @@ class AndroidPlatformContext : public PlatformContext { const uint32_t stridePixels = actualDesc.stride; void *vaddr = nullptr; - int rc = AHardwareBuffer_lock(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, - -1, nullptr, &vaddr); + int rc = AHardwareBuffer_lock( + buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1, nullptr, &vaddr); if (rc != 0 || !vaddr) { AHardwareBuffer_release(buffer); throw std::runtime_error( @@ -302,6 +302,21 @@ class AndroidPlatformContext : public PlatformContext { AHardwareBuffer_Desc desc = {}; AHardwareBuffer_describe(buffer, &desc); + // Dawn derives the importable WebGPU usages from the AHB's usage bits; + // without GPU_SAMPLED_IMAGE the imported texture never gets + // TextureBinding and import fails deep inside Dawn with an opaque + // validation error. Surface the real cause here instead. (CameraX's + // default ImageReaders allocate CPU-only buffers; with + // react-native-vision-camera, use pixelFormat: 'native' which allocates + // GPU-sampleable buffers.) + if ((desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) == 0) { + throw std::runtime_error( + "wrapNativeBuffer: this AHardwareBuffer was allocated without " + "AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, so the GPU cannot sample " + "it. Camera/CPU pipelines must allocate frames with GPU usage (for " + "react-native-vision-camera, use pixelFormat: 'native')."); + } + AHardwareBuffer_acquire(buffer); VideoFrameHandle handle; @@ -312,13 +327,13 @@ class AndroidPlatformContext : public PlatformContext { // Dawn's OpaqueYCbCrAndroidForExternalTexture path. Single-plane RGBA AHBs // take the plain BGRA8 path (sampled as a regular 2D texture). switch (desc.format) { - case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420: - case AHARDWAREBUFFER_FORMAT_YCbCr_P010: - handle.pixelFormat = VideoPixelFormat::NV12; - break; - default: - handle.pixelFormat = VideoPixelFormat::BGRA8; - break; + case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420: + case AHARDWAREBUFFER_FORMAT_YCbCr_P010: + handle.pixelFormat = VideoPixelFormat::NV12; + break; + default: + handle.pixelFormat = VideoPixelFormat::BGRA8; + break; } handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); }; return handle; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp index 255cd6f80..44b2ea4b0 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp @@ -1,5 +1,6 @@ #include "GPUExternalTexture.h" +#include #include #include #include @@ -56,44 +57,76 @@ static const float kIdentityTransferParams[7] = { 0.0f, // F }; -// BT.709 limited-range YUV -> R'G'B' as a 3x4 row-major matrix mapping -// [Y, Cb, Cr, 1] to gamma-encoded R'G'B' (NOT linear; the sRGB decode in -// srcTransferFunctionParameters linearizes afterwards). Same values the Apple -// NV12 path computes from the CVPixelBuffer; used for Android buffers that -// arrive as a *defined* biplanar format (where we split the planes and convert -// ourselves) rather than an opaque external-format AHB. Camera streams are -// limited-range BT.709 in the overwhelming majority of cases; full-range / -// BT.601 would need different coefficients (refine from the buffer's suggested -// range if it matters). -[[maybe_unused]] static const float kBT709LimitedToRgb[12] = { - 1.164383f, 0.000000f, 1.792741f, -0.972945f, // - 1.164383f, -0.213249f, -0.532909f, 0.301517f, // - 1.164383f, 2.112402f, 0.000000f, -1.133402f, // +// Identity passthrough for GPUExternalTexture.yuvToRgbMatrix: the sampled +// texel is already RGB (Apple's biplanar path where Dawn converts in the +// sampler transform, RGBA surfaces, or an Android AHB whose driver reports +// RGB_IDENTITY). A 3x4 row-major matrix mapping [c.r, c.g, c.b, 1] to itself. +static constexpr std::array kYuvPassthroughMatrix = { + 1.0f, 0.0f, 0.0f, 0.0f, // + 0.0f, 1.0f, 0.0f, 0.0f, // + 0.0f, 0.0f, 1.0f, 0.0f, // }; -// True for the multi-planar Y + CbCr formats whose planes we can view as -// Plane0Only (luma) / Plane1Only (chroma) and convert with an explicit matrix. -// Excludes OpaqueYCbCrAndroid (external format, no plane views) and triplanar -// formats (would need a third plane). Only referenced on Android. -[[maybe_unused]] static bool isBiplanarYuvFormat(wgpu::TextureFormat format) { - switch (format) { - case wgpu::TextureFormat::R8BG8Biplanar420Unorm: - case wgpu::TextureFormat::R8BG8Biplanar422Unorm: - case wgpu::TextureFormat::R8BG8Biplanar444Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar420Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar422Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar444Unorm: - return true; +// Build a 3x4 row-major matrix mapping the *sampled* gamma-encoded +// [Y, Cb, Cr, 1] (each normalized to [0,1]) to gamma-encoded R'G'B', from the +// Vulkan-suggested YCbCr model + range of an Android external-format buffer. +// Needed because Dawn's opaque-YCbCr sampling path hard-codes an RGB_IDENTITY +// conversion (SamplerVk.cpp::GetYCbCrForTextureView, crbug.com/497675620), so +// the shader receives raw Y/Cb/Cr and must convert itself. +// The VkSamplerYcbcrModelConversion / VkSamplerYcbcrRange values are inlined +// to avoid a Vulkan header dependency in this cross-platform file. +[[maybe_unused]] static std::array +makeYuvToRgbMatrix(uint32_t vkModel, uint32_t vkRange) { + constexpr uint32_t kModelRgbIdentity = 0; // VK_..._RGB_IDENTITY + constexpr uint32_t kModel709 = 2; // VK_..._YCBCR_709 + constexpr uint32_t kModel601 = 3; // VK_..._YCBCR_601 + constexpr uint32_t kModel2020 = 4; // VK_..._YCBCR_2020 + constexpr uint32_t kRangeItuNarrow = 1; // VK_SAMPLER_YCBCR_RANGE_ITU_NARROW + + if (vkModel == kModelRgbIdentity) { + // The buffer content is RGB; nothing to convert. + return kYuvPassthroughMatrix; + } + float kr; + float kb; + switch (vkModel) { + case kModel709: + kr = 0.2126f; + kb = 0.0722f; + break; + case kModel2020: + kr = 0.2627f; + kb = 0.0593f; + break; + case kModel601: default: - return false; + // YCBCR_IDENTITY and unknown models: Android camera streams are BT.601 in + // practice, so that is the sane default. + kr = 0.299f; + kb = 0.114f; + break; } + const float kg = 1.0f - kr - kb; + const bool narrow = vkRange == kRangeItuNarrow; + const float yScale = narrow ? 255.0f / 219.0f : 1.0f; + const float cScale = narrow ? 255.0f / 224.0f : 1.0f; + const float yOffset = narrow ? 16.0f / 255.0f : 0.0f; + const float cOffset = 128.0f / 255.0f; + const float crR = 2.0f * (1.0f - kr) * cScale; + const float cbG = -2.0f * kb * (1.0f - kb) / kg * cScale; + const float crG = -2.0f * kr * (1.0f - kr) / kg * cScale; + const float cbB = 2.0f * (1.0f - kb) * cScale; + return { + yScale, 0.0f, crR, -yScale * yOffset - crR * cOffset, // + yScale, cbG, crG, -yScale * yOffset - (cbG + crG) * cOffset, // + yScale, cbB, 0.0f, -yScale * yOffset - cbB * cOffset, // + }; } // Map a rotation in degrees (0 / 90 / 180 / 270) to Dawn's enum. Anything that // isn't a clean multiple of 90 snaps to the nearest quadrant; Dawn only // supports those four steps for external textures. -static wgpu::ExternalTextureRotation -toExternalTextureRotation(double degrees) { +static wgpu::ExternalTextureRotation toExternalTextureRotation(double degrees) { int quadrant = static_cast(std::lround(degrees / 90.0)) & 3; switch (quadrant) { case 1: @@ -216,7 +249,11 @@ std::shared_ptr GPUExternalTexture::Create( return std::make_shared( std::move(external), std::move(memory), std::move(texture), - std::move(descriptor->source), std::move(label)); + std::move(descriptor->source), std::move(label), + // Dawn's Metal path applies the YUV->RGB conversion (from + // frame.yuvToRgbMatrix above) inside the sampling transform, so the + // sampled texel is already RGB. + kYuvPassthroughMatrix); #elif defined(__ANDROID__) // 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this // yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid @@ -257,32 +294,39 @@ std::shared_ptr GPUExternalTexture::Create( "GPUExternalTexture::Create(): BeginAccess failed"); } - // 4. Build the ExternalTextureDescriptor. There are two cases depending on - // how Dawn imported the AHB (see SharedTextureMemoryVk.cpp): - // - // a. *External* format (camera buffers whose layout has no Vulkan - // equivalent) -> OpaqueYCbCrAndroid, a single opaque plane. Sampling - // routes through a Vulkan SamplerYcbcrConversion whose model Dawn - // copies verbatim from the AHB's suggestedYcbcrModel. We pass a single - // plane + identity transfer and let that conversion (if any) run. NOTE: - // when the driver reports RGB_IDENTITY the sample comes back as raw - // Y/Cb/Cr; there is no public hook to override the model on this path. - // - // b. *Defined* biplanar format (e.g. R8BG8Biplanar420Unorm, exposed by the - // dawn-multi-planar-formats feature) -> we split Plane0Only (luma) / - // Plane1Only (chroma) and hand Dawn an explicit BT.709 matrix + sRGB - // transfer, exactly like the iOS NV12 path. This makes numPlanes == 2 - // so the matrix is actually applied (the single-plane branch in Dawn's - // Tint transform ignores yuvToRgbConversionMatrix). - // - // Either way we must pass non-null gamut/transfer arrays: - // ComputeExternalTextureParams dereferences them unconditionally - // (kIdentityTransferParams is defined at file scope). - const bool isBiplanar = frame.pixelFormat == VideoPixelFormat::NV12 && - isBiplanarYuvFormat(texture.GetFormat()); + // 4. Resolve the YUV->RGB conversion the *shader* must apply. Dawn imports + // every YUV / implementation-defined AHB format as OpaqueYCbCrAndroid + // (AHBFunctions.cpp::FormatFromAHardwareBufferFormat) and samples it + // through a Vulkan SamplerYcbcrConversion that is hard-coded to + // RGB_IDENTITY (SamplerVk.cpp::GetYCbCrForTextureView, + // crbug.com/497675620), so the sample always comes back as raw + // [Y, Cb, Cr]. The driver's *suggested* model + range are captured at + // import time though; read them back from the shared memory's properties + // and derive the correct conversion matrix, exposed to JS as + // GPUExternalTexture.yuvToRgbMatrix. Defined color formats (e.g. an RGBA + // AHB -> RGBA8Unorm) sample as RGB directly and get the passthrough. + std::array yuvToRgbMatrix = kYuvPassthroughMatrix; + if (texture.GetFormat() == wgpu::TextureFormat::OpaqueYCbCrAndroid) { + wgpu::SharedTextureMemoryAHardwareBufferProperties ahbProps{}; + wgpu::SharedTextureMemoryProperties props{}; + props.nextInChain = &ahbProps; + if (memory.GetProperties(&props)) { + yuvToRgbMatrix = makeYuvToRgbMatrix(ahbProps.yCbCrInfo.vkYCbCrModel, + ahbProps.yCbCrInfo.vkYCbCrRange); + } else { + // Fall back to the Android camera norm rather than passthrough. + yuvToRgbMatrix = makeYuvToRgbMatrix(/* YCBCR_601 */ 3, + /* ITU_NARROW */ 1); + } + } - wgpu::TextureView plane0; - wgpu::TextureView plane1; + // 5. Build the ExternalTextureDescriptor: a single (opaque or RGB) plane + // with identity transfer. The YUV->RGB conversion cannot run inside + // Dawn's external-texture transform here: the single-plane branch of the + // Tint transform ignores yuvToRgbConversionMatrix, which is why the + // matrix is surfaced to the shader instead. The gamut/transfer arrays + // must still be non-null: ComputeExternalTextureParams dereferences them + // unconditionally. wgpu::ExternalTextureDescriptor extDesc{}; if (!label.empty()) { extDesc.label = wgpu::StringView(label.c_str(), label.size()); @@ -291,24 +335,10 @@ std::shared_ptr GPUExternalTexture::Create( extDesc.cropSize = {frame.width, frame.height}; extDesc.apparentSize = {frame.width, frame.height}; extDesc.gamutConversionMatrix = kIdentityGamutMatrix; - if (isBiplanar) { - wgpu::TextureViewDescriptor v0{}; - v0.aspect = wgpu::TextureAspect::Plane0Only; - plane0 = texture.CreateView(&v0); - wgpu::TextureViewDescriptor v1{}; - v1.aspect = wgpu::TextureAspect::Plane1Only; - plane1 = texture.CreateView(&v1); - extDesc.plane0 = plane0; - extDesc.plane1 = plane1; - extDesc.yuvToRgbConversionMatrix = kBT709LimitedToRgb; - extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; - extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; - } else { - plane0 = texture.CreateView(); - extDesc.plane0 = plane0; - extDesc.srcTransferFunctionParameters = kIdentityTransferParams; - extDesc.dstTransferFunctionParameters = kIdentityTransferParams; - } + wgpu::TextureView plane0 = texture.CreateView(); + extDesc.plane0 = plane0; + extDesc.srcTransferFunctionParameters = kIdentityTransferParams; + extDesc.dstTransferFunctionParameters = kIdentityTransferParams; extDesc.mirrored = descriptor->mirrored.value_or(false); extDesc.rotation = toExternalTextureRotation(descriptor->rotation.value_or(0)); @@ -324,7 +354,8 @@ std::shared_ptr GPUExternalTexture::Create( return std::make_shared( std::move(external), std::move(memory), std::move(texture), - std::move(descriptor->source), std::move(label)); + std::move(descriptor->source), std::move(label), + std::move(yuvToRgbMatrix)); #else throw std::runtime_error( "GPUExternalTexture::Create(): not yet implemented on this " diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h index 217ab45d9..917909eb9 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h @@ -1,8 +1,10 @@ #pragma once +#include #include #include #include +#include #include "Unions.h" @@ -36,10 +38,12 @@ class GPUExternalTexture : public NativeObject { // the producer (e.g. AVPlayer) can reclaim the IOSurface. GPUExternalTexture(wgpu::ExternalTexture instance, wgpu::SharedTextureMemory memory, wgpu::Texture texture, - std::shared_ptr source, std::string label) + std::shared_ptr source, std::string label, + std::array yuvToRgbMatrix) : NativeObject(CLASS_NAME), _instance(std::move(instance)), _memory(std::move(memory)), _texture(std::move(texture)), - _source(std::move(source)), _label(std::move(label)) {} + _source(std::move(source)), _label(std::move(label)), + _yuvToRgbMatrix(yuvToRgbMatrix) {} ~GPUExternalTexture() override { destroy(); } @@ -69,11 +73,25 @@ class GPUExternalTexture : public NativeObject { _instance.SetLabel(_label.c_str()); } + // Non-spec extension. A 3x4 row-major matrix mapping the *sampled* texel + // [c.r, c.g, c.b, 1] to gamma-encoded R'G'B'. When Dawn's sampler already + // produces RGB (Apple's biplanar path, RGBA surfaces), this is the identity + // passthrough; on the Android opaque-YCbCr path the sample comes back as raw + // [Y, Cb, Cr] (Dawn hard-codes an RGB_IDENTITY Vulkan conversion, see + // crbug.com/497675620) and this matrix is derived from the driver's + // suggested YCbCr model + range for the buffer. Shaders can therefore apply + // it unconditionally after textureSampleBaseClampToEdge. + std::vector getYuvToRgbMatrix() { + return std::vector(_yuvToRgbMatrix.begin(), _yuvToRgbMatrix.end()); + } + static void definePrototype(jsi::Runtime &runtime, jsi::Object &prototype) { installGetter(runtime, prototype, "__brand", &GPUExternalTexture::getBrand); installGetterSetter(runtime, prototype, "label", &GPUExternalTexture::getLabel, &GPUExternalTexture::setLabel); + installGetter(runtime, prototype, "yuvToRgbMatrix", + &GPUExternalTexture::getYuvToRgbMatrix); installMethod(runtime, prototype, "destroy", &GPUExternalTexture::destroy); } @@ -85,6 +103,7 @@ class GPUExternalTexture : public NativeObject { wgpu::Texture _texture; std::shared_ptr _source; std::string _label; + std::array _yuvToRgbMatrix; }; } // namespace rnwgpu diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 58728ad32..80c1df5e6 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -93,6 +93,15 @@ declare global { // pool (e.g. a camera/video player) and pile up GPU resources. interface GPUExternalTexture { destroy(): void; + // Non-spec extension: a 3x4 row-major matrix (12 numbers) mapping the + // sampled texel [r, g, b, 1] to gamma-encoded R'G'B'. On the Android + // opaque-YCbCr camera path, textureSampleBaseClampToEdge returns raw + // [Y, Cb, Cr] (Dawn hard-codes an RGB_IDENTITY Vulkan conversion); this + // matrix is derived per-buffer from the driver's suggested YCbCr model and + // range (BT.601/709/2020, full/narrow). Everywhere else (iOS, RGBA + // surfaces) it is the identity passthrough, so shaders can apply it + // unconditionally. Upload it as a uniform and multiply after sampling. + readonly yuvToRgbMatrix: number[]; } // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes)