Skip to content

feat(🤖): correct per-buffer YUV conversion for Android camera frames #387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wcandillon wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(🤖): correct per-buffer YUV conversion for Android camera frames #387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 44 additions & 20 deletions apps/example/src/VisionCamera/VisionCamera.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ const CameraView = () => {
// This is what lets us call the interop factory off RNWebGPU, where the native
// platform context already lives, instead of off `device`.
const rnwgpu = RNWebGPU;
// Captured as a plain bool so the worklet doesn't reach for Platform.
const isAndroid = Platform.OS === "android";
const devices = useCameraDevices();
// Pick back camera if available, otherwise front, otherwise anything. The
// iOS simulator returns an empty list since there are no cameras, in which
Expand Down Expand Up @@ -254,7 +256,8 @@ const CameraView = () => {
minFilter: "linear",
});
const uniformBuffer = device.createBuffer({
size: 32,
// vec4f params + vec4u modes + 3x vec4f yuvToRgbMatrix rows.
size: 80,
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
});

Expand Down Expand Up @@ -282,7 +285,8 @@ const CameraView = () => {
primitive: { topology: "triangle-list" },
});
const prepassUniformBuffer = device.createBuffer({
size: 16,
// 2x vec2f sizes + 3x vec4f yuvToRgbMatrix rows.
size: 64,
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
});

Expand Down Expand Up @@ -469,8 +473,11 @@ const CameraView = () => {
// Orientation. The sensor buffer is landscape; frame.orientation is
// the rotation needed to bring it upright. We hand that to Dawn via
// importExternalTexture's `rotation`, which de-rotates the frame into
// the portrait canvas. The residual vertical flip (Android buffer
// Y-origin) and YUV->RGB are corrected in the shader (CAMERA_PRELUDE).
// the portrait canvas. On Android the buffer additionally arrives
// mirrored on both axes relative to the canvas (buffer Y-origin),
// which is the same as an extra 180° rotation, folded in below.
// YUV->RGB is applied in the shader via the
// GPUExternalTexture.yuvToRgbMatrix uniform (see CAMERA_PRELUDE).
let rotationDeg: 0 | 90 | 180 | 270 = 0;
if (frame.orientation === "right") {
rotationDeg = 90;
Expand All @@ -479,6 +486,9 @@ const CameraView = () => {
} else if (frame.orientation === "left") {
rotationDeg = 270;
}
if (isAndroid) {
rotationDeg = ((rotationDeg + 180) % 360) as 0 | 90 | 180 | 270;
}
// A 90/270 rotation swaps the displayed width & height, so cover-fit
// uses the post-rotation dimensions.
const rotated = rotationDeg === 90 || rotationDeg === 270;
Expand All @@ -496,21 +506,6 @@ const CameraView = () => {
} else {
sy = frameAR / canvasAR;
}
// 32-byte uniform: vec4f params + vec4u modes. Built on a single
// ArrayBuffer so the f32/u32 halves go up in one writeBuffer call.
const uniformData = new ArrayBuffer(32);
const uniformF32 = new Float32Array(uniformData);
const uniformU32 = new Uint32Array(uniformData);
uniformF32[0] = sx;
uniformF32[1] = sy;
uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0;
uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0;
uniformU32[4] = modes.effect;
uniformU32[5] = modes.tint;
uniformU32[6] = modes.vignette;
uniformU32[7] = blurMode;
device.queue.writeBuffer(uniformBuffer, 0, uniformData);

let externalTex;
try {
externalTex = device.importExternalTexture({
Expand All @@ -525,6 +520,29 @@ const CameraView = () => {
);
throw e;
}
// Per-buffer YUV->RGB conversion for the shader: the driver-derived
// matrix on the Android opaque-YCbCr path, an identity passthrough
// on iOS / RGBA surfaces (see CAMERA_PRELUDE).
const yuvMatrix = externalTex.yuvToRgbMatrix;

// 80-byte uniform: vec4f params + vec4u modes + the three vec4f
// yuvToRgbMatrix rows. Built on a single ArrayBuffer so the f32/u32
// halves go up in one writeBuffer call.
const uniformData = new ArrayBuffer(80);
const uniformF32 = new Float32Array(uniformData);
const uniformU32 = new Uint32Array(uniformData);
uniformF32[0] = sx;
uniformF32[1] = sy;
uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0;
uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0;
uniformU32[4] = modes.effect;
uniformU32[5] = modes.tint;
uniformU32[6] = modes.vignette;
uniformU32[7] = blurMode;
for (let i = 0; i < 12; i++) {
uniformF32[8 + i] = yuvMatrix[i] ?? 0;
}
device.queue.writeBuffer(uniformBuffer, 0, uniformData);
const bindGroup = device.createBindGroup({
layout: pipeline.getBindGroupLayout(0),
entries: [
Expand All @@ -542,7 +560,13 @@ const CameraView = () => {
device.queue.writeBuffer(
prepassUniformBuffer,
0,
new Float32Array([dispW, dispH, canvasWidth, canvasHeight]),
new Float32Array([
dispW,
dispH,
canvasWidth,
canvasHeight,
...yuvMatrix,
]),
);
const prepassBindGroup = device.createBindGroup({
layout: prepassPipeline.getBindGroupLayout(0),
Expand Down
14 changes: 9 additions & 5 deletions apps/example/src/VisionCamera/blurShaders.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ export const PREPASS_SHADER = /* wgsl */ `
struct PrepassUniforms {
texSize: vec2f,
canvasSize: vec2f,
// The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE).
yuv0: vec4f,
yuv1: vec4f,
yuv2: vec4f,
};

@group(0) @binding(0) var srcTex: texture_external;
Expand Down Expand Up @@ -63,14 +67,14 @@ fn fs_main(in: VsOut) -> @location(0) vec4f {
scale = vec2f(1.0, texAR / canvasAR);
}
let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * scale;
// cameraCoord (vertical flip) + cameraDecode (YUV->RGB) come from
// CAMERA_PRELUDE, prepended when this module is compiled. They are no-ops on
// iOS and handle the Android opaque-YUV case.
// cameraDecode (from CAMERA_PRELUDE, prepended when this module is
// compiled) applies GPUExternalTexture.yuvToRgbMatrix: the real YUV->RGB
// conversion on the Android opaque-YUV path, an identity passthrough on iOS.
let c = cameraDecode(textureSampleBaseClampToEdge(
srcTex,
srcSampler,
cameraCoord(clamp(uv, vec2f(0.0), vec2f(1.0))),
));
clamp(uv, vec2f(0.0), vec2f(1.0)),
), u.yuv0, u.yuv1, u.yuv2);
return vec4f(c.rgb, 1.0);
}
`;
Expand Down
65 changes: 23 additions & 42 deletions apps/example/src/VisionCamera/shaders.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
import { Platform } from "react-native";

// Main camera-effects shader: samples the imported external texture (camera
// frame) with cover-fit + optional chromatic aberration / pixelate, optionally
// mixes in the pre-blurred backdrop, then applies effect / tint / vignette.
// `cameraCoord` / `cameraDecode` come from CAMERA_PRELUDE, which is prepended
// at shader-module creation time.
// `cameraDecode` comes from CAMERA_PRELUDE, which is prepended at
// shader-module creation time.
export const SHADER = /* wgsl */ `
struct VsOut {
@builtin(position) position: vec4f,
Expand All @@ -22,6 +20,10 @@ struct Uniforms {
// w: blurMode (0 off, 1 strong - blurred everywhere (prepass bakes
// cover-fit), 2 overlay - blurred backdrop + sharp card)
modes: vec4u,
// The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE).
yuv0: vec4f,
yuv1: vec4f,
yuv2: vec4f,
};

@group(0) @binding(0) var srcTex: texture_external;
Expand Down Expand Up @@ -56,7 +58,8 @@ fn snap(uv: vec2f, block: f32) -> vec2f {

fn sampleExternal(uv: vec2f, block: f32) -> vec4f {
return cameraDecode(
textureSampleBaseClampToEdge(srcTex, srcSampler, cameraCoord(snap(uv, block))),
textureSampleBaseClampToEdge(srcTex, srcSampler, snap(uv, block)),
u.yuv0, u.yuv1, u.yuv2,
);
}

Expand Down Expand Up @@ -177,43 +180,21 @@ fn fs_main(in: VsOut) -> @location(0) vec4f {
// YCbCr conversion that is hard-coded to RGB_IDENTITY (Dawn's
// SamplerVk.cpp::GetYCbCrForTextureView; see crbug.com/497675620), so the
// external sample comes back as raw [Y, Cb, Cr] on *every* device — this is by
// design, not a driver quirk — and we do the BT.709 YUV->RGB ourselves below.
// (Dawn's own SharedTextureMemoryOpaqueYCbCrAndroidForExternalTexture
// .NoopSampleY8Cb8Cr8AHB test asserts the same raw passthrough.) The frame also
// comes out mirrored on both axes relative to the canvas (Android buffer
// origin), so we flip X and Y. iOS goes through the native two-plane path,
// which already converts and orients, so this correction is Android-only. The
// prelude is prepended to every shader module that samples the camera (main
// pass + blur prepass).
// design, not a driver quirk. The correct conversion depends on the buffer:
// react-native-webgpu derives it from the driver's suggested YCbCr model and
// range (BT.601/709/2020, full/narrow) and exposes it as
// GPUExternalTexture.yuvToRgbMatrix; the worklet uploads its three rows as
// vec4f uniforms and cameraDecode applies them. On iOS (native two-plane path,
// already converted by Dawn) and for RGBA surfaces the matrix is the identity
// passthrough, so the decode is safe to apply unconditionally. The prelude is
// prepended to every shader module that samples the camera (main pass + blur
// prepass).
export const CAMERA_PRELUDE = /* wgsl */ `
const CAMERA_IS_YUV: bool = ${Platform.OS === "android"};
const CAMERA_FLIP_X: bool = ${Platform.OS === "android"};
const CAMERA_FLIP_Y: bool = ${Platform.OS === "android"};

fn cameraCoord(uv: vec2f) -> vec2f {
var c = uv;
if (CAMERA_FLIP_X) {
c.x = 1.0 - c.x;
}
if (CAMERA_FLIP_Y) {
c.y = 1.0 - c.y;
}
return c;
}

// BT.709 limited-range YUV -> RGB. On the Android opaque path the sampled
// channels are always raw [Y, Cb, Cr] (Dawn forces an RGB_IDENTITY Vulkan
// conversion); a no-op passthrough on every other platform.
fn cameraDecode(c: vec4f) -> vec4f {
if (!CAMERA_IS_YUV) {
return c;
}
let y = c.r - 0.0627451;
let cb = c.g - 0.5;
let cr = c.b - 0.5;
let r = 1.164384 * y + 1.792741 * cr;
let g = 1.164384 * y - 0.213249 * cb - 0.532909 * cr;
let b = 1.164384 * y + 2.112402 * cb;
return vec4f(clamp(vec3f(r, g, b), vec3f(0.0), vec3f(1.0)), 1.0);
// Apply a 3x4 row-major [c.r, c.g, c.b, 1] -> R'G'B' matrix
// (GPUExternalTexture.yuvToRgbMatrix, see above).
fn cameraDecode(c: vec4f, m0: vec4f, m1: vec4f, m2: vec4f) -> vec4f {
let v = vec4f(c.rgb, 1.0);
let rgb = vec3f(dot(m0, v), dot(m1, v), dot(m2, v));
return vec4f(clamp(rgb, vec3f(0.0), vec3f(1.0)), 1.0);
}
`;
33 changes: 24 additions & 9 deletions packages/webgpu/android/cpp/AndroidPlatformContext.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,8 +241,8 @@ class AndroidPlatformContext : public PlatformContext {
const uint32_t stridePixels = actualDesc.stride;

void *vaddr = nullptr;
int rc = AHardwareBuffer_lock(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY,
-1, nullptr, &vaddr);
int rc = AHardwareBuffer_lock(
buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1, nullptr, &vaddr);
if (rc != 0 || !vaddr) {
AHardwareBuffer_release(buffer);
throw std::runtime_error(
Expand Down Expand Up @@ -302,6 +302,21 @@ class AndroidPlatformContext : public PlatformContext {
AHardwareBuffer_Desc desc = {};
AHardwareBuffer_describe(buffer, &desc);

// Dawn derives the importable WebGPU usages from the AHB's usage bits;
// without GPU_SAMPLED_IMAGE the imported texture never gets
// TextureBinding and import fails deep inside Dawn with an opaque
// validation error. Surface the real cause here instead. (CameraX's
// default ImageReaders allocate CPU-only buffers; with
// react-native-vision-camera, use pixelFormat: 'native' which allocates
// GPU-sampleable buffers.)
if ((desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) == 0) {
throw std::runtime_error(
"wrapNativeBuffer: this AHardwareBuffer was allocated without "
"AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, so the GPU cannot sample "
"it. Camera/CPU pipelines must allocate frames with GPU usage (for "
"react-native-vision-camera, use pixelFormat: 'native').");
}

AHardwareBuffer_acquire(buffer);

VideoFrameHandle handle;
Expand All @@ -312,13 +327,13 @@ class AndroidPlatformContext : public PlatformContext {
// Dawn's OpaqueYCbCrAndroidForExternalTexture path. Single-plane RGBA AHBs
// take the plain BGRA8 path (sampled as a regular 2D texture).
switch (desc.format) {
case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
case AHARDWAREBUFFER_FORMAT_YCbCr_P010:
handle.pixelFormat = VideoPixelFormat::NV12;
break;
default:
handle.pixelFormat = VideoPixelFormat::BGRA8;
break;
case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
case AHARDWAREBUFFER_FORMAT_YCbCr_P010:
handle.pixelFormat = VideoPixelFormat::NV12;
break;
default:
handle.pixelFormat = VideoPixelFormat::BGRA8;
break;
}
handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); };
return handle;
Expand Down
Loading
Loading