Clips Desktop Native Recording - upload during the recording

templates/clips/actions/get-feature-flags.ts

@@ -0,0 +1,30 @@
+/**
+ * Read global, server-controlled feature flags (e.g. desktop capture
+ * pipeline toggles) so they can change without a redeploy. The desktop app
+ * polls this using the same session cookie/bearer token it already sends to
+ * other actions (e.g. `list-meetings`).
+ *
+ * Usage:
+ *   pnpm action get-feature-flags
+ */
+
+import { defineAction } from "@agent-native/core";
+import { getSetting } from "@agent-native/core/settings";
+import { z } from "zod";
+
+import {
+  FEATURE_FLAGS_KEY,
+  withFeatureFlagDefaults,
+} from "../shared/feature-flags.js";
+
+export default defineAction({
+  description:
+    "Get global server-controlled feature flags (e.g. desktop capture pipeline toggles). Returns defaults for any flag never explicitly set.",
+  schema: z.object({}),
+  http: { method: "GET" },
+  readOnly: true,
+  run: async () => {
+    const stored = await getSetting(FEATURE_FLAGS_KEY);
+    return withFeatureFlagDefaults(stored);
+  },
+});

templates/clips/app/components/player/video-player.tsx

@@ -28,6 +28,7 @@ import {
   DropdownMenuTrigger,
 } from "@/components/ui/dropdown-menu";
 import { Spinner } from "@/components/ui/spinner";
+import { useMseVideoSource } from "@/hooks/use-mse-video-source";
 import {
   parsePlaybackSpeed,
   readPlaybackSpeedPreference,
@@ -365,6 +366,32 @@ export const VideoPlayer = forwardRef<VideoPlayerHandle, VideoPlayerProps>(
       [resolvedVideoSrc],
     );
 
+    // Media Source Extensions path for raw fragmented-MP4 recordings (desktop
+    // live-stream uploads). Those files declare no up-front duration, so the
+    // native progressive pipeline scans the whole file before it can play from
+    // a CDN. When the asset sniffs as fragmented, `mse.mode === "mse"` and we
+    // hand the element a MediaSource object URL instead of the raw URL, with the
+    // duration supplied from the DB. Everything else (classic MP4, WebM, Loom,
+    // browsers without MediaSource) stays on the native `<video src>` path,
+    // byte-for-byte unchanged.
+    const mse = useMseVideoSource({
+      videoRef,
+      sourceUrl: resolvedVideoSrc,
+      durationMs,
+      videoFormat,
+      disabled: isLoomEmbed || unsupportedFormat,
+    });
+    const mseActive = mse.mode === "mse" && Boolean(mse.objectUrl);
+    // The URL actually put on the <video> element: the MediaSource object URL
+    // while MSE drives playback, nothing while we're still sniffing an eligible
+    // asset (so the browser never starts the slow native scan), otherwise the
+    // normal resolved/cache-busted source.
+    const domVideoSrc = mseActive
+      ? mse.objectUrl
+      : mse.mode === "pending"
+        ? undefined
+        : activeVideoSrc;
+
     useEffect(() => {
       if (!resolvedVideoSrc) {
         setActiveVideoSrc(undefined);
@@ -1164,7 +1191,7 @@ export const VideoPlayer = forwardRef<VideoPlayerHandle, VideoPlayerProps>(
         ) : activeVideoSrc ? (
           <video
             ref={videoRef}
-            src={activeVideoSrc}
+            src={domVideoSrc}
             poster={resolveLocalUrl(thumbnailUrl)}
             // `crossOrigin` is only needed so the owner's canvas thumbnail
             // capture isn't tainted. For everyone else (viewers, and the Slack
@@ -1328,6 +1355,17 @@ export const VideoPlayer = forwardRef<VideoPlayerHandle, VideoPlayerProps>(
               playAttemptPendingRef.current = false;
               setIsPlayPending(false);
 
+              // If the MSE pipeline surfaced a media error, tear it down and let
+              // the native <video src> path take over the raw asset URL instead
+              // of running the cache-bust retry against a MediaSource blob URL.
+              if (mseActive) {
+                mse.fallbackToNative();
+                setIsBuffering(false);
+                setIsPreparing(true);
+                setCanPlay(false);
+                return;
+              }
+
               // Most "format not supported" / decode errors reported here are
               // transient — e.g. the share page's video element started
               // fetching a moment before a background seekable-remux pass

templates/clips/app/hooks/use-mse-video-source.ts

@@ -0,0 +1,146 @@
+/**
+ * Decides whether a recording should play through Media Source Extensions and,
+ * when it should, owns the `MseVideoLoader` lifecycle.
+ *
+ * Raw fragmented-MP4 recordings (from the desktop live-streaming pipeline)
+ * declare no up-front duration, so the browser's progressive `<video src>`
+ * pipeline stalls when they're served from a CDN. This hook sniffs the asset,
+ * and for fragmented files hands the video element a `MediaSource` object URL
+ * driven by range requests with the duration supplied from the DB. Classic MP4,
+ * WebM, Loom, and anything the browser can't MSE fall back to the native path
+ * unchanged.
+ */
+
+import {
+  type RefObject,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
+
+import { sniffFragmentedMp4 } from "@/lib/fmp4";
+import { MseVideoLoader, isMediaSourceSupported } from "@/lib/mse-video-loader";
+
+export type MseSourceMode = "pending" | "native" | "mse";
+
+export interface UseMseVideoSourceParams {
+  videoRef: RefObject<HTMLVideoElement | null>;
+  /** The resolved asset URL (proxied same-origin or provider URL). */
+  sourceUrl: string | undefined;
+  /** Authoritative duration in ms from the DB. */
+  durationMs: number;
+  /** Container format hint. WebM is never eligible for this MP4-only loader. */
+  videoFormat?: "webm" | "mp4" | null;
+  /** True for Loom embeds / formats the browser cannot decode — never MSE. */
+  disabled?: boolean;
+}
+
+export interface UseMseVideoSourceResult {
+  mode: MseSourceMode;
+  /** MediaSource object URL to use as `<video src>` when `mode === "mse"`. */
+  objectUrl: string | undefined;
+  /** Drop MSE and revert to the plain `<video src>` path. */
+  fallbackToNative: () => void;
+}
+
+export function useMseVideoSource({
+  videoRef,
+  sourceUrl,
+  durationMs,
+  videoFormat,
+  disabled,
+}: UseMseVideoSourceParams): UseMseVideoSourceResult {
+  // WebM recordings use the native Infinity-duration workaround; only MP4 (and
+  // unknown-format, which the sniff will confirm) are candidates.
+  const eligible =
+    !disabled &&
+    !!sourceUrl &&
+    videoFormat !== "webm" &&
+    isMediaSourceSupported();
+
+  const [mode, setMode] = useState<MseSourceMode>(() =>
+    eligible ? "pending" : "native",
+  );
+  const [objectUrl, setObjectUrl] = useState<string | undefined>(undefined);
+  const loaderRef = useRef<MseVideoLoader | null>(null);
+
+  // Latest duration, kept in a ref so the (deliberately duration-independent)
+  // rebuild effect below seeds a freshly created loader with the current value
+  // even if it changed during the async sniff. Updating a ref in render is
+  // side-effect-free and safe.
+  const durationMsRef = useRef(durationMs);
+  durationMsRef.current = durationMs;
+
+  useEffect(() => {
+    loaderRef.current?.destroy();
+    loaderRef.current = null;
+    setObjectUrl(undefined);
+
+    if (!eligible || !sourceUrl) {
+      setMode("native");
+      return;
+    }
+
+    setMode("pending");
+    let cancelled = false;
+
+    void sniffFragmentedMp4(sourceUrl)
+      .then((isFragmented) => {
+        if (cancelled) return;
+        const video = videoRef.current;
+        if (!isFragmented || !video) {
+          setMode("native");
+          return;
+        }
+        try {
+          const loader = new MseVideoLoader({
+            url: sourceUrl,
+            durationMs: durationMsRef.current,
+            video,
+            onFatal: () => {
+              if (cancelled) return;
+              loaderRef.current?.destroy();
+              loaderRef.current = null;
+              setObjectUrl(undefined);
+              setMode("native");
+            },
+          });
+          loaderRef.current = loader;
+          setObjectUrl(loader.objectUrl);
+          setMode("mse");
+        } catch {
+          setMode("native");
+        }
+      })
+      .catch(() => {
+        if (!cancelled) setMode("native");
+      });
+
+    return () => {
+      cancelled = true;
+      loaderRef.current?.destroy();
+      loaderRef.current = null;
+    };
+    // videoRef is a stable ref object. `durationMs` is intentionally excluded:
+    // it only seeds the timeline and is pushed to the live loader by the effect
+    // below, so a metadata-poll update must not rebuild the loader (which would
+    // revoke the object URL and restart playback from byte zero).
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [eligible, sourceUrl]);
+
+  // Recording metadata polling can raise the DB duration while the same
+  // fragmented asset is still playing; apply it to the live loader in place.
+  useEffect(() => {
+    loaderRef.current?.setDuration(durationMs);
+  }, [durationMs]);
+
+  const fallbackToNative = useCallback(() => {
+    loaderRef.current?.destroy();
+    loaderRef.current = null;
+    setObjectUrl(undefined);
+    setMode("native");
+  }, []);
+
+  return { mode, objectUrl, fallbackToNative };
+}

templates/clips/app/lib/fmp4.test.ts

@@ -0,0 +1,148 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  findMoofOffset,
+  indexOfAscii,
+  isFragmentedMp4Head,
+  parseAvcCodec,
+  parseInitSegment,
+  readTopLevelBoxes,
+} from "./fmp4";
+
+const enc = new TextEncoder();
+
+function u32(n: number): number[] {
+  return [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
+}
+
+/** Build a box: 4-byte size, 4-byte type, payload. */
+function box(type: string, payload: number[]): number[] {
+  const size = 8 + payload.length;
+  return [...u32(size), ...Array.from(enc.encode(type)), ...payload];
+}
+
+describe("indexOfAscii", () => {
+  it("finds a marker and respects the from offset", () => {
+    const bytes = new Uint8Array(enc.encode("__ftyp__ftyp"));
+    expect(indexOfAscii(bytes, "ftyp")).toBe(2);
+    expect(indexOfAscii(bytes, "ftyp", 3)).toBe(8);
+    expect(indexOfAscii(bytes, "nope")).toBe(-1);
+  });
+});
+
+describe("isFragmentedMp4Head", () => {
+  it("detects the hlsf brand", () => {
+    const ftyp = box("ftyp", [
+      ...Array.from(enc.encode("isom")), // major brand
+      ...u32(0), // minor version
+      ...Array.from(enc.encode("iso5")),
+      ...Array.from(enc.encode("hlsf")),
+    ]);
+    expect(isFragmentedMp4Head(new Uint8Array(ftyp))).toBe(true);
+  });
+
+  it("detects an mvex box", () => {
+    const ftyp = box("ftyp", [...Array.from(enc.encode("isom")), ...u32(0)]);
+    const moov = box("moov", box("mvex", []));
+    expect(isFragmentedMp4Head(new Uint8Array([...ftyp, ...moov]))).toBe(true);
+  });
+
+  it("returns false for a classic (non-fragmented) mp4", () => {
+    const ftyp = box("ftyp", [
+      ...Array.from(enc.encode("isom")),
+      ...u32(0),
+      ...Array.from(enc.encode("mp42")),
+    ]);
+    const moov = box("moov", box("mvhd", u32(1000)));
+    expect(isFragmentedMp4Head(new Uint8Array([...ftyp, ...moov]))).toBe(false);
+  });
+
+  it("returns false when it is not an mp4", () => {
+    expect(
+      isFragmentedMp4Head(new Uint8Array(enc.encode("not an mp4 file"))),
+    ).toBe(false);
+  });
+});
+
+describe("readTopLevelBoxes", () => {
+  it("walks sequential boxes", () => {
+    const ftyp = box("ftyp", u32(0));
+    const moov = box("moov", []);
+    const boxes = readTopLevelBoxes(new Uint8Array([...ftyp, ...moov]));
+    expect(boxes.map((b) => b.type)).toEqual(["ftyp", "moov"]);
+    expect(boxes[1].start).toBe(ftyp.length);
+    expect(boxes[1].size).toBe(moov.length);
+  });
+});
+
+describe("parseAvcCodec", () => {
+  it("builds avc1.PPCCLL from an avcC box", () => {
+    // avcC payload: version=1, profile=0x4d, compat=0x40, level=0x1f
+    const avcc = box("avcC", [0x01, 0x4d, 0x40, 0x1f, 0xff]);
+    expect(parseAvcCodec(new Uint8Array(avcc))).toBe("avc1.4d401f");
+  });
+
+  it("returns null without an avcC box", () => {
+    expect(parseAvcCodec(new Uint8Array(box("mvhd", u32(1))))).toBeNull();
+  });
+});
+
+describe("parseInitSegment", () => {
+  it("returns init length and codecs for video+audio", () => {
+    const avcc = box("avcC", [0x01, 0x64, 0x00, 0x28]);
+    const stsd = box("stsd", [...box("avc1", avcc), ...box("mp4a", [])]);
+    const moov = box("moov", stsd);
+    const ftyp = box("ftyp", [...Array.from(enc.encode("isom")), ...u32(0)]);
+    const bytes = new Uint8Array([...ftyp, ...moov, 0xaa, 0xbb]); // trailing media
+
+    const parsed = parseInitSegment(bytes);
+    expect(parsed).not.toBeNull();
+    expect(parsed!.initLength).toBe(ftyp.length + moov.length);
+    expect(parsed!.codecs).toBe("avc1.640028,mp4a.40.2");
+    expect(parsed!.hasVideo).toBe(true);
+    expect(parsed!.hasAudio).toBe(true);
+  });
+
+  it("omits audio when no mp4a box is present", () => {
+    const avcc = box("avcC", [0x01, 0x42, 0xc0, 0x1e]);
+    const moov = box("moov", box("stsd", box("avc1", avcc)));
+    const ftyp = box("ftyp", u32(0));
+    const parsed = parseInitSegment(new Uint8Array([...ftyp, ...moov]));
+    expect(parsed!.codecs).toBe("avc1.42c01e");
+    expect(parsed!.hasAudio).toBe(false);
+  });
+
+  it("returns null when moov is truncated", () => {
+    const ftyp = box("ftyp", u32(0));
+    // Declare a moov larger than the bytes provided.
+    const truncatedMoov = [
+      ...u32(999),
+      ...Array.from(enc.encode("moov")),
+      0x00,
+    ];
+    expect(
+      parseInitSegment(new Uint8Array([...ftyp, ...truncatedMoov])),
+    ).toBeNull();
+  });
+});
+
+describe("findMoofOffset", () => {
+  it("finds a validated moof box start (size field), not the ascii marker", () => {
+    const prefix = [0x00, 0x11, 0x22, 0x33];
+    const moof = box("moof", box("mfhd", [0, 0, 0, 1]));
+    const mdat = box("mdat", [0xde, 0xad, 0xbe, 0xef]);
+    const bytes = new Uint8Array([...prefix, ...moof, ...mdat]);
+    expect(findMoofOffset(bytes)).toBe(prefix.length);
+  });
+
+  it("ignores a spurious 'moof' inside mdat payload", () => {
+    // "moof" appears as raw bytes inside media data — must not be treated as a
+    // box boundary (no mfhd child follows).
+    const mdat = box("mdat", [...Array.from(enc.encode("moof")), 1, 2, 3, 4]);
+    expect(findMoofOffset(new Uint8Array(mdat))).toBe(-1);
+  });
+
+  it("returns -1 when no moof is present", () => {
+    expect(findMoofOffset(new Uint8Array(box("mdat", [1, 2, 3])))).toBe(-1);
+  });
+});