Initial commit

README.md

+# SGLang Diffusion Realtime WebUI
+
+Standalone browser demo for `/v1/realtime_video/generate`.
+
+Open `index.html` directly in a browser, point it at an SGLang Diffusion server,
+and generate. The app sends msgpack init / event messages and renders lossless
+raw RGB frame batches on a canvas.
+
+The first version is intentionally static: no npm install, no build step, and no
+server-side dependencies. Presets are UI-side templates for prompt, LingBot
+example images, album artwork references, and session parameters. The default
+preset preloads a reference image so the demo can be tested without a file
+upload.
+
+By default, `Continuous session` is enabled for long-running camera control.
+Keyboard and pointer controls send state transitions instead of scripted preset
+actions. The telemetry `Chunk wait` measures request-to-chunk arrival time, not
+client-side RGB decode time. Continuous playback adapts to the measured chunk
+production rate so the canvas does not play a chunk at target FPS and then sit
+on the last frame while waiting for the next chunk.
+
+The interface shape follows camera-control-first video playgrounds such as
+Reactor LingBot: reference image, scene prompt, enhancement, clip controls,
+move/look camera controls, recordings history, and model telemetry.

decoder_worker.js

+const RAW_RGB_CONTENT_TYPE = "application/x-raw-rgb";
+const RAW_RGB_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgb-delta-gzip";
+const RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip";
+const WEBP_FRAME_CONTENT_TYPE = "image/webp";
+const JPEG_FRAME_CONTENT_TYPE = "image/jpeg";
+
+let lastFrame = null;
+
+function reset() {
+  lastFrame = null;
+}
+
+async function gunzipBytes(payload) {
+  if (typeof DecompressionStream === "undefined") {
+    throw new Error("This browser does not support gzip stream decoding");
+  }
+  const stream = new Blob([payload]).stream().pipeThrough(new DecompressionStream("gzip"));
+  return new Uint8Array(await new Response(stream).arrayBuffer());
+}
+
+async function restoreDeltaGzipFrames(header, payload) {
+  const frameBytes = Number(header.bytes_per_frame);
+  const count = Number(header.num_frames);
+  const expectedSize = frameBytes * count;
+  const restored = await gunzipBytes(payload);
+  if (restored.length !== expectedSize) {
+    throw new Error(`delta payload size mismatch: expected ${expectedSize}, got ${restored.length}`);
+  }
+
+  let previous = header.delta_reference === "previous-frame" ? lastFrame : null;
+  if (header.delta_reference === "previous-frame") {
+    if (!previous) throw new Error("Missing previous frame for delta payload");
+    if (previous.byteLength !== frameBytes) {
+      throw new Error("Previous frame size does not match current delta payload");
+    }
+  }
+
+  for (let f = 0; f < count; f++) {
+    const offset = f * frameBytes;
+    if (previous) {
+      for (let i = 0; i < frameBytes; i++) restored[offset + i] ^= previous[i];
+    }
+    previous = restored.slice(offset, offset + frameBytes);
+  }
+  lastFrame = previous;
+  return restored;
+}
+
+function rawFramesToRgbaBuffers(header, payload) {
+  const width = Number(header.width);
+  const height = Number(header.height);
+  const channels = Number(header.channels);
+  const count = Number(header.num_frames);
+  const frameBytes = Number(header.bytes_per_frame);
+  const pixels = width * height;
+  const buffers = [];
+
+  for (let f = 0; f < count; f++) {
+    const offset = f * frameBytes;
+    if (channels === 4) {
+      buffers.push(payload.buffer.slice(
+        payload.byteOffset + offset,
+        payload.byteOffset + offset + frameBytes,
+      ));
+      continue;
+    }
+
+    const rgba = new Uint8ClampedArray(pixels * 4);
+    let src = offset;
+    let dst = 0;
+    for (let p = 0; p < pixels; p++) {
+      rgba[dst++] = payload[src++];
+      rgba[dst++] = payload[src++];
+      rgba[dst++] = payload[src++];
+      src += channels - 3;
+      rgba[dst++] = 255;
+    }
+    buffers.push(rgba.buffer);
+  }
+  return buffers;
+}
+
+function splitEncodedPayload(header, payload) {
+  const bytes = payload instanceof Uint8Array ? payload : new Uint8Array(payload);
+  const lengths = Array.isArray(header.payload_lengths) && header.payload_lengths.length
+    ? header.payload_lengths.map(Number)
+    : [bytes.byteLength];
+  const payloads = [];
+  let offset = 0;
+  for (const length of lengths) {
+    payloads.push(bytes.buffer.slice(
+      bytes.byteOffset + offset,
+      bytes.byteOffset + offset + length,
+    ));
+    offset += length;
+  }
+  return payloads;
+}
+
+async function encodedFramesToImageBitmaps(header, payload) {
+  if (typeof createImageBitmap === "undefined") {
+    throw new Error("This browser does not support worker image decoding");
+  }
+
+  const frames = await Promise.all(splitEncodedPayload(header, payload).map((framePayload) => (
+    createImageBitmap(new Blob([framePayload], { type: header.content_type }))
+  )));
+  return {
+    width: frames[0]?.width || 0,
+    height: frames[0]?.height || 0,
+    frame_type: "bitmap",
+    frames,
+  };
+}
+
+async function decode(header, payload) {
+  let rawPayload;
+  if (
+    header.content_type === WEBP_FRAME_CONTENT_TYPE ||
+    header.content_type === JPEG_FRAME_CONTENT_TYPE
+  ) {
+    const decoded = await encodedFramesToImageBitmaps(header, payload);
+    return {
+      id: header.__decode_id,
+      width: decoded.width,
+      height: decoded.height,
+      chunk: Number(header.chunk_index),
+      frame_type: decoded.frame_type,
+      frames: decoded.frames,
+    };
+  } else if (header.content_type === RAW_RGB_CONTENT_TYPE) {
+    rawPayload = new Uint8Array(payload);
+    const frameBytes = Number(header.bytes_per_frame);
+    const count = Number(header.num_frames);
+    lastFrame = count > 0
+      ? rawPayload.slice((count - 1) * frameBytes, count * frameBytes)
+      : null;
+  } else if (
+    header.content_type === RAW_RGB_DELTA_GZIP_CONTENT_TYPE ||
+    header.content_type === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE
+  ) {
+    rawPayload = await restoreDeltaGzipFrames(header, payload);
+  } else {
+    throw new Error(`Unsupported content type ${header.content_type}`);
+  }
+
+  return {
+    id: header.__decode_id,
+    width: Number(header.width),
+    height: Number(header.height),
+    chunk: Number(header.chunk_index),
+    frames: rawFramesToRgbaBuffers(header, rawPayload),
+  };
+}
+
+self.onmessage = async (event) => {
+  const message = event.data;
+  try {
+    if (message.type === "reset") {
+      reset();
+      return;
+    }
+    const result = await decode(message.header, message.payload);
+    self.postMessage({ type: "decoded", ...result }, result.frames);
+  } catch (error) {
+    self.postMessage({
+      type: "error",
+      id: message.header?.__decode_id,
+      message: error.message || "decode failed",
+    });
+  }
+};

index.html

+<!doctype html>
+<html lang="zh-CN">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>觉梦·三千世界 实时推演</title>
+    <link rel="stylesheet" href="./styles.css?v=juemeng-v4" />
+  </head>
+  <body>
+    <header class="app-header" aria-label="品牌">
+      <img src="./logo.png" class="app-logo" alt="觉梦 LOGO" />
+      <h1 class="app-title">觉梦·三千世界</h1>
+      <span class="app-subtitle">实时推演Reaction</span>
+    </header>
+    <main class="shell">
+      <section class="panel controls" aria-label="Session controls">
+        <label>服务器<input id="serverUrl" value="ws://114.94.190.4:13669/v1/realtime_video/generate" /></label>
+        <label>模型<input id="model" value="" placeholder="auto from /v1/models" /></label>
+        <div class="section-title">参考图 Reference</div>
+        <label class="reference-upload">
+          <input id="firstFrame" type="file" accept="image/*" />
+          <canvas id="referencePreview" width="320" height="180"></canvas>
+          <span id="referenceName">Preset reference</span>
+        </label>
+        <div class="section-title">场景生成 Generate</div>
+        <label>世界引导词素<textarea id="prompt" rows="4">A cinematic handheld shot of a quiet city street at dusk, soft reflections, natural motion.</textarea></label>
+        <button id="enhanceBtn" class="wide">增强 Enhance</button>
+        <div class="split">
+          <label>尺寸<input id="size" value="832x480" /></label>
+          <label>帧率<input id="fps" type="number" value="25" min="1" max="60" /></label>
+        </div>
+        <div class="split">
+          <label>帧数<input id="numFrames" type="number" value="9" min="5" step="4" /></label>
+          <label>种子<input id="seed" type="number" value="42" /></label>
+        </div>
+        <div class="split">
+          <label>步数<input id="steps" type="number" value="4" min="1" /></label>
+          <label>引导<input id="guidance" type="number" value="1" step="0.1" /></label>
+        </div>
+        <div class="split">
+          <label>Sink<input id="sinkSize" type="number" value="9" min="0" /></label>
+          <label>窗口<input id="windowFrames" type="number" value="18" min="1" /></label>
+        </div>
+        <div class="split">
+          <label>传输
+            <select id="transportFormat">
+              <option value="webp" selected>WebP preview</option>
+              <option value="jpeg">JPEG preview</option>
+              <option value="">Lossless delta</option>
+              <option value="raw">Raw RGB</option>
+            </select>
+          </label>
+          <label>质量<input id="transportQuality" type="number" value="95" min="1" max="100" /></label>
+        </div>
+        <div class="split output-options">
+          <label class="toggle-row"><input id="superResolution" type="checkbox" />超分 SR</label>
+          <label>倍率
+            <select id="upscalingScale">
+              <option value="2" selected>2x</option>
+              <option value="4">4x</option>
+            </select>
+          </label>
+        </div>
+        <label>SR 模型
+          <select id="upscalingModel">
+            <option value="">Quality x2</option>
+            <option
+              value="https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth"
+              selected
+            >
+              Fast general
+            </option>
+            <option value="/scratch/realesr-animevideov3.pth">Fast anime</option>
+          </select>
+        </label>
+        <label class="toggle-row"><input id="frameInterpolation" type="checkbox" />插帧平滑 2x</label>
+        <label class="toggle-row"><input id="continuous" type="checkbox" checked />持续会话 Continuous</label>
+        <div class="actions">
+          <button id="connectBtn" class="primary">开始推演</button>
+          <button id="stopBtn">关闭会话</button>
+        </div>
+        <button id="sendPromptBtn" class="wide">发送提示词更新</button>
+        <div class="section-title">技能编辑 Skills</div>
+        <div id="skillList" class="skill-list"></div>
+        <button id="addSkillBtn" class="wide">＋ 添加技能</button>
+      </section>
+
+      <section class="workspace" aria-label="Realtime workspace">
+        <section class="stage" aria-label="Realtime preview">
+          <div class="topbar">
+            <span id="statusDot" class="dot"></span>
+            <span id="statusText">Idle</span>
+            <span id="chunkText">chunk -</span>
+            <button id="recordBtn" class="record-button" type="button" aria-pressed="false" title="世界快照">
+              <span class="record-button-icon" aria-hidden="true"></span>
+              <span id="recordLabel">世界快照</span>
+              <span id="recordDuration" class="record-button-duration">00:00</span>
+            </button>
+            <span class="topbar-spacer"></span>
+            <label class="preview-scale-control">世界时钟
+              <input id="previewScale" type="range" min="80" max="170" value="120" />
+              <b id="previewScaleText">120%</b>
+            </label>
+            <span class="stage-stat">相界倍率 <b id="outputSizeText"></b>3960x6480</b></span>
+            <span class="stage-stat">成相体征 <b id="renderFps">0</b> 生命体</span>
+            <span class="stage-stat">源质 <b id="theoreticalFpsText">-</b></span>
+            <span class="stage-stat">世界缓存</span> <b id="stageLatencyText">-</b></span>
+          </div>
+          <div class="preview-frame">
+            <canvas id="viewport" width="1280" height="720"></canvas>
+            <div id="previewOverlay" class="preview-overlay" aria-hidden="true">
+              <span class="preview-loader"></span>
+            </div>
+          </div>
+          <div class="stage-controls" aria-label="Camera controls">
+            <div class="control-cluster move-cluster" aria-label="移动">
+              <span class="control-title">移动 MOVE</span>
+              <div class="wasd-pad">
+                <button data-action="w" data-key="W" class="wasd wasd-w" aria-label="前进">W</button>
+                <button data-action="a" data-key="A" class="wasd wasd-a" aria-label="左移">A</button>
+                <button data-action="s" data-key="S" class="wasd wasd-s" aria-label="后退">S</button>
+                <button data-action="d" data-key="D" class="wasd wasd-d" aria-label="右移">D</button>
+              </div>
+            </div>
+            <div class="control-cluster attack-cluster" aria-label="技能">
+              <span class="control-title">技能 SKILLS</span>
+              <div id="skillBar" class="skill-bar"></div>
+              <span class="attack-hint">左侧编辑区可自定义技能与快捷键</span>
+            </div>
+            <div class="control-cluster look-cluster" aria-label="视角">
+              <span class="control-title">视角 LOOK</span>
+              <div id="lookStick" class="joystick" aria-label="视角摇杆" role="application">
+                <button data-action="i" data-key="↑" class="joy-zone zone-up" tabindex="-1" aria-hidden="true">▲</button>
+                <button data-action="j" data-key="←" class="joy-zone zone-left" tabindex="-1" aria-hidden="true">◀</button>
+                <button data-action="k" data-key="↓" class="joy-zone zone-down" tabindex="-1" aria-hidden="true">▼</button>
+                <button data-action="l" data-key="→" class="joy-zone zone-right" tabindex="-1" aria-hidden="true">▶</button>
+                <div class="joystick-ring" aria-hidden="true"></div>
+                <div id="lookKnob" class="joystick-knob" aria-hidden="true"></div>
+              </div>
+              <span class="joy-hint">拖拽画面 · 或拨动摇杆</span>
+            </div>
+          </div>
+          <div class="timeline">
+            <span id="queueText">queue 0</span>
+            <span id="frameText">frames 0</span>
+            <span id="byteText">0 MB</span>
+          </div>
+          <div class="telemetry stage-telemetry">
+            <span>Payload<b id="payloadMode">webp</b></span>
+            <span>Server send<b id="serverSendText">-</b></span>
+            <span>Chunk bytes<b id="chunkPayloadText">-</b></span>
+            <span>Chunk wait<b id="latencyText">-</b></span>
+            <span>Decode<b id="decodeText">-</b></span>
+            <span>Display lag<b id="displayLagText">-</b></span>
+          </div>
+        </section>
+
+        <section class="panel presets" aria-label="Presets and camera">
+          <div class="section-title">引擎规格 Engine</div>
+          <div class="spec-grid">
+            <span><b>25 fps</b> target</span>
+            <span><b>chunked</b> stream</span>
+            <span><b>480p/720p</b></span>
+            <span><b>Cam + Act</b></span>
+          </div>
+          <div class="section-title">场景预设 Presets</div>
+          <div id="presetList" class="preset-list"></div>
+          <div class="section-title">推演历史 History</div>
+          <div id="historyList" class="history-list"></div>
+        </section>
+      </section>
+    </main>
+  <script src="./playback_controller.js?v=realtime-playback-v13"></script>
+  <script src="./app.js?v=juemeng-v4"></script>
+</body>
+</html>

playback_controller_test.js

+const assert = require("node:assert/strict");
+const { RealtimePlaybackController } = require("./playback_controller.js");
+
+function frames(count, chunk) {
+  return Array.from({ length: count }, (_, index) => ({
+    image: { close() {} },
+    chunk,
+    index,
+  }));
+}
+
+function enqueueChunk(controller, {
+  chunk,
+  eventId = 0,
+  frameCount = 12,
+  durationMs = 480,
+  now,
+}) {
+  controller.observeServerStats({
+    chunk_index: chunk,
+    num_frames: frameCount,
+    chunk_total_ms: durationMs,
+  }, now);
+  return controller.enqueueDecodedFrames({
+    chunk_index: chunk,
+    event_id: eventId,
+    num_frames: frameCount,
+    __received_at: now,
+    is_final_frame_batch: true,
+  }, frames(frameCount, chunk), now);
+}
+
+function renderFor(controller, startMs, durationMs) {
+  let rendered = 0;
+  for (let now = startMs; now <= startMs + durationMs; now += 16.67) {
+    const decision = controller.render(now, { hasPendingInput: true });
+    if (decision.action === "draw") rendered += 1;
+  }
+  return rendered;
+}
+
+function stableSourceDoesNotDrop() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  let now = 0;
+  for (let chunk = 0; chunk < 8; chunk += 1) {
+    now += 480;
+    enqueueChunk(controller, { chunk, now });
+    renderFor(controller, now, 480);
+  }
+  const snapshot = controller.snapshot();
+  assert.equal(snapshot.droppedFrames, 0);
+  assert.ok(snapshot.sourceFps > 24 && snapshot.sourceFps <= 25);
+}
+
+function slowServerCapsRenderFps() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  let now = 0;
+  for (let chunk = 0; chunk < 8; chunk += 1) {
+    now += 1360;
+    enqueueChunk(controller, { chunk, durationMs: 1360, now });
+    renderFor(controller, now, 1360);
+  }
+  const snapshot = controller.snapshot();
+  assert.ok(snapshot.sourceFps > 8 && snapshot.sourceFps < 10);
+  assert.ok(snapshot.renderFps <= 10);
+}
+
+function backlogDropsContiguousOldFrames() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  let now = 100;
+  for (let chunk = 0; chunk < 13; chunk += 1) {
+    enqueueChunk(controller, { chunk, now, durationMs: 480 });
+    now += 20;
+  }
+  const snapshot = controller.snapshot();
+  assert.ok(snapshot.droppedFrames > 0);
+  assert.equal(snapshot.lastDropReason, "backlog");
+}
+
+function eventCutoverKeepsShortGrace() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  enqueueChunk(controller, { chunk: 1, frameCount: 24, durationMs: 960, now: 1000 });
+  controller.noteInputEvent(5, 1050);
+  const result = enqueueChunk(controller, {
+    chunk: 2,
+    eventId: 5,
+    frameCount: 12,
+    durationMs: 480,
+    now: 1150,
+  });
+  assert.ok(result.cutover);
+  assert.ok(result.droppedFrames.length >= 14);
+  assert.equal(controller.queue[0].chunk, 1);
+  assert.equal(controller.queue[0].index, 0);
+}
+
+function settleEventCutoverKeepsWiderGrace() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  enqueueChunk(controller, { chunk: 1, frameCount: 24, durationMs: 960, now: 1000 });
+  controller.noteInputEvent(5, 1050, { cutoverMode: "settle" });
+  const result = enqueueChunk(controller, {
+    chunk: 2,
+    eventId: 5,
+    frameCount: 12,
+    durationMs: 480,
+    now: 1150,
+  });
+  assert.ok(result.cutover);
+  assert.ok(result.droppedFrames.length <= 12);
+}
+
+stableSourceDoesNotDrop();
+slowServerCapsRenderFps();
+backlogDropsContiguousOldFrames();
+eventCutoverKeepsShortGrace();
+settleEventCutoverKeepsWiderGrace();