gns3-server/gns3server/static/web-ui/assets/xpra-html5/js/VideoDecoder.js

/*
 * This file is part of Xpra.
 * Copyright (C) 2021 Tijs van der Zwaan <tijzwa@vpo.nl>
 * Licensed under MPL 2.0, see:
 * http://www.mozilla.org/MPL/2.0/
 *
 */

/*
 * Receives native video packets and decodes them via VideoDecoder.
 * https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder
 * VideoDecoder is only working in Chrome 94+ and Android
 *
 * Example taken from: https://github.com/w3c/webcodecs/blob/main/explainer.md
 *
 */

const XpraVideoDecoderLoader = {
  hasNativeDecoder() {
    return typeof VideoDecoder !== "undefined";
  },
};

class XpraVideoDecoder {
  constructor() {
    this.initialized = false;
    this.had_first_key = false;
    this.draining = false;

    this.decoder_queue = [];
    this.decoded_frames = [];
    this.erroneous_frame = null;

    this.codec = null;
    this.vp9_params = null;
    this.frameWaitTimeout = 1; // Interval for wait loop while frame is being decoded

    this.frame_threshold = 250;
  }

  prepareVP9params(csc) {
    // Check if we have the right VP9 params before init.
    // This is needed because we can only set the params when the video decoder is created.
    // We close the decoder when the coding changes.
    if (csc === "YUV444P" && this.vp9_params !== ".01.10.08") {
      this.vp9_params = ".01.10.08";
      this._close();
    } else if (csc === "YUV444P10" && this.vp9_params !== ".03.10.10") {
      this.vp9_params = ".03.10.10";
      this._close();
    } else if (this.vp9_params !== ".00.20.08.01.02.02") {
      // chroma shift for YUV420P, both X and Y are downscaled by 2^1
      this.vp9_params = ".00.20.08.01.02.02";
      this._close();
    }
  }

  init(coding) {
    this.draining = false;

    this.codec = this.resolveCodec(coding);
    this.videoDecoder = new VideoDecoder({
      output: this._on_decoded_frame.bind(this),
      error: this._on_decoder_error.bind(this),
    });

    // ToDo: hardwareAcceleration can be "no-preference" / "prefer-hardware" / "prefer-software"
    // Figure out when "prefer-hardware" is the right choise and when not (ie: no GPU, remote session like RDP?)
    this.videoDecoder.configure({
      codec: this.codec,
      hardwareAcceleration: "no-preference",
      optimizeForLatency: true,
      colorSpace: new VideoColorSpace({
        fullRange: true
      })
    });
    this.last_timestamp = 0;
    this.initialized = true;
  }

  resolveCodec(coding) {
    if (coding === "h264") return "avc1.42C01E";
    if (coding === "vp8") return "vp8";
    if (coding === "vp9") return `vp09${this.vp9_params}`;
    throw `No codec defined for coding ${coding}`;
  }

  _on_decoded_frame(videoFrame) {
    if (this.decoder_queue.length === 0) {
      videoFrame.close();
      return;
    }

    // Find the frame
    const frame_timestamp = videoFrame.timestamp;
    let current_frame = this.decoder_queue.filter(
      (q) => q.p[10]["frame"] === frame_timestamp
    );
    if (current_frame.length === 1) {
      // We found our frame!
      this.decoder_queue = this.decoder_queue.filter(
        (q) => q.p[10]["frame"] !== frame_timestamp
      );
      current_frame = current_frame[0];
    } else {
      // We decoded a frame the is no longer queued??
      // TODO: handle error??
      videoFrame.close();
      return;
    }

    if (frame_timestamp === 0) {
      this.last_timestamp = 0;
    }

    if (
      this.decoder_queue.length > this.frame_threshold ||
      this.last_timestamp > frame_timestamp
    ) {
      // Skip if the decoders queue is growing too big or this frames timestamp is smaller then the last one painted.
      videoFrame.close();

      const packet = current_frame.p;
      packet[6] = "throttle";
      packet[7] = null;
      this.decoded_frames.push(packet);
      return;
    }

    this.last_timestamp = frame_timestamp;
    const packet = current_frame.p;

    // Latest possible check for draining
    if (this.draining) {
      videoFrame.close();
      return;
    }

    packet[6] = `frame:${packet[6]}`;
    packet[7] = videoFrame;
    this.decoded_frames.push(packet);
  }

  _on_decoder_error(error) {
    // TODO: Handle error? Or just assume we will catch up?
    this.erroneous_frame = `Error decoding frame: ${error}`;
    console.error(this.erroneous_frame);
  }

  queue_frame(packet) {
    // eslint-disable-next-line no-async-promise-executor
    return new Promise(async (resolve, reject) => {
      const options = packet[10] || {};
      const data = packet[7];
      const packet_sequence = packet[8];

      // H264 (avc1.42C01E) needs key frames
      if (
        this.codec.startsWith("avc1") &&
        !this.had_first_key &&
        !(options["type"] &&
        options["type"] == "IDR")
      ) {
        reject(
          new Error(
            `first h264 frame must be a key frame but packet ${packet_sequence} is not: ${options}`
          )
        );
        return;
      }

      if (this.videoDecoder.state === "closed") {
        reject(new Error("video decoder is closed"));
        return;
      }
      if (this.draining) {
        reject(new Error("video decoder is draining"));
        return;
      }

      this.had_first_key = true;
      this.decoder_queue.push({
        p: packet
      });
      const init = {
        type: options["type"] === "IDR" ? "key" : "delta",
        data,
        timestamp: options["frame"],
      };
      const chunk = new EncodedVideoChunk(init);
      this.videoDecoder.decode(chunk);

      let frame_out = this.decoded_frames.filter(
        (p) => p[8] === packet_sequence
      );
      while (frame_out.length === 0) {
        // Await our frame
        await new Promise((r) => setTimeout(r, this.frameWaitTimeout));
        if (this.erroneous_frame != null) {
          // The last frame was erroneous, break the wait loop
          break;
        }
        frame_out = this.decoded_frames.filter((p) => p[8] === packet_sequence);
      }

      if (this.erroneous_frame != null) {
        // Last frame was erroneous. Reject the promise and reset the state.
        const msg = this.erroneous_frame;
        this.erroneous_frame = null;
        reject(new Error(msg));
      }
      // Remove the frame from decoded frames list
      this.decoded_frames = this.decoded_frames.filter(
        (p) => p[8] !== packet_sequence
      );
      resolve(frame_out[0]);
    });
  }

  _close() {
    if (this.initialized) {
      if (this.videoDecoder.state !== "closed") {
        this.videoDecoder.close();
      }
      this.had_first_key = false;
      this.draining = true;
      this.decoder_queue = [];
    }
    this.initialized = false;
  }
}