Merge pull request #519 from movie-web/chromecasting-and-more

Chromecasting and more

src/assets/locales/en.json

@@ -94,6 +94,9 @@
         "failure": "Error occurred"
       }
     },
+    "casting": {
+      "enabled": "Casting to device..."
+    },
     "playbackError": {
       "badge": "Playback error",
       "title": "Failed to play video!",

src/components/overlays/OverlayDisplay.tsx

@@ -70,6 +70,11 @@ export function OverlayPortal(props: {
                     className="absolute inset-0 pointer-events-none"
                     isChild
                   >
+                    {/* a tabable index that does nothing - used so focus trap doesn't error when nothing is rendered yet */}
+                    <div
+                      tabIndex={1}
+                      className="focus:ring-0 focus:outline-none opacity-0"
+                    />
                     {props.children}
                   </Transition>
                 </div>

src/components/player/atoms/CastingNotification.tsx

@@ -0,0 +1,22 @@
+import { useTranslation } from "react-i18next";
+
+import { Icon, Icons } from "@/components/Icon";
+import { usePlayerStore } from "@/stores/player/store";
+
+export function CastingNotification() {
+  const { t } = useTranslation();
+  const isLoading = usePlayerStore((s) => s.mediaPlaying.isLoading);
+  const display = usePlayerStore((s) => s.display);
+  const isCasting = display?.getType() === "casting";
+
+  if (isLoading || !isCasting) return null;
+
+  return (
+    <div className="flex flex-col items-center justify-center gap-4">
+      <div className="rounded-full bg-opacity-10 bg-video-buttonBackground p-3 brightness-100 grayscale">
+        <Icon icon={Icons.CASTING} />
+      </div>
+      <p className="text-center">{t("player.casting.enabled")}</p>
+    </div>
+  );
+}

src/components/player/atoms/index.ts

@@ -15,3 +15,4 @@ export * from "./Airplay";
 export * from "./VolumeChangedPopout";
 export * from "./NextEpisodeButton";
 export * from "./Chromecast";
+export * from "./CastingNotification";

src/components/player/base/SubtitleView.tsx

@@ -107,8 +107,10 @@ export function SubtitleRenderer() {
 export function SubtitleView(props: { controlsShown: boolean }) {
   const caption = usePlayerStore((s) => s.caption.selected);
   const captionAsTrack = usePlayerStore((s) => s.caption.asTrack);
+  const display = usePlayerStore((s) => s.display);
+  const isCasting = display?.getType() === "casting";
 
-  if (captionAsTrack || !caption) return null;
+  if (captionAsTrack || !caption || isCasting) return null;
 
   return (
     <Transition

src/components/player/display/base.ts

@@ -253,6 +253,9 @@ export function makeVideoElementDisplayInterface(): DisplayInterface {
   return {
     on,
     off,
+    getType() {
+      return "web";
+    },
     destroy: () => {
       destroyVideoElement();
       fscreen.removeEventListener("fullscreenchange", fullscreenChange);
@@ -282,6 +285,8 @@ export function makeVideoElementDisplayInterface(): DisplayInterface {
     processContainerElement(container) {
       containerElement = container;
     },
+    setMeta() {},
+    setCaption() {},
 
     pause() {
       videoElement?.pause();

src/components/player/display/chromecast.ts

@@ -1,8 +1,11 @@
 import fscreen from "fscreen";
 
+import { MWMediaType } from "@/backend/metadata/types/mw";
 import {
+  DisplayCaption,
   DisplayInterface,
   DisplayInterfaceEvents,
+  DisplayMeta,
 } from "@/components/player/display/displayInterface";
 import { LoadableSource } from "@/stores/player/utils/qualities";
 import {
@@ -18,13 +21,17 @@ export interface ChromeCastDisplayInterfaceOptions {
   instance: cast.framework.CastContext;
 }
 
-// TODO check all functionality
+/*
-// TODO listen for events to update the state
+ ** Chromecasting is unfinished, here is its limitations:
+ **  1. Captions - chromecast requires only VTT, but needs it from a URL. we only have SRT urls
+ **  2. HLS - we've having some issues with content types. sometimes it loads, sometimes it doesn't
+ */
+
 export function makeChromecastDisplayInterface(
   ops: ChromeCastDisplayInterfaceOptions
 ): DisplayInterface {
   const { emit, on, off } = makeEmitter<DisplayInterfaceEvents>();
-  const isPaused = false;
+  let isPaused = false;
   let playbackRate = 1;
   let source: LoadableSource | null = null;
   let videoElement: HTMLVideoElement | null = null;
@@ -33,8 +40,64 @@ export function makeChromecastDisplayInterface(
   let isPausedBeforeSeeking = false;
   let isSeeking = false;
   let startAt = 0;
-  // let automaticQuality = false;
+  let meta: DisplayMeta = {
-  // let preferenceQuality: SourceQuality | null = null;
+    title: "",
+    type: MWMediaType.MOVIE,
+  };
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  let caption: DisplayCaption | null = null;
+
+  function listenForEvents() {
+    const listen = async (e: cast.framework.RemotePlayerChangedEvent) => {
+      switch (e.field) {
+        case "volumeLevel":
+          if (await canChangeVolume()) emit("volumechange", e.value);
+          break;
+        case "currentTime":
+          emit("time", e.value);
+          break;
+        case "duration":
+          emit("duration", e.value ?? 0);
+          break;
+        case "mediaInfo":
+          if (e.value) emit("duration", e.value.duration ?? 0);
+          break;
+        case "playerState":
+          emit("loading", e.value === "BUFFERING");
+          if (e.value === "PLAYING") emit("play", undefined);
+          else if (e.value === "PAUSED") emit("pause", undefined);
+          isPaused = e.value === "PAUSED";
+          break;
+        case "isMuted":
+          emit("volumechange", e.value ? 1 : 0);
+          break;
+        case "displayStatus":
+        case "canSeek":
+        case "title":
+        case "isPaused":
+        case "canPause":
+        case "isMediaLoaded":
+        case "statusText":
+        case "isConnected":
+        case "displayName":
+        case "canControlVolume":
+        case "savedPlayerState":
+          break;
+        default:
+          break;
+      }
+    };
+    ops.controller?.addEventListener(
+      cast.framework.RemotePlayerEventType.ANY_CHANGE,
+      listen
+    );
+    return () => {
+      ops.controller?.removeEventListener(
+        cast.framework.RemotePlayerEventType.ANY_CHANGE,
+        listen
+      );
+    };
+  }
 
   function setupSource() {
     if (!source) {
@@ -42,30 +105,33 @@ export function makeChromecastDisplayInterface(
       return;
     }
 
-    if (source.type === "hls") {
+    let type = "video/mp4";
-      // TODO hls support
+    if (source.type === "hls") type = "application/x-mpegurl";
-      return;
-    }
 
-    // TODO movie meta
+    const metaData = new chrome.cast.media.GenericMediaMetadata();
-    const movieMeta = new chrome.cast.media.MovieMediaMetadata();
+    metaData.title = meta.title;
-    movieMeta.title = "";
 
-    const mediaInfo = new chrome.cast.media.MediaInfo("video", "video/mp4");
+    const mediaInfo = new chrome.cast.media.MediaInfo("video", type);
     (mediaInfo as any).contentUrl = source.url;
     mediaInfo.streamType = chrome.cast.media.StreamType.BUFFERED;
-    mediaInfo.metadata = movieMeta;
+    mediaInfo.metadata = metaData;
     mediaInfo.customData = {
       playbackRate,
     };
 
     const request = new chrome.cast.media.LoadRequest(mediaInfo);
     request.autoplay = true;
+    request.currentTime = startAt;
+
+    if (source.type === "hls") {
+      const staticMedia = chrome.cast.media as any;
+      const media = request.media as any;
+      media.hlsSegmentFormat = staticMedia.HlsSegmentFormat.FMP4;
+      media.hlsVideoSegmentFormat = staticMedia.HlsVideoSegmentFormat.FMP4;
+    }
 
-    ops.player.currentTime = startAt;
     const session = ops.instance.getCurrentSession();
     session?.loadMedia(request);
-    ops.controller.seek();
   }
 
   function setSource() {
@@ -86,25 +152,32 @@ export function makeChromecastDisplayInterface(
   }
   fscreen.addEventListener("fullscreenchange", fullscreenChange);
 
+  // start listening immediately
+  const stopListening = listenForEvents();
+
   return {
     on,
     off,
+    getType() {
+      return "casting";
+    },
     destroy: () => {
+      stopListening();
       destroyVideoElement();
       fscreen.removeEventListener("fullscreenchange", fullscreenChange);
     },
     load(loadOps) {
-      // automaticQuality = loadOps.automaticQuality;
-      // preferenceQuality = loadOps.preferredQuality;
       source = loadOps.source;
       emit("loading", true);
       startAt = loadOps.startAt;
       setSource();
     },
-    changeQuality(_newAutomaticQuality, _newPreferredQuality) {
+    changeQuality() {
-      // if (source?.type !== "hls") return;
+      // cant control qualities
-      // automaticQuality = newAutomaticQuality;
+    },
-      // preferenceQuality = newPreferredQuality;
+    setCaption(newCaption) {
+      caption = newCaption;
+      setSource();
     },
 
     processVideoElement(video) {
@@ -115,12 +188,22 @@ export function makeChromecastDisplayInterface(
     processContainerElement(container) {
       containerElement = container;
     },
+    setMeta(data) {
+      meta = data;
+      setSource();
+    },
 
     pause() {
-      if (!isPaused) ops.controller.playOrPause();
+      if (!isPaused) {
+        ops.controller.playOrPause();
+        isPaused = true;
+      }
     },
     play() {
-      if (isPaused) ops.controller.playOrPause();
+      if (isPaused) {
+        ops.controller.playOrPause();
+        isPaused = false;
+      }
     },
     setSeeking(active) {
       if (active === isSeeking) return;
@@ -156,6 +239,7 @@ export function makeChromecastDisplayInterface(
       if (isChangeable) {
         ops.player.volumeLevel = volume;
         ops.controller.setVolumeLevel();
+        emit("volumechange", volume);
       } else {
         // For browsers where it can't be changed
         emit("volumechange", volume === 0 ? 0 : 1);

src/components/player/display/displayInterface.ts

@@ -1,3 +1,4 @@
+import { MWMediaType } from "@/backend/metadata/types/mw";
 import { LoadableSource, SourceQuality } from "@/stores/player/utils/qualities";
 import { Listener } from "@/utils/events";
 
@@ -34,6 +35,19 @@ export interface qualityChangeOptions {
   startAt: number;
 }
 
+export interface DisplayMeta {
+  title: string;
+  type: MWMediaType;
+}
+
+export interface DisplayCaption {
+  srtData: string;
+  language: string;
+  url?: string;
+}
+
+export type DisplayType = "web" | "casting";
+
 export interface DisplayInterface extends Listener<DisplayInterfaceEvents> {
   play(): void;
   pause(): void;
@@ -52,4 +66,7 @@ export interface DisplayInterface extends Listener<DisplayInterfaceEvents> {
   destroy(): void;
   startAirplay(): void;
   setPlaybackRate(rate: number): void;
+  setMeta(meta: DisplayMeta): void;
+  setCaption(caption: DisplayCaption | null): void;
+  getType(): DisplayType;
 }

src/components/player/internals/CastingInternal.tsx

@@ -1,5 +1,6 @@
 import { useEffect, useRef } from "react";
 
+import { mediaItemTypeToMediaType } from "@/backend/metadata/tmdb";
 import { makeVideoElementDisplayInterface } from "@/components/player/display/base";
 import { makeChromecastDisplayInterface } from "@/components/player/display/chromecast";
 import { useChromecastAvailable } from "@/hooks/useChromecastAvailable";
@@ -11,26 +12,39 @@ export function CastingInternal() {
   const setPlayer = usePlayerStore((s) => s.casting.setPlayer);
   const setIsCasting = usePlayerStore((s) => s.casting.setIsCasting);
   const isCasting = usePlayerStore((s) => s.interface.isCasting);
+  const caption = usePlayerStore((s) => s.caption?.selected);
   const setDisplay = usePlayerStore((s) => s.setDisplay);
   const redisplaySource = usePlayerStore((s) => s.redisplaySource);
   const available = useChromecastAvailable();
+  const display = usePlayerStore((s) => s.display);
 
   const controller = usePlayerStore((s) => s.casting.controller);
   const player = usePlayerStore((s) => s.casting.player);
   const instance = usePlayerStore((s) => s.casting.instance);
+  const time = usePlayerStore((s) => s.progress.time);
+  const metaTitle = usePlayerStore((s) => s.meta?.title);
+  const metaType = usePlayerStore((s) => s.meta?.type);
 
   const dataRef = useRef({
     controller,
     player,
     instance,
+    time,
+    metaTitle,
+    metaType,
+    caption,
   });
   useEffect(() => {
     dataRef.current = {
       controller,
       player,
       instance,
+      time,
+      metaTitle,
+      metaType,
+      caption,
     };
-  }, [controller, player, instance]);
+  }, [controller, player, instance, time, metaTitle, metaType, caption]);
 
   useEffect(() => {
     if (isCasting) {
@@ -44,15 +58,28 @@ export function CastingInternal() {
           instance: dataRef.current.instance,
           player: dataRef.current.player,
         });
+        newDisplay.setMeta({
+          title: dataRef.current.metaTitle ?? "",
+          type: mediaItemTypeToMediaType(dataRef.current.metaType ?? "movie"),
+        });
+        newDisplay.setCaption(dataRef.current.caption);
         setDisplay(newDisplay);
-        redisplaySource(0); // TODO right start time
+        redisplaySource(dataRef.current.time ?? 0);
       }
     } else {
       const newDisplay = makeVideoElementDisplayInterface();
       setDisplay(newDisplay);
+      redisplaySource(dataRef.current.time ?? 0);
     }
   }, [isCasting, setDisplay, redisplaySource]);
 
+  useEffect(() => {
+    display?.setMeta({
+      title: dataRef.current.metaTitle ?? "",
+      type: mediaItemTypeToMediaType(dataRef.current.metaType ?? "movie"),
+    });
+  }, [metaTitle, metaType, display]);
+
   useEffect(() => {
     if (!available) return;
 

src/components/player/internals/ThumbnailScraper.tsx

@@ -92,6 +92,7 @@ class ThumnbnailWorker {
     );
     const imgUrl = this.canvasEl.toDataURL();
     if (this.interrupted) return;
+    if (imgUrl === "data:," || !imgUrl) return; // failed image rendering
 
     this.cb({
       at,

src/pages/parts/player/PlayerPart.tsx

@@ -31,10 +31,15 @@ export function PlayerPart(props: PlayerPartProps) {
       <Player.SubtitleView controlsShown={showTargets} />
 
       {status === playerStatus.PLAYING ? (
-        <Player.CenterControls>
+        <>
-          <Player.LoadingSpinner />
+          <Player.CenterControls>
-          <Player.AutoPlayStart />
+            <Player.LoadingSpinner />
-        </Player.CenterControls>
+            <Player.AutoPlayStart />
+          </Player.CenterControls>
+          <Player.CenterControls>
+            <Player.CastingNotification />
+          </Player.CenterControls>
+        </>
       ) : null}
 
       <Player.CenterMobileControls

src/stores/player/slices/source.ts

@@ -132,6 +132,8 @@ export const createSourceSlice: MakeSlice<SourceSlice> = (set, get) => ({
     });
   },
   setCaption(caption) {
+    const store = get();
+    store.display?.setCaption(caption);
     set((s) => {
       s.caption.selected = caption;
     });