thumbnail scraping

src/components/player/atoms/ProgressBar.tsx

@@ -1,8 +1,48 @@
-import { useCallback, useEffect, useRef } from "react";
+import {
+  MouseEvent,
+  RefObject,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
 
 import { useProgressBar } from "@/hooks/useProgressBar";
+import { nearestImageAt } from "@/stores/player/slices/thumbnails";
 import { usePlayerStore } from "@/stores/player/store";
 
+function ThumbnailDisplay(props: { at: number }) {
+  const thumbnailImages = usePlayerStore((s) => s.thumbnails.images);
+  const currentThumbnail = useMemo(() => {
+    return nearestImageAt(thumbnailImages, props.at)?.image;
+  }, [thumbnailImages, props.at]);
+
+  if (!currentThumbnail) return null;
+  return <img src={currentThumbnail.data} className="h-12" />;
+}
+
+function useMouseHoverPosition(barRef: RefObject<HTMLDivElement>) {
+  const [mousePos, setMousePos] = useState(-1);
+
+  const mouseMove = useCallback(
+    (e: MouseEvent<HTMLDivElement>) => {
+      const bar = barRef.current;
+      if (!bar) return;
+      const rect = barRef.current.getBoundingClientRect();
+      const pos = (e.pageX - rect.left) / barRef.current.offsetWidth;
+      setMousePos(pos * 100);
+    },
+    [setMousePos, barRef]
+  );
+
+  const mouseLeave = useCallback(() => {
+    setMousePos(-1);
+  }, [setMousePos]);
+
+  return { mousePos, mouseMove, mouseLeave };
+}
+
 export function ProgressBar() {
   const { duration, time, buffered } = usePlayerStore((s) => s.progress);
   const display = usePlayerStore((s) => s.display);
@@ -18,6 +58,7 @@ export function ProgressBar() {
   );
 
   const ref = useRef<HTMLDivElement>(null);
+  const { mouseMove, mouseLeave, mousePos } = useMouseHoverPosition(ref);
 
   const { dragging, dragPercentage, dragMouseDown } = useProgressBar(
     ref,
@@ -31,45 +72,67 @@ export function ProgressBar() {
     setDraggingTime((dragPercentage / 100) * duration);
   }, [setDraggingTime, duration, dragPercentage]);
 
-  return (
-    <div className="w-full" ref={ref}>
-      <div
-        className="group w-full h-8 flex items-center cursor-pointer"
-        onMouseDown={dragMouseDown}
-        onTouchStart={dragMouseDown}
-      >
-        <div
-          className={[
-            "relative w-full h-1 bg-video-progress-background bg-opacity-25 rounded-full transition-[height] duration-100 group-hover:h-1.5",
-            dragging ? "!h-1.5" : "",
-          ].join(" ")}
-        >
-          {/* Pre-loaded content bar */}
-          <div
-            className="absolute top-0 left-0 h-full rounded-full bg-video-progress-preloaded bg-opacity-50 flex justify-end items-center"
-            style={{
-              width: `${(buffered / duration) * 100}%`,
-            }}
-          />
+  const mousePosition = Math.floor(dragPercentage * duration);
 
-          {/* Actual progress bar */}
+  return (
+    <div className="w-full relative">
+      <div className="top-0 absolute inset-x-0">
+        {mousePos > -1 ? (
           <div
-            className="absolute top-0 left-0 h-full rounded-full bg-video-progress-watched flex justify-end items-center"
+            className="absolute bottom-0"
             style={{
-              width: `${
-                Math.max(
-                  0,
-                  Math.min(1, dragging ? dragPercentage / 100 : time / duration)
-                ) * 100
-              }%`,
+              left: `${mousePos}%`,
             }}
           >
+            <ThumbnailDisplay at={mousePosition} />
+          </div>
+        ) : null}
+      </div>
+
+      <div className="w-full" ref={ref}>
+        <div
+          className="group w-full h-8 flex items-center cursor-pointer"
+          onMouseDown={dragMouseDown}
+          onTouchStart={dragMouseDown}
+          onMouseLeave={mouseLeave}
+          onMouseMove={mouseMove}
+        >
+          <div
+            className={[
+              "relative w-full h-1 bg-video-progress-background bg-opacity-25 rounded-full transition-[height] duration-100 group-hover:h-1.5",
+              dragging ? "!h-1.5" : "",
+            ].join(" ")}
+          >
+            {/* Pre-loaded content bar */}
             <div
-              className={[
-                "w-[1rem] min-w-[1rem] h-[1rem] rounded-full transform translate-x-1/2 scale-0 group-hover:scale-100 bg-white transition-[transform] duration-100",
-                isSeeking ? "scale-100" : "",
-              ].join(" ")}
+              className="absolute top-0 left-0 h-full rounded-full bg-video-progress-preloaded bg-opacity-50 flex justify-end items-center"
+              style={{
+                width: `${(buffered / duration) * 100}%`,
+              }}
             />
+
+            {/* Actual progress bar */}
+            <div
+              className="absolute top-0 left-0 h-full rounded-full bg-video-progress-watched flex justify-end items-center"
+              style={{
+                width: `${
+                  Math.max(
+                    0,
+                    Math.min(
+                      1,
+                      dragging ? dragPercentage / 100 : time / duration
+                    )
+                  ) * 100
+                }%`,
+              }}
+            >
+              <div
+                className={[
+                  "w-[1rem] min-w-[1rem] h-[1rem] rounded-full transform translate-x-1/2 scale-0 group-hover:scale-100 bg-white transition-[transform] duration-100",
+                  isSeeking ? "scale-100" : "",
+                ].join(" ")}
+              />
+            </div>
           </div>
         </div>
       </div>

src/components/player/base/Container.tsx

@@ -5,6 +5,7 @@ import { CastingInternal } from "@/components/player/internals/CastingInternal";
 import { HeadUpdater } from "@/components/player/internals/HeadUpdater";
 import { KeyboardEvents } from "@/components/player/internals/KeyboardEvents";
 import { ProgressSaver } from "@/components/player/internals/ProgressSaver";
+import { ThumbnailScraper } from "@/components/player/internals/ThumbnailScraper";
 import { VideoClickTarget } from "@/components/player/internals/VideoClickTarget";
 import { VideoContainer } from "@/components/player/internals/VideoContainer";
 import { PlayerHoverState } from "@/stores/player/slices/interface";
@@ -82,6 +83,7 @@ export function Container(props: PlayerProps) {
   return (
     <div className="relative">
       <BaseContainer>
+        <ThumbnailScraper />
         <CastingInternal />
         <VideoContainer />
         <ProgressSaver />

src/components/player/internals/ThumbnailScraper.tsx

@@ -0,0 +1,127 @@
+import Hls from "hls.js";
+import { useEffect, useMemo, useRef } from "react";
+
+import { ThumbnailImage } from "@/stores/player/slices/thumbnails";
+import { usePlayerStore } from "@/stores/player/store";
+import { LoadableSource, selectQuality } from "@/stores/player/utils/qualities";
+
+class ThumnbnailWorker {
+  interrupted: boolean;
+
+  videoEl: HTMLVideoElement | null = null;
+
+  canvasEl: HTMLCanvasElement | null = null;
+
+  hls: Hls | null = null;
+
+  cb: (img: ThumbnailImage) => void;
+
+  constructor(ops: { addImage: (img: ThumbnailImage) => void }) {
+    this.cb = ops.addImage;
+    this.interrupted = false;
+  }
+
+  start(source: LoadableSource) {
+    const el = document.createElement("video");
+    const canvas = document.createElement("canvas");
+    this.hls = new Hls();
+    if (source.type === "mp4") {
+      el.src = source.url;
+      el.crossOrigin = "anonymous";
+    } else if (source.type === "hls") {
+      this.hls.attachMedia(el);
+      this.hls.loadSource(source.url);
+    } else throw new Error("Invalid loadable source type");
+    this.videoEl = el;
+    this.canvasEl = canvas;
+    this.begin().catch((err) => console.error(err));
+  }
+
+  destroy() {
+    this.hls?.detachMedia();
+    this.hls?.destroy();
+    this.hls = null;
+    this.interrupted = true;
+    this.videoEl = null;
+    this.canvasEl = null;
+  }
+
+  private async initVideo() {
+    if (!this.videoEl || !this.canvasEl) return;
+    await new Promise((resolve, reject) => {
+      this.videoEl?.addEventListener("loadedmetadata", resolve);
+      this.videoEl?.addEventListener("error", reject);
+    });
+    if (!this.videoEl || !this.canvasEl) return;
+    this.canvasEl.height = this.videoEl.videoHeight;
+    this.canvasEl.width = this.videoEl.videoWidth;
+  }
+
+  private async takeSnapshot(at: number) {
+    if (!this.videoEl || !this.canvasEl) return;
+    this.videoEl.currentTime = at;
+    await new Promise((resolve) => {
+      this.videoEl?.addEventListener("seeked", resolve);
+    });
+    if (!this.videoEl || !this.canvasEl) return;
+    const ctx = this.canvasEl.getContext("2d");
+    if (!ctx) return;
+    ctx.drawImage(
+      this.videoEl,
+      0,
+      0,
+      this.canvasEl.width,
+      this.canvasEl.height
+    );
+    const imgUrl = this.canvasEl.toDataURL();
+    this.cb({
+      at,
+      data: imgUrl,
+    });
+  }
+
+  private async begin() {
+    const vid = this.videoEl;
+    if (!vid) return;
+    await this.initVideo();
+    if (this.interrupted) return;
+    await this.takeSnapshot(vid.duration / 2);
+  }
+}
+
+export function ThumbnailScraper() {
+  const addImage = usePlayerStore((s) => s.thumbnails.addImage);
+  const source = usePlayerStore((s) => s.source);
+  const workerRef = useRef<ThumnbnailWorker | null>(null);
+
+  const inputStream = useMemo(() => {
+    if (!source) return null;
+    return selectQuality(source, {
+      automaticQuality: false,
+      lastChosenQuality: "360",
+    });
+  }, [source]);
+
+  // TODO stop worker on meta change
+
+  // start worker with the stream
+  useEffect(() => {
+    // dont interrupt existing working
+    if (workerRef.current) return;
+    if (!inputStream) return;
+    const ins = new ThumnbnailWorker({
+      addImage,
+    });
+    workerRef.current = ins;
+    ins.start(inputStream.stream);
+  }, [inputStream, addImage]);
+
+  // destroy worker on unmount
+  useEffect(() => {
+    return () => {
+      if (workerRef.current) workerRef.current.destroy();
+    };
+  }, []);
+
+  return null;
+}

src/stores/player/slices/thumbnails.ts

@@ -0,0 +1,106 @@
+import { MakeSlice } from "@/stores/player/slices/types";
+
+export interface ThumbnailImage {
+  at: number;
+  data: string;
+}
+
+export interface ThumbnailSlice {
+  thumbnails: {
+    images: ThumbnailImage[];
+    addImage(img: ThumbnailImage): void;
+  };
+}
+
+export interface ThumbnailImagePosition {
+  index: number;
+  image: ThumbnailImage;
+}
+
+/**
+ * get nearest image at the timestamp provided
+ * @param images images, must be sorted
+ */
+export function nearestImageAt(
+  images: ThumbnailImage[],
+  at: number
+): ThumbnailImagePosition | null {
+  // no images, early return
+  if (images.length === 0) return null;
+
+  const indexPastTimestamp = images.findIndex((v) => v.at < at);
+
+  // no image found past timestamp, so last image must be closest
+  if (indexPastTimestamp === -1)
+    return {
+      index: images.length - 1,
+      image: images[images.length - 1],
+    };
+
+  const imagePastTimestamp = images[indexPastTimestamp];
+
+  // if past timestamp is first image, just return that image
+  if (indexPastTimestamp === 0)
+    return {
+      index: indexPastTimestamp,
+      image: imagePastTimestamp,
+    };
+
+  //             distance before             distance past
+  //                    |                          |
+  //  [before] --------------------- [at] --------------------- [past]
+  const imageBeforeTimestamp = images[indexPastTimestamp - 1];
+  const distanceBefore = at - imageBeforeTimestamp.at;
+  const distancePast = imagePastTimestamp.at - at;
+
+  // if distance of before timestamp is smaller than the distance past
+  // before is closer, return that
+  //  [before] --X-------------- [past]
+  if (distanceBefore < distancePast)
+    return {
+      index: indexPastTimestamp - 1,
+      image: imageBeforeTimestamp,
+    };
+
+  // must be closer to past here, return past
+  //  [before] --------------X-- [past]
+  return {
+    index: indexPastTimestamp,
+    image: imagePastTimestamp,
+  };
+}
+
+export const createThumbnailSlice: MakeSlice<ThumbnailSlice> = (set, get) => ({
+  thumbnails: {
+    images: [],
+    addImage(img) {
+      const store = get();
+      const exactOrPastImageIndex = store.thumbnails.images.findIndex(
+        (v) => v.at <= img.at
+      );
+
+      // not found past or exact, so just append to the end
+      if (exactOrPastImageIndex === -1) {
+        set((s) => {
+          s.thumbnails.images.push(img);
+        });
+        return;
+      }
+
+      const exactOrPastImage = store.thumbnails.images[exactOrPastImageIndex];
+
+      // found exact, replace data
+      if (exactOrPastImage.at === img.at) {
+        set((s) => {
+          s.thumbnails.images[exactOrPastImageIndex] = img;
+        });
+        return;
+      }
+
+      // found one past, insert right before it
+      set((s) => {
+        s.thumbnails.images.splice(exactOrPastImageIndex, 0, img);
+      });
+    },
+  },
+});

src/stores/player/slices/types.ts

@@ -6,13 +6,15 @@ import { InterfaceSlice } from "@/stores/player/slices/interface";
 import { PlayingSlice } from "@/stores/player/slices/playing";
 import { ProgressSlice } from "@/stores/player/slices/progress";
 import { SourceSlice } from "@/stores/player/slices/source";
+import { ThumbnailSlice } from "@/stores/player/slices/thumbnails";
 
 export type AllSlices = InterfaceSlice &
   PlayingSlice &
   ProgressSlice &
   SourceSlice &
   DisplaySlice &
-  CastingSlice;
+  CastingSlice &
+  ThumbnailSlice;
 export type MakeSlice<Slice> = StateCreator<
   AllSlices,
   [["zustand/immer", never]],

src/stores/player/store.ts

@@ -7,6 +7,7 @@ import { createInterfaceSlice } from "@/stores/player/slices/interface";
 import { createPlayingSlice } from "@/stores/player/slices/playing";
 import { createProgressSlice } from "@/stores/player/slices/progress";
 import { createSourceSlice } from "@/stores/player/slices/source";
+import { createThumbnailSlice } from "@/stores/player/slices/thumbnails";
 import { AllSlices } from "@/stores/player/slices/types";
 
 export const usePlayerStore = create(
@@ -17,5 +18,6 @@ export const usePlayerStore = create(
     ...createSourceSlice(...a),
     ...createDisplaySlice(...a),
     ...createCastingSlice(...a),
+    ...createThumbnailSlice(...a),
   }))
 );