From 32f031ab23f771c7a766aee621f94ef1e833733f Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 21 Oct 2023 04:50:14 +0200 Subject: [PATCH] thumbnail scraping --- src/components/player/atoms/ProgressBar.tsx | 129 +++++++++++++----- src/components/player/base/Container.tsx | 2 + .../player/internals/ThumbnailScraper.tsx | 127 +++++++++++++++++ src/stores/player/slices/thumbnails.ts | 106 ++++++++++++++ src/stores/player/slices/types.ts | 4 +- src/stores/player/store.ts | 2 + 6 files changed, 336 insertions(+), 34 deletions(-) create mode 100644 src/components/player/internals/ThumbnailScraper.tsx create mode 100644 src/stores/player/slices/thumbnails.ts diff --git a/src/components/player/atoms/ProgressBar.tsx b/src/components/player/atoms/ProgressBar.tsx index ff4a8f43..34e5840f 100644 --- a/src/components/player/atoms/ProgressBar.tsx +++ b/src/components/player/atoms/ProgressBar.tsx @@ -1,8 +1,48 @@ -import { useCallback, useEffect, useRef } from "react"; +import { + MouseEvent, + RefObject, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { useProgressBar } from "@/hooks/useProgressBar"; +import { nearestImageAt } from "@/stores/player/slices/thumbnails"; import { usePlayerStore } from "@/stores/player/store"; +function ThumbnailDisplay(props: { at: number }) { + const thumbnailImages = usePlayerStore((s) => s.thumbnails.images); + const currentThumbnail = useMemo(() => { + return nearestImageAt(thumbnailImages, props.at)?.image; + }, [thumbnailImages, props.at]); + + if (!currentThumbnail) return null; + return ; +} + +function useMouseHoverPosition(barRef: RefObject) { + const [mousePos, setMousePos] = useState(-1); + + const mouseMove = useCallback( + (e: MouseEvent) => { + const bar = barRef.current; + if (!bar) return; + const rect = barRef.current.getBoundingClientRect(); + const pos = (e.pageX - rect.left) / barRef.current.offsetWidth; + setMousePos(pos * 100); + }, + [setMousePos, barRef] + ); + + const mouseLeave = useCallback(() => { + setMousePos(-1); + }, [setMousePos]); + + return { mousePos, mouseMove, mouseLeave }; +} + export function ProgressBar() { const { duration, time, buffered } = usePlayerStore((s) => s.progress); const display = usePlayerStore((s) => s.display); @@ -18,6 +58,7 @@ export function ProgressBar() { ); const ref = useRef(null); + const { mouseMove, mouseLeave, mousePos } = useMouseHoverPosition(ref); const { dragging, dragPercentage, dragMouseDown } = useProgressBar( ref, @@ -31,45 +72,67 @@ export function ProgressBar() { setDraggingTime((dragPercentage / 100) * duration); }, [setDraggingTime, duration, dragPercentage]); - return ( -
-
-
- {/* Pre-loaded content bar */} -
+ const mousePosition = Math.floor(dragPercentage * duration); - {/* Actual progress bar */} + return ( +
+
+ {mousePos > -1 ? (
+ +
+ ) : null} +
+ +
+
+
+ {/* Pre-loaded content bar */}
+ + {/* Actual progress bar */} +
+
+
diff --git a/src/components/player/base/Container.tsx b/src/components/player/base/Container.tsx index 9fd9a2bf..7ef8b652 100644 --- a/src/components/player/base/Container.tsx +++ b/src/components/player/base/Container.tsx @@ -5,6 +5,7 @@ import { CastingInternal } from "@/components/player/internals/CastingInternal"; import { HeadUpdater } from "@/components/player/internals/HeadUpdater"; import { KeyboardEvents } from "@/components/player/internals/KeyboardEvents"; import { ProgressSaver } from "@/components/player/internals/ProgressSaver"; +import { ThumbnailScraper } from "@/components/player/internals/ThumbnailScraper"; import { VideoClickTarget } from "@/components/player/internals/VideoClickTarget"; import { VideoContainer } from "@/components/player/internals/VideoContainer"; import { PlayerHoverState } from "@/stores/player/slices/interface"; @@ -82,6 +83,7 @@ export function Container(props: PlayerProps) { return (
+ diff --git a/src/components/player/internals/ThumbnailScraper.tsx b/src/components/player/internals/ThumbnailScraper.tsx new file mode 100644 index 00000000..34d457a1 --- /dev/null +++ b/src/components/player/internals/ThumbnailScraper.tsx @@ -0,0 +1,127 @@ +import Hls from "hls.js"; +import { useEffect, useMemo, useRef } from "react"; + +import { ThumbnailImage } from "@/stores/player/slices/thumbnails"; +import { usePlayerStore } from "@/stores/player/store"; +import { LoadableSource, selectQuality } from "@/stores/player/utils/qualities"; + +class ThumnbnailWorker { + interrupted: boolean; + + videoEl: HTMLVideoElement | null = null; + + canvasEl: HTMLCanvasElement | null = null; + + hls: Hls | null = null; + + cb: (img: ThumbnailImage) => void; + + constructor(ops: { addImage: (img: ThumbnailImage) => void }) { + this.cb = ops.addImage; + this.interrupted = false; + } + + start(source: LoadableSource) { + const el = document.createElement("video"); + const canvas = document.createElement("canvas"); + this.hls = new Hls(); + if (source.type === "mp4") { + el.src = source.url; + el.crossOrigin = "anonymous"; + } else if (source.type === "hls") { + this.hls.attachMedia(el); + this.hls.loadSource(source.url); + } else throw new Error("Invalid loadable source type"); + this.videoEl = el; + this.canvasEl = canvas; + this.begin().catch((err) => console.error(err)); + } + + destroy() { + this.hls?.detachMedia(); + this.hls?.destroy(); + this.hls = null; + this.interrupted = true; + this.videoEl = null; + this.canvasEl = null; + } + + private async initVideo() { + if (!this.videoEl || !this.canvasEl) return; + await new Promise((resolve, reject) => { + this.videoEl?.addEventListener("loadedmetadata", resolve); + this.videoEl?.addEventListener("error", reject); + }); + if (!this.videoEl || !this.canvasEl) return; + this.canvasEl.height = this.videoEl.videoHeight; + this.canvasEl.width = this.videoEl.videoWidth; + } + + private async takeSnapshot(at: number) { + if (!this.videoEl || !this.canvasEl) return; + this.videoEl.currentTime = at; + await new Promise((resolve) => { + this.videoEl?.addEventListener("seeked", resolve); + }); + if (!this.videoEl || !this.canvasEl) return; + const ctx = this.canvasEl.getContext("2d"); + if (!ctx) return; + ctx.drawImage( + this.videoEl, + 0, + 0, + this.canvasEl.width, + this.canvasEl.height + ); + const imgUrl = this.canvasEl.toDataURL(); + this.cb({ + at, + data: imgUrl, + }); + } + + private async begin() { + const vid = this.videoEl; + if (!vid) return; + await this.initVideo(); + if (this.interrupted) return; + await this.takeSnapshot(vid.duration / 2); + } +} + +export function ThumbnailScraper() { + const addImage = usePlayerStore((s) => s.thumbnails.addImage); + const source = usePlayerStore((s) => s.source); + const workerRef = useRef(null); + + const inputStream = useMemo(() => { + if (!source) return null; + return selectQuality(source, { + automaticQuality: false, + lastChosenQuality: "360", + }); + }, [source]); + + // TODO stop worker on meta change + + // start worker with the stream + useEffect(() => { + // dont interrupt existing working + if (workerRef.current) return; + if (!inputStream) return; + const ins = new ThumnbnailWorker({ + addImage, + }); + workerRef.current = ins; + ins.start(inputStream.stream); + }, [inputStream, addImage]); + + // destroy worker on unmount + useEffect(() => { + return () => { + if (workerRef.current) workerRef.current.destroy(); + }; + }, []); + + return null; +} diff --git a/src/stores/player/slices/thumbnails.ts b/src/stores/player/slices/thumbnails.ts new file mode 100644 index 00000000..588d4351 --- /dev/null +++ b/src/stores/player/slices/thumbnails.ts @@ -0,0 +1,106 @@ +import { MakeSlice } from "@/stores/player/slices/types"; + +export interface ThumbnailImage { + at: number; + data: string; +} + +export interface ThumbnailSlice { + thumbnails: { + images: ThumbnailImage[]; + addImage(img: ThumbnailImage): void; + }; +} + +export interface ThumbnailImagePosition { + index: number; + image: ThumbnailImage; +} + +/** + * get nearest image at the timestamp provided + * @param images images, must be sorted + */ +export function nearestImageAt( + images: ThumbnailImage[], + at: number +): ThumbnailImagePosition | null { + // no images, early return + if (images.length === 0) return null; + + const indexPastTimestamp = images.findIndex((v) => v.at < at); + + // no image found past timestamp, so last image must be closest + if (indexPastTimestamp === -1) + return { + index: images.length - 1, + image: images[images.length - 1], + }; + + const imagePastTimestamp = images[indexPastTimestamp]; + + // if past timestamp is first image, just return that image + if (indexPastTimestamp === 0) + return { + index: indexPastTimestamp, + image: imagePastTimestamp, + }; + + // distance before distance past + // | | + // [before] --------------------- [at] --------------------- [past] + const imageBeforeTimestamp = images[indexPastTimestamp - 1]; + const distanceBefore = at - imageBeforeTimestamp.at; + const distancePast = imagePastTimestamp.at - at; + + // if distance of before timestamp is smaller than the distance past + // before is closer, return that + // [before] --X-------------- [past] + if (distanceBefore < distancePast) + return { + index: indexPastTimestamp - 1, + image: imageBeforeTimestamp, + }; + + // must be closer to past here, return past + // [before] --------------X-- [past] + return { + index: indexPastTimestamp, + image: imagePastTimestamp, + }; +} + +export const createThumbnailSlice: MakeSlice = (set, get) => ({ + thumbnails: { + images: [], + addImage(img) { + const store = get(); + const exactOrPastImageIndex = store.thumbnails.images.findIndex( + (v) => v.at <= img.at + ); + + // not found past or exact, so just append to the end + if (exactOrPastImageIndex === -1) { + set((s) => { + s.thumbnails.images.push(img); + }); + return; + } + + const exactOrPastImage = store.thumbnails.images[exactOrPastImageIndex]; + + // found exact, replace data + if (exactOrPastImage.at === img.at) { + set((s) => { + s.thumbnails.images[exactOrPastImageIndex] = img; + }); + return; + } + + // found one past, insert right before it + set((s) => { + s.thumbnails.images.splice(exactOrPastImageIndex, 0, img); + }); + }, + }, +}); diff --git a/src/stores/player/slices/types.ts b/src/stores/player/slices/types.ts index 8a47023e..6f358945 100644 --- a/src/stores/player/slices/types.ts +++ b/src/stores/player/slices/types.ts @@ -6,13 +6,15 @@ import { InterfaceSlice } from "@/stores/player/slices/interface"; import { PlayingSlice } from "@/stores/player/slices/playing"; import { ProgressSlice } from "@/stores/player/slices/progress"; import { SourceSlice } from "@/stores/player/slices/source"; +import { ThumbnailSlice } from "@/stores/player/slices/thumbnails"; export type AllSlices = InterfaceSlice & PlayingSlice & ProgressSlice & SourceSlice & DisplaySlice & - CastingSlice; + CastingSlice & + ThumbnailSlice; export type MakeSlice = StateCreator< AllSlices, [["zustand/immer", never]], diff --git a/src/stores/player/store.ts b/src/stores/player/store.ts index 42e771d8..e81214b9 100644 --- a/src/stores/player/store.ts +++ b/src/stores/player/store.ts @@ -7,6 +7,7 @@ import { createInterfaceSlice } from "@/stores/player/slices/interface"; import { createPlayingSlice } from "@/stores/player/slices/playing"; import { createProgressSlice } from "@/stores/player/slices/progress"; import { createSourceSlice } from "@/stores/player/slices/source"; +import { createThumbnailSlice } from "@/stores/player/slices/thumbnails"; import { AllSlices } from "@/stores/player/slices/types"; export const usePlayerStore = create( @@ -17,5 +18,6 @@ export const usePlayerStore = create( ...createSourceSlice(...a), ...createDisplaySlice(...a), ...createCastingSlice(...a), + ...createThumbnailSlice(...a), })) );