From c1f9382f50f92d6265f1e341573716286b712ce6 Mon Sep 17 00:00:00 2001 From: ssssobek Date: Tue, 12 Mar 2024 23:45:34 +0100 Subject: [PATCH] Add embedded subtitles support --- package.json | 2 +- pnpm-lock.yaml | 59 +++++----- src/backend/helpers/subs.ts | 13 +++ .../player/atoms/settings/CaptionsView.tsx | 9 +- src/components/player/display/base.ts | 52 +++++++++ src/components/player/display/chromecast.ts | 9 ++ .../player/display/displayInterface.ts | 6 + src/components/player/hooks/useCaptions.ts | 106 +++++++++++++++--- src/stores/player/slices/source.ts | 1 + 9 files changed, 211 insertions(+), 46 deletions(-) diff --git a/package.json b/package.json index becaa2e7..2f9f6480 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "focus-trap-react": "^10.2.3", "fscreen": "^1.2.0", "fuse.js": "^7.0.0", - "hls.js": "^1.4.14", + "hls.js": "^1.5.7", "i18next": "^23.7.11", "immer": "^10.0.3", "jwt-decode": "^4.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0cdcb77..2a0870aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -67,8 +67,8 @@ dependencies: specifier: ^7.0.0 version: 7.0.0 hls.js: - specifier: ^1.4.14 - version: 1.4.14 + specifier: ^1.5.7 + version: 1.5.7 i18next: specifier: ^23.7.11 version: 23.7.11 @@ -274,7 +274,7 @@ devDependencies: version: 0.5.9(prettier@3.1.1) rollup-plugin-visualizer: specifier: ^5.11.0 - version: 5.11.0(@rollup/wasm-node@4.12.1) + version: 5.11.0(@rollup/wasm-node@4.13.0) tailwind-scrollbar: specifier: ^3.0.5 version: 3.0.5(tailwindcss@3.4.0) @@ -2068,7 +2068,7 @@ packages: engines: {node: '>=14.0.0'} dev: false - /@rollup/plugin-babel@5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.12.1): + /@rollup/plugin-babel@5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.13.0): resolution: {integrity: sha512-WFfdLWU/xVWKeRQnKmIAQULUI7Il0gZnBIH/ZFO069wYIfPu+8zrfp/KMW0atmELoRDq8FbiP3VCss9MhCut7Q==} engines: {node: '>= 10.0.0'} peerDependencies: @@ -2081,36 +2081,36 @@ packages: dependencies: '@babel/core': 7.23.6 '@babel/helper-module-imports': 7.22.15 - '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.12.1) - rollup: /@rollup/wasm-node@4.12.1 + '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.13.0) + rollup: /@rollup/wasm-node@4.13.0 dev: true - /@rollup/plugin-node-resolve@11.2.1(@rollup/wasm-node@4.12.1): + /@rollup/plugin-node-resolve@11.2.1(@rollup/wasm-node@4.13.0): resolution: {integrity: sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg==} engines: {node: '>= 10.0.0'} peerDependencies: rollup: npm:@rollup/wasm-node dependencies: - '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.12.1) + '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.13.0) '@types/resolve': 1.17.1 builtin-modules: 3.3.0 deepmerge: 4.3.1 is-module: 1.0.0 resolve: 1.22.4 - rollup: /@rollup/wasm-node@4.12.1 + rollup: /@rollup/wasm-node@4.13.0 dev: true - /@rollup/plugin-replace@2.4.2(@rollup/wasm-node@4.12.1): + /@rollup/plugin-replace@2.4.2(@rollup/wasm-node@4.13.0): resolution: {integrity: sha512-IGcu+cydlUMZ5En85jxHH4qj2hta/11BHq95iHEyb2sbgiN0eCdzvUcHw5gt9pBL5lTi4JDYJ1acCoMGpTvEZg==} peerDependencies: rollup: npm:@rollup/wasm-node dependencies: - '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.12.1) + '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.13.0) magic-string: 0.25.9 - rollup: /@rollup/wasm-node@4.12.1 + rollup: /@rollup/wasm-node@4.13.0 dev: true - /@rollup/pluginutils@3.1.0(@rollup/wasm-node@4.12.1): + /@rollup/pluginutils@3.1.0(@rollup/wasm-node@4.13.0): resolution: {integrity: sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg==} engines: {node: '>= 8.0.0'} peerDependencies: @@ -2119,11 +2119,11 @@ packages: '@types/estree': 0.0.39 estree-walker: 1.0.1 picomatch: 2.3.1 - rollup: /@rollup/wasm-node@4.12.1 + rollup: /@rollup/wasm-node@4.13.0 dev: true - /@rollup/wasm-node@4.12.1: - resolution: {integrity: sha512-5j3BVQEccCzCb8fkl++IbDgAsnlsKBPz049C4C//j5s3pFKxKGlybl63QApdJKl1fNLr7HIwQEJcBImQtA3ZHg==} + /@rollup/wasm-node@4.13.0: + resolution: {integrity: sha512-oFX11wzU7RTaiW06WBtRpzIVN/oaG0I3XkevNO0brBklYnY9zpLhTfksN4b+TdBt6CfXV/KdVhdWLbb0fQIR7A==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true dependencies: @@ -4388,8 +4388,8 @@ packages: function-bind: 1.1.2 dev: true - /hls.js@1.4.14: - resolution: {integrity: sha512-UppQjyvPVclg+6t2KY/Rv03h0+bA5u6zwqVoz4LAC/L0fgYmIaCD7ZCrwe8WI1Gv01be1XL0QFsRbSdIHV/Wbw==} + /hls.js@1.5.7: + resolution: {integrity: sha512-Hnyf7ojTBtXHeOW1/t6wCBJSiK1WpoKF9yg7juxldDx8u3iswrkPt2wbOA/1NiwU4j27DSIVoIEJRAhcdMef/A==} dev: false /hoist-non-react-statics@3.3.2: @@ -5112,7 +5112,7 @@ packages: '@babel/plugin-syntax-typescript': 7.23.3(@babel/core@7.23.6) '@babel/types': 7.23.6 kleur: 4.1.5 - rollup: /@rollup/wasm-node@4.12.1 + rollup: /@rollup/wasm-node@4.13.0 unplugin: 1.5.1 transitivePeerDependencies: - supports-color @@ -6040,7 +6040,7 @@ packages: glob: 7.2.3 dev: true - /rollup-plugin-terser@7.0.2(@rollup/wasm-node@4.12.1): + /rollup-plugin-terser@7.0.2(@rollup/wasm-node@4.13.0): resolution: {integrity: sha512-w3iIaU4OxcF52UUXiZNsNeuXIMDvFrr+ZXK6bFZ0Q60qyVfq4uLptoS4bbq3paG3x216eQllFZX7zt6TIImguQ==} deprecated: This package has been deprecated and is no longer maintained. Please use @rollup/plugin-terser peerDependencies: @@ -6048,12 +6048,12 @@ packages: dependencies: '@babel/code-frame': 7.23.5 jest-worker: 26.6.2 - rollup: /@rollup/wasm-node@4.12.1 + rollup: /@rollup/wasm-node@4.13.0 serialize-javascript: 4.0.0 terser: 5.19.3 dev: true - /rollup-plugin-visualizer@5.11.0(@rollup/wasm-node@4.12.1): + /rollup-plugin-visualizer@5.11.0(@rollup/wasm-node@4.13.0): resolution: {integrity: sha512-exM0Ms2SN3AgTzMeW7y46neZQcyLY7eKwWAop1ZoRTCZwyrIRdMMJ6JjToAJbML77X/9N8ZEpmXG4Z/Clb9k8g==} engines: {node: '>=14'} hasBin: true @@ -6065,7 +6065,7 @@ packages: dependencies: open: 8.4.2 picomatch: 2.3.1 - rollup: /@rollup/wasm-node@4.12.1 + rollup: /@rollup/wasm-node@4.13.0 source-map: 0.7.4 yargs: 17.7.2 dev: true @@ -7051,7 +7051,7 @@ packages: '@types/node': 20.10.5 esbuild: 0.19.10 postcss: 8.4.32 - rollup: /@rollup/wasm-node@4.12.1 + rollup: /@rollup/wasm-node@4.13.0 optionalDependencies: fsevents: 2.3.3 dev: true @@ -7313,9 +7313,9 @@ packages: '@babel/core': 7.23.6 '@babel/preset-env': 7.23.6(@babel/core@7.23.6) '@babel/runtime': 7.23.6 - '@rollup/plugin-babel': 5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.12.1) - '@rollup/plugin-node-resolve': 11.2.1(@rollup/wasm-node@4.12.1) - '@rollup/plugin-replace': 2.4.2(@rollup/wasm-node@4.12.1) + '@rollup/plugin-babel': 5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.13.0) + '@rollup/plugin-node-resolve': 11.2.1(@rollup/wasm-node@4.13.0) + '@rollup/plugin-replace': 2.4.2(@rollup/wasm-node@4.13.0) '@surma/rollup-plugin-off-main-thread': 2.2.3 ajv: 8.12.0 common-tags: 1.8.2 @@ -7324,8 +7324,8 @@ packages: glob: 7.2.3 lodash: 4.17.21 pretty-bytes: 5.6.0 - rollup: /@rollup/wasm-node@4.12.1 - rollup-plugin-terser: 7.0.2(@rollup/wasm-node@4.12.1) + rollup: /@rollup/wasm-node@4.13.0 + rollup-plugin-terser: 7.0.2(@rollup/wasm-node@4.13.0) source-map: 0.8.0-beta.0 stringify-object: 3.3.0 strip-comments: 2.0.1 @@ -7370,6 +7370,7 @@ packages: /workbox-google-analytics@7.0.0: resolution: {integrity: sha512-MEYM1JTn/qiC3DbpvP2BVhyIH+dV/5BjHk756u9VbwuAhu0QHyKscTnisQuz21lfRpOwiS9z4XdqeVAKol0bzg==} + deprecated: It is not compatible with newer versions of GA starting with v4, as long as you are using GAv3 it should be ok, but the package is not longer being maintained dependencies: workbox-background-sync: 7.0.0 workbox-core: 7.0.0 diff --git a/src/backend/helpers/subs.ts b/src/backend/helpers/subs.ts index 092de6a9..bd3d99f7 100644 --- a/src/backend/helpers/subs.ts +++ b/src/backend/helpers/subs.ts @@ -51,3 +51,16 @@ export async function downloadCaption( downloadCache.set(caption.url, output, expirySeconds); return output; } + +/** + * Downloads the WebVTT content. No different than a simple + * get request with a cache. + */ +export async function downloadWebVTT(url: string): Promise { + const cached = downloadCache.get(url); + if (cached) return cached; + + // Q: should this use proxiedFetch or sendExtensionRequest? + const data = await fetch(url).then((v) => v.text()); + return data; +} diff --git a/src/components/player/atoms/settings/CaptionsView.tsx b/src/components/player/atoms/settings/CaptionsView.tsx index 92104f1b..8524ecc8 100644 --- a/src/components/player/atoms/settings/CaptionsView.tsx +++ b/src/components/player/atoms/settings/CaptionsView.tsx @@ -122,9 +122,16 @@ export function CaptionsView({ id }: { id: string }) { >(null); const { selectCaptionById, disable } = useCaptions(); const captionList = usePlayerStore((s) => s.captionList); + const getHlsCaptionList = usePlayerStore((s) => s.display?.getCaptionList); + + const captions = useMemo( + () => + captionList.length !== 0 ? captionList : getHlsCaptionList?.() ?? [], + [captionList, getHlsCaptionList], + ); const [searchQuery, setSearchQuery] = useState(""); - const subtitleList = useSubtitleList(captionList, searchQuery); + const subtitleList = useSubtitleList(captions, searchQuery); const [downloadReq, startDownload] = useAsyncFn( async (captionId: string) => { diff --git a/src/components/player/display/base.ts b/src/components/player/display/base.ts index a12661ab..4b3b8b9a 100644 --- a/src/components/player/display/base.ts +++ b/src/components/player/display/base.ts @@ -67,6 +67,11 @@ export function makeVideoElementDisplayInterface(): DisplayInterface { let preferenceQuality: SourceQuality | null = null; let lastVolume = 1; + const languagePromises = new Map< + string, + (value: void | PromiseLike) => void + >(); + function reportLevels() { if (!hls) return; const levels = hls.levels; @@ -133,6 +138,7 @@ export function makeVideoElementDisplayInterface(): DisplayInterface { }, }, }, + renderTextTracksNatively: false, }); hls.on(Hls.Events.ERROR, (event, data) => { console.error("HLS error", data); @@ -173,6 +179,17 @@ export function makeVideoElementDisplayInterface(): DisplayInterface { const quality = hlsLevelToQuality(hls.levels[hls.currentLevel]); emit("changedquality", quality); }); + hls.on(Hls.Events.SUBTITLE_TRACK_LOADED, () => { + for (const [lang, resolve] of languagePromises) { + const track = hls?.subtitleTracks.find((t) => t.lang === lang); + if (track) { + resolve(); + languagePromises.delete(lang); + break; + } + } + console.log("Subtitle tracks loaded", hls?.subtitleTracks); + }); } hls.attachMedia(vid); @@ -413,5 +430,40 @@ export function makeVideoElementDisplayInterface(): DisplayInterface { setPlaybackRate(rate) { if (videoElement) videoElement.playbackRate = rate; }, + getCaptionList() { + return ( + hls?.subtitleTracks.map((track) => { + return { + id: track.id.toString(), + language: track.lang ?? "unknown", + url: track.url, + needsProxy: false, + hls: true, + }; + }) ?? [] + ); + }, + getSubtitleTracks() { + return hls?.subtitleTracks ?? []; + }, + async setSubtitlePreference(lang) { + // default subtitles are already loaded by hls.js + const track = hls?.subtitleTracks.find((t) => t.lang === lang); + if (track?.details !== undefined) return Promise.resolve(); + + // need to wait a moment before hls loads the subtitles + const promise = new Promise((resolve, reject) => { + languagePromises.set(lang, resolve); + + // reject after some time, if hls.js fails to load the subtitles + // for any reason + setTimeout(() => { + reject(); + languagePromises.delete(lang); + }, 5000); + }); + hls?.setSubtitleOption({ lang }); + return promise; + }, }; } diff --git a/src/components/player/display/chromecast.ts b/src/components/player/display/chromecast.ts index dbaab0df..1a318f16 100644 --- a/src/components/player/display/chromecast.ts +++ b/src/components/player/display/chromecast.ts @@ -274,5 +274,14 @@ export function makeChromecastDisplayInterface( playbackRate = rate; setSource(); }, + getCaptionList() { + return []; + }, + getSubtitleTracks() { + return []; + }, + async setSubtitlePreference() { + return Promise.resolve(); + }, }; } diff --git a/src/components/player/display/displayInterface.ts b/src/components/player/display/displayInterface.ts index 8ba8480a..134bef44 100644 --- a/src/components/player/display/displayInterface.ts +++ b/src/components/player/display/displayInterface.ts @@ -1,4 +1,7 @@ +import { MediaPlaylist } from "hls.js"; + import { MWMediaType } from "@/backend/metadata/types/mw"; +import { CaptionListItem } from "@/stores/player/slices/source"; import { LoadableSource, SourceQuality } from "@/stores/player/utils/qualities"; import { Listener } from "@/utils/events"; @@ -70,4 +73,7 @@ export interface DisplayInterface extends Listener { setMeta(meta: DisplayMeta): void; setCaption(caption: DisplayCaption | null): void; getType(): DisplayType; + getCaptionList(): CaptionListItem[]; + getSubtitleTracks(): MediaPlaylist[]; + setSubtitlePreference(lang: string): Promise; } diff --git a/src/components/player/hooks/useCaptions.ts b/src/components/player/hooks/useCaptions.ts index 458c704a..662108dd 100644 --- a/src/components/player/hooks/useCaptions.ts +++ b/src/components/player/hooks/useCaptions.ts @@ -1,6 +1,8 @@ -import { useCallback } from "react"; +import { useCallback, useMemo } from "react"; +import subsrt from "subsrt-ts"; +import { ContentCaption } from "subsrt-ts/dist/types/handler"; -import { downloadCaption } from "@/backend/helpers/subs"; +import { downloadCaption, downloadWebVTT } from "@/backend/helpers/subs"; import { usePlayerStore } from "@/stores/player/store"; import { useSubtitleStore } from "@/stores/subtitles"; @@ -12,32 +14,106 @@ export function useCaptions() { ); const setCaption = usePlayerStore((s) => s.setCaption); const lastSelectedLanguage = useSubtitleStore((s) => s.lastSelectedLanguage); + const captionList = usePlayerStore((s) => s.captionList); + const getHlsCaptionList = usePlayerStore((s) => s.display?.getCaptionList); + + const getSubtitleTracks = usePlayerStore((s) => s.display?.getSubtitleTracks); + const setSubtitlePreference = usePlayerStore( + (s) => s.display?.setSubtitlePreference, + ); + + const captions = useMemo( + () => + captionList.length !== 0 ? captionList : getHlsCaptionList?.() ?? [], + [captionList, getHlsCaptionList], + ); const selectCaptionById = useCallback( async (captionId: string) => { - const caption = captionList.find((v) => v.id === captionId); + const caption = captions.find((v) => v.id === captionId); if (!caption) return; - const srtData = await downloadCaption(caption); - setCaption({ - id: caption.id, - language: caption.language, - srtData, - url: caption.url, - }); - resetSubtitleSpecificSettings(); - setLanguage(caption.language); + if (!caption.hls) { + const srtData = await downloadCaption(caption); + setCaption({ + id: caption.id, + language: caption.language, + srtData, + url: caption.url, + }); + resetSubtitleSpecificSettings(); + setLanguage(caption.language); + } else { + // request a language change to hls, so it can load the subtitles + await setSubtitlePreference?.(caption.language); + const track = getSubtitleTracks?.().find( + (t) => t.id.toString() === caption.id && t.details !== undefined, + ); + if (!track) return; + + const fragments = + track.details?.fragments?.filter( + (frag) => frag !== null && frag.url !== null, + ) ?? []; + + const vttCaptions = ( + await Promise.all( + fragments.map(async (frag) => { + const vtt = await downloadWebVTT(frag.url); + const parsed = subsrt.parse(vtt); + return parsed.filter( + (c) => c.type === "caption", + ) as ContentCaption[]; + }), + ) + ).flat(); + + // for some reason, in some cases there will be captions + // with the same start/end times, the same text duplicated + const filtered = vttCaptions.reduce( + (acc: ContentCaption[], cap: ContentCaption) => { + const lastCap = acc[acc.length - 1]; + const isSameAsLast = + lastCap?.start === cap.start && + lastCap?.end === cap.end && + lastCap?.content === cap.content; + if (lastCap === undefined || !isSameAsLast) { + acc.push(cap); + } + return acc; + }, + [], + ); + + const srtData = subsrt.build(filtered, { format: "srt" }); + + setCaption({ + id: caption.id, + language: caption.language, + srtData, + url: caption.url, + }); + resetSubtitleSpecificSettings(); + setLanguage(caption.language); + } }, - [setLanguage, captionList, setCaption, resetSubtitleSpecificSettings], + [ + setLanguage, + captions, + setCaption, + resetSubtitleSpecificSettings, + getSubtitleTracks, + setSubtitlePreference, + ], ); const selectLanguage = useCallback( async (language: string) => { - const caption = captionList.find((v) => v.language === language); + const caption = captions.find((v) => v.language === language); if (!caption) return; return selectCaptionById(caption.id); }, - [captionList, selectCaptionById], + [captions, selectCaptionById], ); const disable = useCallback(async () => { diff --git a/src/stores/player/slices/source.ts b/src/stores/player/slices/source.ts index 0312b8b0..5d04ef49 100644 --- a/src/stores/player/slices/source.ts +++ b/src/stores/player/slices/source.ts @@ -53,6 +53,7 @@ export interface CaptionListItem { language: string; url: string; needsProxy: boolean; + hls?: boolean; } export interface SourceSlice {