From a52fac701a27d9926f72ea4748a094680f62ee4e Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 18 Dec 2023 21:47:19 +0100 Subject: [PATCH 01/16] Parse provider API urls + use new provider api in runAll scrape --- src/hooks/useProviderScrape.tsx | 235 +++++++++++++++++++++----------- src/utils/providers.ts | 29 ++-- src/utils/proxyUrls.ts | 77 +++++++++++ 3 files changed, 249 insertions(+), 92 deletions(-) create mode 100644 src/utils/proxyUrls.ts diff --git a/src/hooks/useProviderScrape.tsx b/src/hooks/useProviderScrape.tsx index 8eeceff6..d1120996 100644 --- a/src/hooks/useProviderScrape.tsx +++ b/src/hooks/useProviderScrape.tsx @@ -1,7 +1,11 @@ -import { ScrapeMedia } from "@movie-web/providers"; +import { + FullScraperEvents, + RunOutput, + ScrapeMedia, +} from "@movie-web/providers"; import { RefObject, useCallback, useEffect, useRef, useState } from "react"; -import { providers } from "@/utils/providers"; +import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; export interface ScrapingItems { id: string; @@ -18,96 +22,169 @@ export interface ScrapingSegment { percentage: number; } -export function useScrape() { +type ScraperEvent = Parameters< + NonNullable +>[0]; + +function useBaseScrape() { const [sources, setSources] = useState>({}); const [sourceOrder, setSourceOrder] = useState([]); const [currentSource, setCurrentSource] = useState(); + const lastId = useRef(null); + + const initEvent = useCallback((evt: ScraperEvent<"init">) => { + setSources( + evt.sourceIds + .map((v) => { + const source = providers.getMetadata(v); + if (!source) throw new Error("invalid source id"); + const out: ScrapingSegment = { + name: source.name, + id: source.id, + status: "waiting", + percentage: 0, + }; + return out; + }) + .reduce>((a, v) => { + a[v.id] = v; + return a; + }, {}) + ); + setSourceOrder(evt.sourceIds.map((v) => ({ id: v, children: [] }))); + }, []); + + const startEvent = useCallback((id: ScraperEvent<"start">) => { + setSources((s) => { + if (s[id]) s[id].status = "pending"; + return { ...s }; + }); + setCurrentSource(id); + lastId.current = id; + }, []); + + const updateEvent = useCallback((evt: ScraperEvent<"update">) => { + setSources((s) => { + if (s[evt.id]) { + s[evt.id].status = evt.status; + s[evt.id].reason = evt.reason; + s[evt.id].error = evt.error; + s[evt.id].percentage = evt.percentage; + } + return { ...s }; + }); + }, []); + + const discoverEmbedsEvent = useCallback( + (evt: ScraperEvent<"discoverEmbeds">) => { + setSources((s) => { + evt.embeds.forEach((v) => { + const source = providers.getMetadata(v.embedScraperId); + if (!source) throw new Error("invalid source id"); + const out: ScrapingSegment = { + embedId: v.embedScraperId, + name: source.name, + id: v.id, + status: "waiting", + percentage: 0, + }; + s[v.id] = out; + }); + return { ...s }; + }); + setSourceOrder((s) => { + const source = s.find((v) => v.id === evt.sourceId); + if (!source) throw new Error("invalid source id"); + source.children = evt.embeds.map((v) => v.id); + return [...s]; + }); + }, + [] + ); + + const startScrape = useCallback(() => { + lastId.current = null; + }, []); + + const getResult = useCallback((output: RunOutput | null) => { + if (output && lastId.current) { + setSources((s) => { + if (!lastId.current) return s; + if (s[lastId.current]) s[lastId.current].status = "success"; + return { ...s }; + }); + } + return output; + }, []); + + return { + initEvent, + startEvent, + updateEvent, + discoverEmbedsEvent, + startScrape, + getResult, + sources, + sourceOrder, + currentSource, + }; +} + +export function useScrape() { + const { + sources, + sourceOrder, + currentSource, + updateEvent, + discoverEmbedsEvent, + initEvent, + getResult, + startEvent, + startScrape, + } = useBaseScrape(); const startScraping = useCallback( async (media: ScrapeMedia) => { - if (!providers) return null; + const providerApiUrl = getLoadbalancedProviderApiUrl(); + if (providerApiUrl) { + startScrape(); + const sseOutput = await new Promise( + (resolve, reject) => { + const scrapeEvents = new EventSource(providerApiUrl); + scrapeEvents.addEventListener("error", (err) => reject(err)); + scrapeEvents.addEventListener("init", (e) => initEvent(e.data)); + scrapeEvents.addEventListener("start", (e) => startEvent(e.data)); + scrapeEvents.addEventListener("update", (e) => updateEvent(e.data)); + scrapeEvents.addEventListener("discoverEmbeds", (e) => + discoverEmbedsEvent(e.data) + ); + scrapeEvents.addEventListener("finish", (e) => resolve(e.data)); + } + ); + return getResult(sseOutput); + } - let lastId: string | null = null; + if (!providers) return null; + startScrape(); const output = await providers.runAll({ media, events: { - init(evt) { - setSources( - evt.sourceIds - .map((v) => { - const source = providers.getMetadata(v); - if (!source) throw new Error("invalid source id"); - const out: ScrapingSegment = { - name: source.name, - id: source.id, - status: "waiting", - percentage: 0, - }; - return out; - }) - .reduce>((a, v) => { - a[v.id] = v; - return a; - }, {}) - ); - setSourceOrder(evt.sourceIds.map((v) => ({ id: v, children: [] }))); - }, - start(id) { - setSources((s) => { - if (s[id]) s[id].status = "pending"; - return { ...s }; - }); - setCurrentSource(id); - lastId = id; - }, - update(evt) { - setSources((s) => { - if (s[evt.id]) { - s[evt.id].status = evt.status; - s[evt.id].reason = evt.reason; - s[evt.id].error = evt.error; - s[evt.id].percentage = evt.percentage; - } - return { ...s }; - }); - }, - discoverEmbeds(evt) { - setSources((s) => { - evt.embeds.forEach((v) => { - const source = providers.getMetadata(v.embedScraperId); - if (!source) throw new Error("invalid source id"); - const out: ScrapingSegment = { - embedId: v.embedScraperId, - name: source.name, - id: v.id, - status: "waiting", - percentage: 0, - }; - s[v.id] = out; - }); - return { ...s }; - }); - setSourceOrder((s) => { - const source = s.find((v) => v.id === evt.sourceId); - if (!source) throw new Error("invalid source id"); - source.children = evt.embeds.map((v) => v.id); - return [...s]; - }); - }, + init: initEvent, + start: startEvent, + update: updateEvent, + discoverEmbeds: discoverEmbedsEvent, }, }); - - if (output && lastId) { - setSources((s) => { - if (!lastId) return s; - if (s[lastId]) s[lastId].status = "success"; - return { ...s }; - }); - } - - return output; + return getResult(output); }, - [setSourceOrder, setSources] + [ + initEvent, + startEvent, + updateEvent, + discoverEmbedsEvent, + getResult, + startScrape, + ] ); return { diff --git a/src/utils/providers.ts b/src/utils/providers.ts index 07b91bff..35ebe80d 100644 --- a/src/utils/providers.ts +++ b/src/utils/providers.ts @@ -7,22 +7,25 @@ import { targets, } from "@movie-web/providers"; -import { conf } from "@/setup/config"; -import { useAuthStore } from "@/stores/auth"; +import { getProviderApiUrls, getProxyUrls } from "@/utils/proxyUrls"; -const originalUrls = conf().PROXY_URLS; -let fetchersIndex = -1; - -export function getLoadbalancedProxyUrl() { - const fetchers = useAuthStore.getState().proxySet ?? originalUrls; - if (fetchersIndex === -1 || fetchersIndex >= fetchers.length) { - fetchersIndex = Math.floor(Math.random() * fetchers.length); - } - const proxyUrl = fetchers[fetchersIndex]; - fetchersIndex = (fetchersIndex + 1) % fetchers.length; - return proxyUrl; +function makeLoadbalancedList(getter: () => string[]) { + let listIndex = -1; + return () => { + const fetchers = getter(); + if (listIndex === -1 || listIndex >= fetchers.length) { + listIndex = Math.floor(Math.random() * fetchers.length); + } + const proxyUrl = fetchers[listIndex]; + listIndex = (listIndex + 1) % fetchers.length; + return proxyUrl; + }; } +const getLoadbalancedProxyUrl = makeLoadbalancedList(getProxyUrls); +export const getLoadbalancedProviderApiUrl = + makeLoadbalancedList(getProviderApiUrls); + function makeLoadBalancedSimpleProxyFetcher() { const fetcher: ProviderBuilderOptions["fetcher"] = (a, b) => { const currentFetcher = makeSimpleProxyFetcher( diff --git a/src/utils/proxyUrls.ts b/src/utils/proxyUrls.ts new file mode 100644 index 00000000..64c91b34 --- /dev/null +++ b/src/utils/proxyUrls.ts @@ -0,0 +1,77 @@ +import { conf } from "@/setup/config"; +import { useAuthStore } from "@/stores/auth"; + +const originalUrls = conf().PROXY_URLS; +const types = ["proxy", "api"] as const; + +type ParsedUrlType = (typeof types)[number]; + +export interface ParsedUrl { + url: string; + type: ParsedUrlType; +} + +function canParseUrl(url: string): boolean { + try { + return !!new URL(url); + } catch { + return false; + } +} + +function isParsedUrlType(type: string): type is ParsedUrlType { + return types.includes(type as any); +} + +/** + * Turn a string like "a=b,c=d,d=e" into a dictionary object + */ +function parseParams(input: string): Record { + const entriesParams = input + .split(",") + .map((param) => param.split("=", 2).filter((part) => part.length !== 0)) + .filter((v) => v.length === 2); + return Object.fromEntries(entriesParams); +} + +export function getParsedUrls() { + const urls = useAuthStore.getState().proxySet ?? originalUrls; + const output: ParsedUrl[] = []; + urls.forEach((url) => { + if (!url.startsWith("|")) { + if (canParseUrl(url)) { + output.push({ + url, + type: "proxy", + }); + return; + } + } + + const match = url.match(/^|([^|])+|(.*)$/g); + if (!match || !match[2]) return; + if (!canParseUrl(match[2])) return; + const params = parseParams(match[1]); + const type = params.type ?? "proxy"; + + if (!isParsedUrlType(type)) return; + output.push({ + url: match[2], + type, + }); + }); + + return output; +} + +export function getProxyUrls() { + return getParsedUrls() + .filter((v) => v.type === "proxy") + .map((v) => v.url); +} + +export function getProviderApiUrls() { + return getParsedUrls() + .filter((v) => v.type === "api") + .map((v) => v.url); +} From 8af4256d958356ab81d2a9a8440959a9d53e5920 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 18 Dec 2023 21:50:30 +0100 Subject: [PATCH 02/16] I cant use commas --- src/utils/proxyUrls.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/proxyUrls.ts b/src/utils/proxyUrls.ts index 64c91b34..c967a01c 100644 --- a/src/utils/proxyUrls.ts +++ b/src/utils/proxyUrls.ts @@ -28,7 +28,7 @@ function isParsedUrlType(type: string): type is ParsedUrlType { */ function parseParams(input: string): Record { const entriesParams = input - .split(",") + .split(";") .map((param) => param.split("=", 2).filter((part) => part.length !== 0)) .filter((v) => v.length === 2); return Object.fromEntries(entriesParams); From f55b39d0fa7309b72a6646092a7e86eaee836a56 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 18 Dec 2023 22:24:34 +0100 Subject: [PATCH 03/16] Properly map events and data to providers api --- src/hooks/useProviderScrape.tsx | 57 ++++++++++++++++++++++++++++----- src/utils/providers.ts | 2 +- src/utils/proxyUrls.ts | 5 +-- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/hooks/useProviderScrape.tsx b/src/hooks/useProviderScrape.tsx index d1120996..80ab63a4 100644 --- a/src/hooks/useProviderScrape.tsx +++ b/src/hooks/useProviderScrape.tsx @@ -150,15 +150,56 @@ export function useScrape() { startScrape(); const sseOutput = await new Promise( (resolve, reject) => { - const scrapeEvents = new EventSource(providerApiUrl); - scrapeEvents.addEventListener("error", (err) => reject(err)); - scrapeEvents.addEventListener("init", (e) => initEvent(e.data)); - scrapeEvents.addEventListener("start", (e) => startEvent(e.data)); - scrapeEvents.addEventListener("update", (e) => updateEvent(e.data)); - scrapeEvents.addEventListener("discoverEmbeds", (e) => - discoverEmbedsEvent(e.data) + const finalUrl = new URL(`${providerApiUrl}/scrape`); + finalUrl.searchParams.append("type", media.type); + finalUrl.searchParams.append( + "releaseYear", + media.releaseYear.toString() ); - scrapeEvents.addEventListener("finish", (e) => resolve(e.data)); + finalUrl.searchParams.append("title", media.title); + finalUrl.searchParams.append("tmdbId", media.tmdbId); + if (media.imdbId) + finalUrl.searchParams.append("imdbId", media.imdbId); + if (media.type === "show") { + finalUrl.searchParams.append( + "episodeNumber", + media.episode.number.toString() + ); + finalUrl.searchParams.append( + "episodeTmdbId", + media.episode.tmdbId + ); + finalUrl.searchParams.append( + "seasonNumber", + media.season.number.toString() + ); + finalUrl.searchParams.append("seasonTmdbId", media.season.tmdbId); + } + const scrapeEvents = new EventSource(finalUrl.toString()); + scrapeEvents.addEventListener("init", (e) => { + initEvent(JSON.parse(e.data)); + }); + scrapeEvents.addEventListener("error", (err) => { + console.error("failed to use provider api", err); + reject(err); + }); + scrapeEvents.addEventListener("start", (e) => + startEvent(JSON.parse(e.data)) + ); + scrapeEvents.addEventListener("update", (e) => + updateEvent(JSON.parse(e.data)) + ); + scrapeEvents.addEventListener("discoverEmbeds", (e) => + discoverEmbedsEvent(JSON.parse(e.data)) + ); + scrapeEvents.addEventListener("completed", (e) => { + scrapeEvents.close(); + resolve(JSON.parse(e.data)); + }); + scrapeEvents.addEventListener("noOutput", () => { + scrapeEvents.close(); + resolve(null); + }); } ); return getResult(sseOutput); diff --git a/src/utils/providers.ts b/src/utils/providers.ts index 35ebe80d..73c3849e 100644 --- a/src/utils/providers.ts +++ b/src/utils/providers.ts @@ -22,7 +22,7 @@ function makeLoadbalancedList(getter: () => string[]) { }; } -const getLoadbalancedProxyUrl = makeLoadbalancedList(getProxyUrls); +export const getLoadbalancedProxyUrl = makeLoadbalancedList(getProxyUrls); export const getLoadbalancedProviderApiUrl = makeLoadbalancedList(getProviderApiUrls); diff --git a/src/utils/proxyUrls.ts b/src/utils/proxyUrls.ts index c967a01c..645c8329 100644 --- a/src/utils/proxyUrls.ts +++ b/src/utils/proxyUrls.ts @@ -24,7 +24,7 @@ function isParsedUrlType(type: string): type is ParsedUrlType { } /** - * Turn a string like "a=b,c=d,d=e" into a dictionary object + * Turn a string like "a=b;c=d;d=e" into a dictionary object */ function parseParams(input: string): Record { const entriesParams = input @@ -48,7 +48,7 @@ export function getParsedUrls() { } } - const match = url.match(/^|([^|])+|(.*)$/g); + const match = /^\|([^|]+)\|(.*)$/g.exec(url); if (!match || !match[2]) return; if (!canParseUrl(match[2])) return; const params = parseParams(match[1]); @@ -61,6 +61,7 @@ export function getParsedUrls() { }); }); + console.log(urls, output); return output; } From ed67c1e63b4e3ffa2abb14875fbd319f7fc04896 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 18 Dec 2023 22:30:46 +0100 Subject: [PATCH 04/16] Remove console log --- src/utils/proxyUrls.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/proxyUrls.ts b/src/utils/proxyUrls.ts index 645c8329..3efd59e3 100644 --- a/src/utils/proxyUrls.ts +++ b/src/utils/proxyUrls.ts @@ -61,7 +61,6 @@ export function getParsedUrls() { }); }); - console.log(urls, output); return output; } From e48af381c52533df53484c214d58aa4bddf05d54 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Mon, 18 Dec 2023 22:53:59 +0100 Subject: [PATCH 05/16] its not finished --- src/backend/helpers/providerApi.ts | 55 ++++++++++++++++++++++++++++++ src/hooks/useProviderScrape.tsx | 29 ++-------------- 2 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 src/backend/helpers/providerApi.ts diff --git a/src/backend/helpers/providerApi.ts b/src/backend/helpers/providerApi.ts new file mode 100644 index 00000000..4292ee7d --- /dev/null +++ b/src/backend/helpers/providerApi.ts @@ -0,0 +1,55 @@ +import { ScrapeMedia } from "@movie-web/providers"; + +function scrapeMediaToQueryMedia(media: ScrapeMedia) { + let extra: Record = {}; + if (media.type === "show") { + extra = { + episodeNumber: media.episode.number.toString(), + episodeTmdbId: media.episode.tmdbId, + seasonNumber: media.season.number.toString(), + seasonTmdbId: media.season.tmdbId, + }; + } + + return { + type: media.type, + releaseYear: media.releaseYear.toString(), + imdbId: media.imdbId, + title: media.title, + ...extra, + }; +} + +function addQueryDataToUrl(url: URL, data: Record) { + Object.entries(data).forEach((entry) => { + if (entry[1]) url.searchParams.set(entry[0], entry[1]); + }); +} + +export function makeProviderUrl(base: string) { + const makeUrl = (p: string) => new URL(`${base}${p}`); + return { + scrapeSource(sourceId: string, media: ScrapeMedia) { + const url = makeUrl("/scrape/source"); + addQueryDataToUrl(url, scrapeMediaToQueryMedia(media)); + addQueryDataToUrl(url, { id: sourceId }); + return url.toString(); + }, + scrapeAll(media: ScrapeMedia) { + const url = makeUrl("/scrape"); + addQueryDataToUrl(url, scrapeMediaToQueryMedia(media)); + return url.toString(); + }, + scrapeEmbed(embedId: string, embedUrl: string) { + const url = makeUrl("/scrape/embed"); + addQueryDataToUrl(url, { id: embedId, url: embedUrl }); + return url.toString(); + }, + }; +} + +export function connectServerSideEvents(url: string, endEvents: string[]) { + const; + + return {}; +} diff --git a/src/hooks/useProviderScrape.tsx b/src/hooks/useProviderScrape.tsx index 80ab63a4..4c16d6a8 100644 --- a/src/hooks/useProviderScrape.tsx +++ b/src/hooks/useProviderScrape.tsx @@ -5,6 +5,7 @@ import { } from "@movie-web/providers"; import { RefObject, useCallback, useEffect, useRef, useState } from "react"; +import { makeProviderUrl } from "@/backend/helpers/providerApi"; import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; export interface ScrapingItems { @@ -150,32 +151,8 @@ export function useScrape() { startScrape(); const sseOutput = await new Promise( (resolve, reject) => { - const finalUrl = new URL(`${providerApiUrl}/scrape`); - finalUrl.searchParams.append("type", media.type); - finalUrl.searchParams.append( - "releaseYear", - media.releaseYear.toString() - ); - finalUrl.searchParams.append("title", media.title); - finalUrl.searchParams.append("tmdbId", media.tmdbId); - if (media.imdbId) - finalUrl.searchParams.append("imdbId", media.imdbId); - if (media.type === "show") { - finalUrl.searchParams.append( - "episodeNumber", - media.episode.number.toString() - ); - finalUrl.searchParams.append( - "episodeTmdbId", - media.episode.tmdbId - ); - finalUrl.searchParams.append( - "seasonNumber", - media.season.number.toString() - ); - finalUrl.searchParams.append("seasonTmdbId", media.season.tmdbId); - } - const scrapeEvents = new EventSource(finalUrl.toString()); + const baseUrlMaker = makeProviderUrl(providerApiUrl); + const scrapeEvents = new EventSource(baseUrlMaker.scrapeAll(media)); scrapeEvents.addEventListener("init", (e) => { initEvent(JSON.parse(e.data)); }); From 2bf0b5b03ca3c35a1cf2a6da9266cd68de7121de Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 00:10:46 +0100 Subject: [PATCH 06/16] Metadata fetching --- src/backend/helpers/providerApi.ts | 50 +++++++++++++++-- .../player/atoms/settings/SettingsMenu.tsx | 7 ++- .../atoms/settings/SourceSelectingView.tsx | 5 +- src/hooks/useProviderScrape.tsx | 53 +++++++------------ src/pages/parts/player/MetaPart.tsx | 15 ++++++ 5 files changed, 89 insertions(+), 41 deletions(-) diff --git a/src/backend/helpers/providerApi.ts b/src/backend/helpers/providerApi.ts index 4292ee7d..c6898d2c 100644 --- a/src/backend/helpers/providerApi.ts +++ b/src/backend/helpers/providerApi.ts @@ -1,4 +1,22 @@ -import { ScrapeMedia } from "@movie-web/providers"; +import { MetaOutput, ScrapeMedia } from "@movie-web/providers"; + +import { mwFetch } from "@/backend/helpers/fetch"; + +let metaDataCache: MetaOutput[] | null = null; + +export function setCachedMetadata(data: MetaOutput[]) { + metaDataCache = data; +} + +export function getCachedMetadata(): MetaOutput[] { + return metaDataCache ?? []; +} + +export async function fetchMetadata(base: string) { + if (metaDataCache) return; + const data = await mwFetch(`${base}/metadata`); + metaDataCache = data.flat(); +} function scrapeMediaToQueryMedia(media: ScrapeMedia) { let extra: Record = {}; @@ -15,6 +33,7 @@ function scrapeMediaToQueryMedia(media: ScrapeMedia) { type: media.type, releaseYear: media.releaseYear.toString(), imdbId: media.imdbId, + tmdbId: media.tmdbId, title: media.title, ...extra, }; @@ -48,8 +67,31 @@ export function makeProviderUrl(base: string) { }; } -export function connectServerSideEvents(url: string, endEvents: string[]) { - const; +export function connectServerSideEvents(url: string, endEvents: string[]) { + const eventSource = new EventSource(url); + let promReject: (reason?: any) => void; + let promResolve: (value: T) => void; + const promise = new Promise((resolve, reject) => { + promResolve = resolve; + promReject = reject; + }); - return {}; + endEvents.forEach((evt) => { + eventSource.addEventListener(evt, (e) => { + eventSource.close(); + promResolve(JSON.parse(e.data)); + }); + }); + + eventSource.addEventListener("error", (err) => { + console.error("Failed to connect to SSE", err); + promReject(err); + }); + + return { + promise: () => promise, + on(event: string, cb: (data: Data) => void) { + eventSource.addEventListener(event, (e) => cb(JSON.parse(e.data))); + }, + }; } diff --git a/src/components/player/atoms/settings/SettingsMenu.tsx b/src/components/player/atoms/settings/SettingsMenu.tsx index 4ab971ff..0691156c 100644 --- a/src/components/player/atoms/settings/SettingsMenu.tsx +++ b/src/components/player/atoms/settings/SettingsMenu.tsx @@ -1,6 +1,7 @@ import { useMemo } from "react"; import { useTranslation } from "react-i18next"; +import { getCachedMetadata } from "@/backend/helpers/providerApi"; import { Toggle } from "@/components/buttons/Toggle"; import { Icon, Icons } from "@/components/Icon"; import { useCaptions } from "@/components/player/hooks/useCaptions"; @@ -10,7 +11,6 @@ import { useOverlayRouter } from "@/hooks/useOverlayRouter"; import { usePlayerStore } from "@/stores/player/store"; import { qualityToString } from "@/stores/player/utils/qualities"; import { useSubtitleStore } from "@/stores/subtitles"; -import { providers } from "@/utils/providers"; export function SettingsMenu({ id }: { id: string }) { const { t } = useTranslation(); @@ -23,7 +23,10 @@ export function SettingsMenu({ id }: { id: string }) { const currentSourceId = usePlayerStore((s) => s.sourceId); const sourceName = useMemo(() => { if (!currentSourceId) return "..."; - return providers.getMetadata(currentSourceId)?.name ?? "..."; + const source = getCachedMetadata().find( + (src) => src.id === currentSourceId + ); + return source?.name ?? "..."; }, [currentSourceId]); const { toggleLastUsed } = useCaptions(); diff --git a/src/components/player/atoms/settings/SourceSelectingView.tsx b/src/components/player/atoms/settings/SourceSelectingView.tsx index 83c08914..c3937791 100644 --- a/src/components/player/atoms/settings/SourceSelectingView.tsx +++ b/src/components/player/atoms/settings/SourceSelectingView.tsx @@ -1,6 +1,7 @@ import { ReactNode, useEffect, useMemo, useRef } from "react"; import { useTranslation } from "react-i18next"; +import { getCachedMetadata } from "@/backend/helpers/providerApi"; import { Loading } from "@/components/layout/Loading"; import { useEmbedScraping, @@ -33,7 +34,7 @@ export function EmbedOption(props: { const embedName = useMemo(() => { if (!props.embedId) return unknownEmbedName; - const sourceMeta = providers.getMetadata(props.embedId); + const sourceMeta = getCachedMetadata().find((s) => s.id === props.embedId); return sourceMeta?.name ?? unknownEmbedName; }, [props.embedId, unknownEmbedName]); @@ -61,7 +62,7 @@ export function EmbedSelectionView({ sourceId, id }: EmbedSelectionViewProps) { const sourceName = useMemo(() => { if (!sourceId) return "..."; - const sourceMeta = providers.getMetadata(sourceId); + const sourceMeta = getCachedMetadata().find((s) => s.id === sourceId); return sourceMeta?.name ?? "..."; }, [sourceId]); diff --git a/src/hooks/useProviderScrape.tsx b/src/hooks/useProviderScrape.tsx index 4c16d6a8..c00d6d60 100644 --- a/src/hooks/useProviderScrape.tsx +++ b/src/hooks/useProviderScrape.tsx @@ -5,7 +5,11 @@ import { } from "@movie-web/providers"; import { RefObject, useCallback, useEffect, useRef, useState } from "react"; -import { makeProviderUrl } from "@/backend/helpers/providerApi"; +import { + connectServerSideEvents, + getCachedMetadata, + makeProviderUrl, +} from "@/backend/helpers/providerApi"; import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; export interface ScrapingItems { @@ -37,7 +41,7 @@ function useBaseScrape() { setSources( evt.sourceIds .map((v) => { - const source = providers.getMetadata(v); + const source = getCachedMetadata().find((s) => s.id === v); if (!source) throw new Error("invalid source id"); const out: ScrapingSegment = { name: source.name, @@ -80,7 +84,9 @@ function useBaseScrape() { (evt: ScraperEvent<"discoverEmbeds">) => { setSources((s) => { evt.embeds.forEach((v) => { - const source = providers.getMetadata(v.embedScraperId); + const source = getCachedMetadata().find( + (src) => src.id === v.embedScraperId + ); if (!source) throw new Error("invalid source id"); const out: ScrapingSegment = { embedId: v.embedScraperId, @@ -149,37 +155,18 @@ export function useScrape() { const providerApiUrl = getLoadbalancedProviderApiUrl(); if (providerApiUrl) { startScrape(); - const sseOutput = await new Promise( - (resolve, reject) => { - const baseUrlMaker = makeProviderUrl(providerApiUrl); - const scrapeEvents = new EventSource(baseUrlMaker.scrapeAll(media)); - scrapeEvents.addEventListener("init", (e) => { - initEvent(JSON.parse(e.data)); - }); - scrapeEvents.addEventListener("error", (err) => { - console.error("failed to use provider api", err); - reject(err); - }); - scrapeEvents.addEventListener("start", (e) => - startEvent(JSON.parse(e.data)) - ); - scrapeEvents.addEventListener("update", (e) => - updateEvent(JSON.parse(e.data)) - ); - scrapeEvents.addEventListener("discoverEmbeds", (e) => - discoverEmbedsEvent(JSON.parse(e.data)) - ); - scrapeEvents.addEventListener("completed", (e) => { - scrapeEvents.close(); - resolve(JSON.parse(e.data)); - }); - scrapeEvents.addEventListener("noOutput", () => { - scrapeEvents.close(); - resolve(null); - }); - } + const baseUrlMaker = makeProviderUrl(providerApiUrl); + const conn = connectServerSideEvents( + baseUrlMaker.scrapeAll(media), + ["completed", "noOutput"] ); - return getResult(sseOutput); + conn.on("init", initEvent); + conn.on("start", startEvent); + conn.on("update", updateEvent); + conn.on("discoverEmbeds", discoverEmbedsEvent); + const sseOutput = await conn.promise(); + + return getResult(sseOutput === "" ? null : sseOutput); } if (!providers) return null; diff --git a/src/pages/parts/player/MetaPart.tsx b/src/pages/parts/player/MetaPart.tsx index 1101542c..86a79a8a 100644 --- a/src/pages/parts/player/MetaPart.tsx +++ b/src/pages/parts/player/MetaPart.tsx @@ -3,6 +3,10 @@ import { useHistory, useParams } from "react-router-dom"; import { useAsync } from "react-use"; import type { AsyncReturnType } from "type-fest"; +import { + fetchMetadata, + setCachedMetadata, +} from "@/backend/helpers/providerApi"; import { DetailedMeta, getMetaFromId } from "@/backend/metadata/getmeta"; import { decodeTMDBId } from "@/backend/metadata/tmdb"; import { MWMediaType } from "@/backend/metadata/types/mw"; @@ -14,6 +18,7 @@ import { Paragraph } from "@/components/text/Paragraph"; import { Title } from "@/components/text/Title"; import { ErrorContainer, ErrorLayout } from "@/pages/layouts/ErrorLayout"; import { conf } from "@/setup/config"; +import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; export interface MetaPartProps { onGetMeta?: (meta: DetailedMeta, episodeId?: string) => void; @@ -36,6 +41,16 @@ export function MetaPart(props: MetaPartProps) { const history = useHistory(); const { error, value, loading } = useAsync(async () => { + const providerApiUrl = getLoadbalancedProviderApiUrl(); + if (providerApiUrl) { + await fetchMetadata(providerApiUrl); + } else { + setCachedMetadata([ + ...providers.listSources(), + ...providers.listEmbeds(), + ]); + } + let data: ReturnType = null; try { data = decodeTMDBId(params.media); From 116501e0c1d7299cff062feac453cf1699a93890 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 00:14:13 +0100 Subject: [PATCH 07/16] Source list --- src/components/player/atoms/settings/SourceSelectingView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/player/atoms/settings/SourceSelectingView.tsx b/src/components/player/atoms/settings/SourceSelectingView.tsx index c3937791..beec49d3 100644 --- a/src/components/player/atoms/settings/SourceSelectingView.tsx +++ b/src/components/player/atoms/settings/SourceSelectingView.tsx @@ -138,8 +138,8 @@ export function SourceSelectionView({ const currentSourceId = usePlayerStore((s) => s.sourceId); const sources = useMemo(() => { if (!metaType) return []; - return providers - .listSources() + return getCachedMetadata() + .filter((v) => v.type === "source") .filter((v) => v.mediaTypes?.includes(metaType)); }, [metaType]); From 15d97dda02d68657ef0657e4dadc1f9cc4bbe660 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 00:14:42 +0100 Subject: [PATCH 08/16] remove unused import --- src/components/player/atoms/settings/SourceSelectingView.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/player/atoms/settings/SourceSelectingView.tsx b/src/components/player/atoms/settings/SourceSelectingView.tsx index beec49d3..2c4440ab 100644 --- a/src/components/player/atoms/settings/SourceSelectingView.tsx +++ b/src/components/player/atoms/settings/SourceSelectingView.tsx @@ -11,7 +11,6 @@ import { Menu } from "@/components/player/internals/ContextMenu"; import { SelectableLink } from "@/components/player/internals/ContextMenu/Links"; import { useOverlayRouter } from "@/hooks/useOverlayRouter"; import { usePlayerStore } from "@/stores/player/store"; -import { providers } from "@/utils/providers"; export interface SourceSelectionViewProps { id: string; From ca2e20fdbc541f7fc399cd73f469c6a8f86d77f8 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 18:09:05 +0100 Subject: [PATCH 09/16] Add provider-api to source selection menu --- .../player/hooks/useSourceSelection.ts | 62 +++++++++++++++---- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/src/components/player/hooks/useSourceSelection.ts b/src/components/player/hooks/useSourceSelection.ts index 094dd458..b3cd5a8e 100644 --- a/src/components/player/hooks/useSourceSelection.ts +++ b/src/components/player/hooks/useSourceSelection.ts @@ -5,6 +5,10 @@ import { } from "@movie-web/providers"; import { useAsyncFn } from "react-use"; +import { + connectServerSideEvents, + makeProviderUrl, +} from "@/backend/helpers/providerApi"; import { scrapeSourceOutputToProviderMetric, useReportProviders, @@ -14,7 +18,7 @@ import { convertRunoutputToSource } from "@/components/player/utils/convertRunou import { useOverlayRouter } from "@/hooks/useOverlayRouter"; import { metaToScrapeMedia } from "@/stores/player/slices/source"; import { usePlayerStore } from "@/stores/player/store"; -import { providers } from "@/utils/providers"; +import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; export function useEmbedScraping( routerId: string, @@ -31,13 +35,23 @@ export function useEmbedScraping( const { report } = useReportProviders(); const [request, run] = useAsyncFn(async () => { + const providerApiUrl = getLoadbalancedProviderApiUrl(); let result: EmbedOutput | undefined; if (!meta) return; try { - result = await providers.runEmbedScraper({ - id: embedId, - url, - }); + if (providerApiUrl) { + const baseUrlMaker = makeProviderUrl(providerApiUrl); + const conn = connectServerSideEvents( + baseUrlMaker.scrapeEmbed(embedId, url), + ["completed"] + ); + result = await conn.promise(); + } else { + result = await providers.runEmbedScraper({ + id: embedId, + url, + }); + } } catch (err) { console.error(`Failed to scrape ${embedId}`, err); const notFound = err instanceof NotFoundError; @@ -85,13 +99,23 @@ export function useSourceScraping(sourceId: string | null, routerId: string) { const [request, run] = useAsyncFn(async () => { if (!sourceId || !meta) return null; const scrapeMedia = metaToScrapeMedia(meta); + const providerApiUrl = getLoadbalancedProviderApiUrl(); let result: SourcererOutput | undefined; try { - result = await providers.runSourceScraper({ - id: sourceId, - media: scrapeMedia, - }); + if (providerApiUrl) { + const baseUrlMaker = makeProviderUrl(providerApiUrl); + const conn = connectServerSideEvents( + baseUrlMaker.scrapeSource(sourceId, scrapeMedia), + ["completed"] + ); + result = await conn.promise(); + } else { + result = await providers.runSourceScraper({ + id: sourceId, + media: scrapeMedia, + }); + } } catch (err) { console.error(`Failed to scrape ${sourceId}`, err); const notFound = err instanceof NotFoundError; @@ -120,10 +144,22 @@ export function useSourceScraping(sourceId: string | null, routerId: string) { let embedResult: EmbedOutput | undefined; if (!meta) return; try { - embedResult = await providers.runEmbedScraper({ - id: result.embeds[0].embedId, - url: result.embeds[0].url, - }); + if (providerApiUrl) { + const baseUrlMaker = makeProviderUrl(providerApiUrl); + const conn = connectServerSideEvents( + baseUrlMaker.scrapeEmbed( + result.embeds[0].embedId, + result.embeds[0].url + ), + ["completed"] + ); + embedResult = await conn.promise(); + } else { + embedResult = await providers.runEmbedScraper({ + id: result.embeds[0].embedId, + url: result.embeds[0].url, + }); + } } catch (err) { console.error(`Failed to scrape ${result.embeds[0].embedId}`, err); const notFound = err instanceof NotFoundError; From 4847980947a490bf7e8393f5b2d60852aaca5e9b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 18:47:54 +0100 Subject: [PATCH 10/16] Improve error handling for providers api --- src/backend/helpers/providerApi.ts | 25 +++++++++++++++++-- .../player/hooks/useSourceSelection.ts | 6 ++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/backend/helpers/providerApi.ts b/src/backend/helpers/providerApi.ts index c6898d2c..affe387f 100644 --- a/src/backend/helpers/providerApi.ts +++ b/src/backend/helpers/providerApi.ts @@ -1,4 +1,4 @@ -import { MetaOutput, ScrapeMedia } from "@movie-web/providers"; +import { MetaOutput, NotFoundError, ScrapeMedia } from "@movie-web/providers"; import { mwFetch } from "@/backend/helpers/fetch"; @@ -83,11 +83,32 @@ export function connectServerSideEvents(url: string, endEvents: string[]) { }); }); - eventSource.addEventListener("error", (err) => { + eventSource.addEventListener("error", (err: MessageEvent) => { + eventSource.close(); + if (err.data) { + const data = JSON.parse(err.data); + let errObj = new Error("scrape error"); + if (data.name === NotFoundError.name) + errObj = new NotFoundError("Notfound from server"); + Object.assign(errObj, data); + promReject(errObj); + return; + } + console.error("Failed to connect to SSE", err); promReject(err); }); + eventSource.addEventListener("message", (ev) => { + if (!ev) { + eventSource.close(); + return; + } + setTimeout(() => { + promReject(new Error("SSE closed improperly")); + }, 1000); + }); + return { promise: () => promise, on(event: string, cb: (data: Data) => void) { diff --git a/src/components/player/hooks/useSourceSelection.ts b/src/components/player/hooks/useSourceSelection.ts index b3cd5a8e..d1a50792 100644 --- a/src/components/player/hooks/useSourceSelection.ts +++ b/src/components/player/hooks/useSourceSelection.ts @@ -43,7 +43,7 @@ export function useEmbedScraping( const baseUrlMaker = makeProviderUrl(providerApiUrl); const conn = connectServerSideEvents( baseUrlMaker.scrapeEmbed(embedId, url), - ["completed"] + ["completed", "noOutput"] ); result = await conn.promise(); } else { @@ -107,7 +107,7 @@ export function useSourceScraping(sourceId: string | null, routerId: string) { const baseUrlMaker = makeProviderUrl(providerApiUrl); const conn = connectServerSideEvents( baseUrlMaker.scrapeSource(sourceId, scrapeMedia), - ["completed"] + ["completed", "noOutput"] ); result = await conn.promise(); } else { @@ -151,7 +151,7 @@ export function useSourceScraping(sourceId: string | null, routerId: string) { result.embeds[0].embedId, result.embeds[0].url ), - ["completed"] + ["completed", "noOutput"] ); embedResult = await conn.promise(); } else { From b5a11ef0008071b5b4c910107b2f55e2a9f365fe Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 20:41:56 +0100 Subject: [PATCH 11/16] turnstile integration for provider api --- index.html | 4 +- package.json | 1 + pnpm-lock.yaml | 13 +++ src/backend/helpers/providerApi.ts | 37 ++++++++- .../player/hooks/useSourceSelection.ts | 6 +- src/hooks/useProviderScrape.tsx | 2 +- src/index.tsx | 8 +- src/setup/config.ts | 5 ++ src/stores/turnstile/index.tsx | 81 +++++++++++++++++++ 9 files changed, 144 insertions(+), 13 deletions(-) create mode 100644 src/stores/turnstile/index.tsx diff --git a/index.html b/index.html index 3335369e..71b10ed7 100644 --- a/index.html +++ b/index.html @@ -20,7 +20,7 @@ - + @@ -59,4 +59,4 @@ - \ No newline at end of file + diff --git a/package.json b/package.json index d2a5dbb2..6a72405a 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "react-i18next": "^12.1.1", "react-router-dom": "^5.2.0", "react-sticky-el": "^2.1.0", + "react-turnstile": "^1.1.2", "react-use": "^17.4.0", "slugify": "^1.6.6", "subsrt-ts": "^2.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c2e88592..3be020b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,9 @@ dependencies: react-sticky-el: specifier: ^2.1.0 version: 2.1.0(react-dom@17.0.2)(react@17.0.2) + react-turnstile: + specifier: ^1.1.2 + version: 1.1.2(react-dom@17.0.2)(react@17.0.2) react-use: specifier: ^17.4.0 version: 17.4.0(react-dom@17.0.2)(react@17.0.2) @@ -5321,6 +5324,16 @@ packages: react-dom: 17.0.2(react@17.0.2) dev: false + /react-turnstile@1.1.2(react-dom@17.0.2)(react@17.0.2): + resolution: {integrity: sha512-wfhSf4JtXlmLRkfxMryU8yEeCbh401muKoInhx+TegYwP8RprUW5XPZa8WnCNZiYpMy1i6IXAb1Ar7xj5HxJag==} + peerDependencies: + react: '>= 17.0.0' + react-dom: '>= 17.0.0' + dependencies: + react: 17.0.2 + react-dom: 17.0.2(react@17.0.2) + dev: false + /react-universal-interface@0.6.2(react@17.0.2)(tslib@2.6.2): resolution: {integrity: sha512-dg8yXdcQmvgR13RIlZbTRQOoUrDciFVoSBZILwjE2LFISxZZ8loVJKAkuzswl5js8BHda79bIb2b84ehU8IjXw==} peerDependencies: diff --git a/src/backend/helpers/providerApi.ts b/src/backend/helpers/providerApi.ts index affe387f..ed79bd48 100644 --- a/src/backend/helpers/providerApi.ts +++ b/src/backend/helpers/providerApi.ts @@ -1,8 +1,10 @@ import { MetaOutput, NotFoundError, ScrapeMedia } from "@movie-web/providers"; import { mwFetch } from "@/backend/helpers/fetch"; +import { getTurnstileToken, isTurnstileInitialized } from "@/stores/turnstile"; let metaDataCache: MetaOutput[] | null = null; +let token: null | string = null; export function setCachedMetadata(data: MetaOutput[]) { metaDataCache = data; @@ -12,6 +14,20 @@ export function getCachedMetadata(): MetaOutput[] { return metaDataCache ?? []; } +function getTokenIfValid(): null | string { + if (!token) return null; + const parts = token.split("."); + if (parts.length !== 3) return null; + try { + const parsedData = JSON.parse(atob(parts[2])); + if (!parsedData.exp) return token; + if (Date.now() < parsedData.exp) return token; + } catch { + // we dont care about parse errors + } + return null; +} + export async function fetchMetadata(base: string) { if (metaDataCache) return; const data = await mwFetch(`${base}/metadata`); @@ -67,8 +83,21 @@ export function makeProviderUrl(base: string) { }; } -export function connectServerSideEvents(url: string, endEvents: string[]) { - const eventSource = new EventSource(url); +export async function connectServerSideEvents( + url: string, + endEvents: string[] +) { + // fetch token to use + let apiToken = getTokenIfValid(); + if (!apiToken && isTurnstileInitialized()) { + apiToken = await getTurnstileToken(); + } + + // insert token, if its set + const parsedUrl = new URL(url); + if (apiToken) parsedUrl.searchParams.set("token", apiToken); + const eventSource = new EventSource(parsedUrl.toString()); + let promReject: (reason?: any) => void; let promResolve: (value: T) => void; const promise = new Promise((resolve, reject) => { @@ -83,6 +112,10 @@ export function connectServerSideEvents(url: string, endEvents: string[]) { }); }); + eventSource.addEventListener("token", (e) => { + token = JSON.parse(e.data); + }); + eventSource.addEventListener("error", (err: MessageEvent) => { eventSource.close(); if (err.data) { diff --git a/src/components/player/hooks/useSourceSelection.ts b/src/components/player/hooks/useSourceSelection.ts index d1a50792..826ed3d1 100644 --- a/src/components/player/hooks/useSourceSelection.ts +++ b/src/components/player/hooks/useSourceSelection.ts @@ -41,7 +41,7 @@ export function useEmbedScraping( try { if (providerApiUrl) { const baseUrlMaker = makeProviderUrl(providerApiUrl); - const conn = connectServerSideEvents( + const conn = await connectServerSideEvents( baseUrlMaker.scrapeEmbed(embedId, url), ["completed", "noOutput"] ); @@ -105,7 +105,7 @@ export function useSourceScraping(sourceId: string | null, routerId: string) { try { if (providerApiUrl) { const baseUrlMaker = makeProviderUrl(providerApiUrl); - const conn = connectServerSideEvents( + const conn = await connectServerSideEvents( baseUrlMaker.scrapeSource(sourceId, scrapeMedia), ["completed", "noOutput"] ); @@ -146,7 +146,7 @@ export function useSourceScraping(sourceId: string | null, routerId: string) { try { if (providerApiUrl) { const baseUrlMaker = makeProviderUrl(providerApiUrl); - const conn = connectServerSideEvents( + const conn = await connectServerSideEvents( baseUrlMaker.scrapeEmbed( result.embeds[0].embedId, result.embeds[0].url diff --git a/src/hooks/useProviderScrape.tsx b/src/hooks/useProviderScrape.tsx index c00d6d60..cf50b6a9 100644 --- a/src/hooks/useProviderScrape.tsx +++ b/src/hooks/useProviderScrape.tsx @@ -156,7 +156,7 @@ export function useScrape() { if (providerApiUrl) { startScrape(); const baseUrlMaker = makeProviderUrl(providerApiUrl); - const conn = connectServerSideEvents( + const conn = await connectServerSideEvents( baseUrlMaker.scrapeAll(media), ["completed", "noOutput"] ); diff --git a/src/index.tsx b/src/index.tsx index 8ef5a560..e943c462 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -10,6 +10,7 @@ import ReactDOM from "react-dom"; import { HelmetProvider } from "react-helmet-async"; import { useTranslation } from "react-i18next"; import { BrowserRouter, HashRouter } from "react-router-dom"; +import Turnstile from "react-turnstile"; import { useAsync } from "react-use"; import { Button } from "@/components/buttons/Button"; @@ -30,16 +31,12 @@ import { useLanguageStore } from "@/stores/language"; import { ProgressSyncer } from "@/stores/progress/ProgressSyncer"; import { SettingsSyncer } from "@/stores/subtitles/SettingsSyncer"; import { ThemeProvider } from "@/stores/theme"; +import { TurnstileProvider } from "@/stores/turnstile"; import { initializeChromecast } from "./setup/chromecast"; import { initializeOldStores } from "./stores/__old/migrations"; // initialize -const key = - (window as any)?.__CONFIG__?.VITE_KEY ?? import.meta.env.VITE_KEY ?? null; -if (key) { - (window as any).initMW(conf().PROXY_URLS, key); -} initializeChromecast(); function LoadingScreen(props: { type: "user" | "lazy" }) { @@ -148,6 +145,7 @@ function TheRouter(props: { children: ReactNode }) { ReactDOM.render( + }> diff --git a/src/setup/config.ts b/src/setup/config.ts index 47d18412..2f16714b 100644 --- a/src/setup/config.ts +++ b/src/setup/config.ts @@ -17,6 +17,7 @@ interface Config { NORMAL_ROUTER: boolean; BACKEND_URL: string; DISALLOWED_IDS: string; + TURNSTILE_KEY: string; } export interface RuntimeConfig { @@ -30,6 +31,7 @@ export interface RuntimeConfig { PROXY_URLS: string[]; BACKEND_URL: string; DISALLOWED_IDS: string[]; + TURNSTILE_KEY: string | null; } const env: Record = { @@ -43,6 +45,7 @@ const env: Record = { NORMAL_ROUTER: import.meta.env.VITE_NORMAL_ROUTER, BACKEND_URL: import.meta.env.VITE_BACKEND_URL, DISALLOWED_IDS: import.meta.env.VITE_DISALLOWED_IDS, + TURNSTILE_KEY: import.meta.env.VITE_TURNSTILE_KEY, }; // loads from different locations, in order: environment (VITE_{KEY}), window (public/config.js) @@ -63,6 +66,7 @@ function getKey(key: keyof Config, defaultString?: string): string { export function conf(): RuntimeConfig { const dmcaEmail = getKey("DMCA_EMAIL"); + const turnstileKey = getKey("TURNSTILE_KEY"); return { APP_VERSION, GITHUB_LINK, @@ -75,6 +79,7 @@ export function conf(): RuntimeConfig { .split(",") .map((v) => v.trim()), NORMAL_ROUTER: getKey("NORMAL_ROUTER", "false") === "true", + TURNSTILE_KEY: turnstileKey.length > 0 ? turnstileKey : null, DISALLOWED_IDS: getKey("DISALLOWED_IDS", "") .split(",") .map((v) => v.trim()) diff --git a/src/stores/turnstile/index.tsx b/src/stores/turnstile/index.tsx new file mode 100644 index 00000000..907ffc2e --- /dev/null +++ b/src/stores/turnstile/index.tsx @@ -0,0 +1,81 @@ +import Turnstile, { BoundTurnstileObject } from "react-turnstile"; +import { create } from "zustand"; +import { immer } from "zustand/middleware/immer"; + +import { conf } from "@/setup/config"; + +export interface TurnstileStore { + turnstile: BoundTurnstileObject | null; + cbs: ((token: string | null) => void)[]; + setTurnstile(v: BoundTurnstileObject | null): void; + getToken(): Promise; + processToken(token: string | null): void; +} + +export const useTurnstileStore = create( + immer((set, get) => ({ + turnstile: null, + cbs: [], + processToken(token) { + const cbs = get().cbs; + cbs.forEach((fn) => fn(token)); + set((s) => { + s.cbs = []; + }); + }, + getToken() { + return new Promise((resolve, reject) => { + set((s) => { + s.cbs = [ + ...s.cbs, + (token) => { + if (!token) reject(new Error("Failed to get token")); + else resolve(token); + }, + ]; + }); + }); + }, + setTurnstile(v) { + set((s) => { + s.turnstile = v; + }); + }, + })) +); + +export function getTurnstile() { + return useTurnstileStore.getState().turnstile; +} + +export function isTurnstileInitialized() { + return !!getTurnstile(); +} + +export function getTurnstileToken() { + const turnstile = getTurnstile(); + turnstile?.reset(); + turnstile?.execute(); + return useTurnstileStore.getState().getToken(); +} + +export function TurnstileProvider() { + const siteKey = conf().TURNSTILE_KEY; + const setTurnstile = useTurnstileStore((s) => s.setTurnstile); + const processToken = useTurnstileStore((s) => s.processToken); + if (!siteKey) return null; + return ( + { + setTurnstile(bound); + }} + onError={() => { + processToken(null); + }} + onVerify={(token) => { + processToken(token); + }} + /> + ); +} From bc51076369c0001e91865fa4ddec3a2fd1a1bd96 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 21:00:25 +0100 Subject: [PATCH 12/16] Remove turnstile cdn link --- index.html | 1 - 1 file changed, 1 deletion(-) diff --git a/index.html b/index.html index 71b10ed7..4dc4ac00 100644 --- a/index.html +++ b/index.html @@ -20,7 +20,6 @@ - From 1acc81205a528a450649b922ba6385a81e4b5caf Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 19 Dec 2023 22:39:14 +0100 Subject: [PATCH 13/16] Add prefixes to tokens --- package.json | 1 + pnpm-lock.yaml | 8 ++++++++ src/backend/helpers/providerApi.ts | 13 ++++++------- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 6a72405a..615f4068 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "i18next": "^22.4.5", "immer": "^10.0.2", "iso-639-1": "^3.1.0", + "jwt-decode": "^4.0.0", "lodash.isequal": "^4.5.0", "nanoid": "^5.0.4", "node-forge": "^1.3.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3be020b8..2039c0fa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ dependencies: iso-639-1: specifier: ^3.1.0 version: 3.1.0 + jwt-decode: + specifier: ^4.0.0 + version: 4.0.0 lodash.isequal: specifier: ^4.5.0 version: 4.5.0 @@ -4526,6 +4529,11 @@ packages: resolution: {integrity: sha512-cxQGGUiit6CGUpuuiezY8N4m1wgF4o7127rXEXDFcxeDUFfdV7gSkwA26Fe2wWBiNQq2SZOgN4gSmMxB/StA8Q==} dev: true + /jwt-decode@4.0.0: + resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==} + engines: {node: '>=18'} + dev: false + /keyv@4.5.3: resolution: {integrity: sha512-QCiSav9WaX1PgETJ+SpNnx2PRRapJ/oRSXM4VO5OGYGSjrxbKPVFVhB3l2OCbLCk329N8qyAtsJjSjvVBWzEug==} dependencies: diff --git a/src/backend/helpers/providerApi.ts b/src/backend/helpers/providerApi.ts index ed79bd48..fa608e6a 100644 --- a/src/backend/helpers/providerApi.ts +++ b/src/backend/helpers/providerApi.ts @@ -1,4 +1,5 @@ import { MetaOutput, NotFoundError, ScrapeMedia } from "@movie-web/providers"; +import { jwtDecode } from "jwt-decode"; import { mwFetch } from "@/backend/helpers/fetch"; import { getTurnstileToken, isTurnstileInitialized } from "@/stores/turnstile"; @@ -16,13 +17,11 @@ export function getCachedMetadata(): MetaOutput[] { function getTokenIfValid(): null | string { if (!token) return null; - const parts = token.split("."); - if (parts.length !== 3) return null; try { - const parsedData = JSON.parse(atob(parts[2])); - if (!parsedData.exp) return token; - if (Date.now() < parsedData.exp) return token; - } catch { + const body = jwtDecode(token); + if (!body.exp) return `jwt|${token}`; + if (Date.now() / 1000 < body.exp) return `jwt|${token}`; + } catch (err) { // we dont care about parse errors } return null; @@ -90,7 +89,7 @@ export async function connectServerSideEvents( // fetch token to use let apiToken = getTokenIfValid(); if (!apiToken && isTurnstileInitialized()) { - apiToken = await getTurnstileToken(); + apiToken = `turnstile|${await getTurnstileToken()}`; } // insert token, if its set From d998dceb1ec283bf1dc41f6ef658d10f1649b14f Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 20 Dec 2023 15:02:05 +0100 Subject: [PATCH 14/16] Fix proxiedFetch not using new turnstile integration --- src/backend/helpers/fetch.ts | 53 +++++++++--------------- src/backend/helpers/providerApi.ts | 14 ++++--- src/pages/parts/admin/WorkerTestPart.tsx | 14 +++---- 3 files changed, 35 insertions(+), 46 deletions(-) diff --git a/src/backend/helpers/fetch.ts b/src/backend/helpers/fetch.ts index fca9ab89..646d8ab9 100644 --- a/src/backend/helpers/fetch.ts +++ b/src/backend/helpers/fetch.ts @@ -1,5 +1,6 @@ -import { FetchOptions, FetchResponse, ofetch } from "ofetch"; +import { ofetch } from "ofetch"; +import { getApiToken } from "@/backend/helpers/providerApi"; import { getLoadbalancedProxyUrl } from "@/utils/providers"; type P = Parameters>; @@ -21,7 +22,11 @@ export function mwFetch(url: string, ops: P[1] = {}): R { return baseFetch(url, ops); } -export function proxiedFetch(url: string, ops: P[1] = {}): R { +export async function singularProxiedFetch( + proxyUrl: string, + url: string, + ops: P[1] = {} +): R { let combinedUrl = ops?.baseURL ?? ""; if ( combinedUrl.length > 0 && @@ -45,45 +50,25 @@ export function proxiedFetch(url: string, ops: P[1] = {}): R { parsedUrl.searchParams.set(k, v); }); - return baseFetch(getLoadbalancedProxyUrl(), { + let headers = ops.headers ?? {}; + const apiToken = await getApiToken(); + if (apiToken) + headers = { + ...headers, + "X-Token": apiToken, + }; + + return baseFetch(proxyUrl, { ...ops, baseURL: undefined, params: { destination: parsedUrl.toString(), }, query: {}, + headers, }); } -export function rawProxiedFetch( - url: string, - ops: FetchOptions = {} -): Promise> { - let combinedUrl = ops?.baseURL ?? ""; - if ( - combinedUrl.length > 0 && - combinedUrl.endsWith("/") && - url.startsWith("/") - ) - combinedUrl += url.slice(1); - else if ( - combinedUrl.length > 0 && - !combinedUrl.endsWith("/") && - !url.startsWith("/") - ) - combinedUrl += `/${url}`; - else combinedUrl += url; - - const parsedUrl = new URL(combinedUrl); - Object.entries(ops?.params ?? {}).forEach(([k, v]) => { - parsedUrl.searchParams.set(k, v); - }); - - return baseFetch.raw(getLoadbalancedProxyUrl(), { - ...ops, - baseURL: undefined, - params: { - destination: parsedUrl.toString(), - }, - }); +export function proxiedFetch(url: string, ops: P[1] = {}): R { + return singularProxiedFetch(getLoadbalancedProxyUrl(), url, ops); } diff --git a/src/backend/helpers/providerApi.ts b/src/backend/helpers/providerApi.ts index fa608e6a..89d3eb13 100644 --- a/src/backend/helpers/providerApi.ts +++ b/src/backend/helpers/providerApi.ts @@ -82,15 +82,19 @@ export function makeProviderUrl(base: string) { }; } -export async function connectServerSideEvents( - url: string, - endEvents: string[] -) { - // fetch token to use +export async function getApiToken(): Promise { let apiToken = getTokenIfValid(); if (!apiToken && isTurnstileInitialized()) { apiToken = `turnstile|${await getTurnstileToken()}`; } + return apiToken; +} + +export async function connectServerSideEvents( + url: string, + endEvents: string[] +) { + const apiToken = await getApiToken(); // insert token, if its set const parsedUrl = new URL(url); diff --git a/src/pages/parts/admin/WorkerTestPart.tsx b/src/pages/parts/admin/WorkerTestPart.tsx index e1314e0a..88230f11 100644 --- a/src/pages/parts/admin/WorkerTestPart.tsx +++ b/src/pages/parts/admin/WorkerTestPart.tsx @@ -2,7 +2,7 @@ import classNames from "classnames"; import { useMemo, useState } from "react"; import { useAsyncFn } from "react-use"; -import { mwFetch } from "@/backend/helpers/fetch"; +import { singularProxiedFetch } from "@/backend/helpers/fetch"; import { Button } from "@/components/buttons/Button"; import { Icon, Icons } from "@/components/Icon"; import { Box } from "@/components/layout/Box"; @@ -69,11 +69,11 @@ export function WorkerTestPart() { }); continue; } - await mwFetch(worker.url, { - query: { - destination: "https://postman-echo.com/get", - }, - }); + await singularProxiedFetch( + worker.url, + "https://postman-echo.com/get", + {} + ); updateWorker(worker.id, { id: worker.id, status: "success", @@ -94,7 +94,7 @@ export function WorkerTestPart() {

{workerList.length} worker(s) registered

{workerList.map((v, i) => { - const s = workerState.find((segment) => segment.id); + const s = workerState.find((segment) => segment.id === v.id); const name = `Worker ${i + 1}`; if (!s) return ; if (s.status === "error") From 4db6dcca48b50843d4ee459208430c7465609ef9 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 20 Dec 2023 15:08:04 +0100 Subject: [PATCH 15/16] Implement setting api token after response --- src/backend/helpers/fetch.ts | 7 ++++++- src/backend/helpers/providerApi.ts | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/backend/helpers/fetch.ts b/src/backend/helpers/fetch.ts index 646d8ab9..96762f74 100644 --- a/src/backend/helpers/fetch.ts +++ b/src/backend/helpers/fetch.ts @@ -1,6 +1,6 @@ import { ofetch } from "ofetch"; -import { getApiToken } from "@/backend/helpers/providerApi"; +import { getApiToken, setApiToken } from "@/backend/helpers/providerApi"; import { getLoadbalancedProxyUrl } from "@/utils/providers"; type P = Parameters>; @@ -66,6 +66,11 @@ export async function singularProxiedFetch( }, query: {}, headers, + onResponse(context) { + const tokenHeader = context.response.headers.get("X-Token"); + if (tokenHeader) setApiToken(tokenHeader); + ops.onResponse?.(context); + }, }); } diff --git a/src/backend/helpers/providerApi.ts b/src/backend/helpers/providerApi.ts index 89d3eb13..632a1eb8 100644 --- a/src/backend/helpers/providerApi.ts +++ b/src/backend/helpers/providerApi.ts @@ -15,6 +15,10 @@ export function getCachedMetadata(): MetaOutput[] { return metaDataCache ?? []; } +export function setApiToken(newToken: string) { + token = newToken; +} + function getTokenIfValid(): null | string { if (!token) return null; try { @@ -116,7 +120,7 @@ export async function connectServerSideEvents( }); eventSource.addEventListener("token", (e) => { - token = JSON.parse(e.data); + setApiToken(JSON.parse(e.data)); }); eventSource.addEventListener("error", (err: MessageEvent) => { From d9fd16613a14eb4ee2addec44fa5a84e924393d1 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 20 Dec 2023 15:22:05 +0100 Subject: [PATCH 16/16] Implement turnstile on provider package simple proxy --- src/utils/providers.ts | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/utils/providers.ts b/src/utils/providers.ts index 73c3849e..d38796ac 100644 --- a/src/utils/providers.ts +++ b/src/utils/providers.ts @@ -7,6 +7,7 @@ import { targets, } from "@movie-web/providers"; +import { getApiToken, setApiToken } from "@/backend/helpers/providerApi"; import { getProviderApiUrls, getProxyUrls } from "@/utils/proxyUrls"; function makeLoadbalancedList(getter: () => string[]) { @@ -26,11 +27,32 @@ export const getLoadbalancedProxyUrl = makeLoadbalancedList(getProxyUrls); export const getLoadbalancedProviderApiUrl = makeLoadbalancedList(getProviderApiUrls); +async function fetchButWithApiTokens( + input: RequestInfo | URL, + init?: RequestInit | undefined +): Promise { + const apiToken = await getApiToken(); + const headers = new Headers(init?.headers); + if (apiToken) headers.set("X-Token", apiToken); + const response = await fetch( + input, + init + ? { + ...init, + headers, + } + : undefined + ); + const newApiToken = response.headers.get("X-Token"); + if (newApiToken) setApiToken(newApiToken); + return response; +} + function makeLoadBalancedSimpleProxyFetcher() { - const fetcher: ProviderBuilderOptions["fetcher"] = (a, b) => { + const fetcher: ProviderBuilderOptions["fetcher"] = async (a, b) => { const currentFetcher = makeSimpleProxyFetcher( getLoadbalancedProxyUrl(), - fetch + fetchButWithApiTokens ); return currentFetcher(a, b); };