From 4f9ef382dc0341c0645550db2b5449bd04df7a0c Mon Sep 17 00:00:00 2001
From: mrjvs <jellevs@gmail.com>
Date: Wed, 22 Feb 2023 20:26:19 +0100
Subject: [PATCH] provider and embed scraper tools

---
 src/setup/App.tsx                          |   4 +
 src/views/developer/DeveloperView.tsx      |   5 +
 src/views/developer/EmbedTesterView.tsx    | 136 ++++++++++++++++++
 src/views/developer/ProviderTesterView.tsx | 159 +++++++++++++++++++++
 src/views/developer/VideoTesterView.tsx    |  40 +++++-
 5 files changed, 337 insertions(+), 7 deletions(-)
 create mode 100644 src/views/developer/EmbedTesterView.tsx
 create mode 100644 src/views/developer/ProviderTesterView.tsx

diff --git a/src/setup/App.tsx b/src/setup/App.tsx
index 375f5d6d..03388d58 100644
--- a/src/setup/App.tsx
+++ b/src/setup/App.tsx
@@ -9,6 +9,8 @@ import { MWMediaType } from "@/backend/metadata/types";
 import { V2MigrationView } from "@/views/other/v2Migration";
 import { DeveloperView } from "@/views/developer/DeveloperView";
 import { VideoTesterView } from "@/views/developer/VideoTesterView";
+import { ProviderTesterView } from "@/views/developer/ProviderTesterView";
+import { EmbedTesterView } from "@/views/developer/EmbedTesterView";
 
 function App() {
   return (
@@ -33,6 +35,8 @@ function App() {
           {/* other */}
           <Route exact path="/dev" component={DeveloperView} />
           <Route exact path="/dev/video" component={VideoTesterView} />
+          <Route exact path="/dev/providers" component={ProviderTesterView} />
+          <Route exact path="/dev/embeds" component={EmbedTesterView} />
           <Route path="*" component={NotFoundPage} />
         </Switch>
       </BookmarkContextProvider>
diff --git a/src/views/developer/DeveloperView.tsx b/src/views/developer/DeveloperView.tsx
index f2a9d30d..419293a5 100644
--- a/src/views/developer/DeveloperView.tsx
+++ b/src/views/developer/DeveloperView.tsx
@@ -14,6 +14,11 @@ export function DeveloperView() {
           direction="right"
           linkText="Provider tester"
         />
+        <ArrowLink
+          to="/dev/embeds"
+          direction="right"
+          linkText="Embed scraper tester"
+        />
         <ArrowLink to="/dev/video" direction="right" linkText="Video tester" />
       </ThinContainer>
     </div>
diff --git a/src/views/developer/EmbedTesterView.tsx b/src/views/developer/EmbedTesterView.tsx
new file mode 100644
index 00000000..dd4cef10
--- /dev/null
+++ b/src/views/developer/EmbedTesterView.tsx
@@ -0,0 +1,136 @@
+import { MWEmbed, MWEmbedScraper, MWEmbedType } from "@/backend/helpers/embed";
+import { getEmbeds } from "@/backend/helpers/register";
+import { runEmbedScraper } from "@/backend/helpers/run";
+import { MWStream } from "@/backend/helpers/streams";
+import { Button } from "@/components/Button";
+import { Navigation } from "@/components/layout/Navigation";
+import { ArrowLink } from "@/components/text/ArrowLink";
+import { Title } from "@/components/text/Title";
+import { useLoading } from "@/hooks/useLoading";
+import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
+
+interface MediaSelectorProps {
+  embedType: MWEmbedType;
+  onSelect: (meta: MWEmbed) => void;
+}
+
+interface EmbedScraperSelectorProps {
+  onSelect: (embedScraperId: string) => void;
+}
+
+interface MediaScraperProps {
+  embed: MWEmbed;
+  scraper: MWEmbedScraper;
+}
+
+function MediaSelector(props: MediaSelectorProps) {
+  const [url, setUrl] = useState("");
+
+  const select = useCallback(
+    (urlSt: string) => {
+      props.onSelect({
+        type: props.embedType,
+        url: urlSt,
+      });
+    },
+    [props]
+  );
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Input embed url</Title>
+      <div className="mb-4 flex gap-4">
+        <input
+          type="text"
+          placeholder="embed url here..."
+          value={url}
+          onChange={(e) => setUrl(e.target.value)}
+        />
+        <Button onClick={() => select(url)}>Run scraper</Button>
+      </div>
+    </div>
+  );
+}
+
+function MediaScraper(props: MediaScraperProps) {
+  const [results, setResults] = useState<MWStream | null>(null);
+  const [percentage, setPercentage] = useState(0);
+
+  const [scrape, loading, error] = useLoading(async (url: string) => {
+    const data = await runEmbedScraper(props.scraper, {
+      url,
+      progress(num) {
+        console.log(`SCRAPING AT ${num}%`);
+        setPercentage(num);
+      },
+    });
+    console.log("got data", data);
+    setResults(data);
+  });
+
+  useEffect(() => {
+    if (props.embed) {
+      scrape(props.embed.url);
+    }
+  }, [props.embed, scrape]);
+
+  if (loading) return <p>Scraping... ({percentage}%)</p>;
+  if (error) return <p>Errored, check console</p>;
+
+  return (
+    <div>
+      <Title className="mb-8">Output data</Title>
+      <code>
+        <pre>{JSON.stringify(results, null, 2)}</pre>
+      </code>
+    </div>
+  );
+}
+
+function EmbedScraperSelector(props: EmbedScraperSelectorProps) {
+  const embedScrapers = getEmbeds();
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Choose embed scraper</Title>
+      {embedScrapers.map((v) => (
+        <ArrowLink
+          key={v.id}
+          onClick={() => props.onSelect(v.id)}
+          direction="right"
+          linkText={v.displayName}
+        />
+      ))}
+    </div>
+  );
+}
+
+export function EmbedTesterView() {
+  const [embed, setEmbed] = useState<MWEmbed | null>(null);
+  const [embedScraperId, setEmbedScraperId] = useState<string | null>(null);
+  const embedScraper = useMemo(
+    () => getEmbeds().find((v) => v.id === embedScraperId),
+    [embedScraperId]
+  );
+
+  let content: ReactNode = null;
+  if (!embedScraperId || !embedScraper) {
+    content = <EmbedScraperSelector onSelect={(id) => setEmbedScraperId(id)} />;
+  } else if (!embed) {
+    content = (
+      <MediaSelector
+        embedType={embedScraper.for}
+        onSelect={(v) => setEmbed(v)}
+      />
+    );
+  } else {
+    content = <MediaScraper scraper={embedScraper} embed={embed} />;
+  }
+
+  return (
+    <div className="py-48">
+      <Navigation />
+      <div className="mx-8 overflow-x-auto">{content}</div>
+    </div>
+  );
+}
diff --git a/src/views/developer/ProviderTesterView.tsx b/src/views/developer/ProviderTesterView.tsx
new file mode 100644
index 00000000..8cc4ed0a
--- /dev/null
+++ b/src/views/developer/ProviderTesterView.tsx
@@ -0,0 +1,159 @@
+import { MWProviderScrapeResult } from "@/backend/helpers/provider";
+import { getProviders } from "@/backend/helpers/register";
+import { runProvider } from "@/backend/helpers/run";
+import { DetailedMeta } from "@/backend/metadata/getmeta";
+import { MWMediaType } from "@/backend/metadata/types";
+import { Navigation } from "@/components/layout/Navigation";
+import { ArrowLink } from "@/components/text/ArrowLink";
+import { Title } from "@/components/text/Title";
+import { useLoading } from "@/hooks/useLoading";
+import { ReactNode, useEffect, useState } from "react";
+
+interface MediaSelectorProps {
+  onSelect: (meta: DetailedMeta) => void;
+}
+
+interface ProviderSelectorProps {
+  onSelect: (providerId: string) => void;
+}
+
+interface MediaScraperProps {
+  media: DetailedMeta | null;
+  id: string;
+}
+
+function MediaSelector(props: MediaSelectorProps) {
+  const options: DetailedMeta[] = [
+    {
+      imdbId: "tt10954562",
+      tmdbId: "572716",
+      meta: {
+        id: "439596",
+        title: "Hamilton",
+        type: MWMediaType.MOVIE,
+        year: "2020",
+        seasons: undefined,
+      },
+    },
+    {
+      imdbId: "tt11126994",
+      tmdbId: "94605",
+      meta: {
+        id: "222333",
+        title: "Arcane",
+        type: MWMediaType.SERIES,
+        year: "2021",
+        seasons: [
+          {
+            id: "230301",
+            number: 1,
+            title: "Season 1",
+          },
+        ],
+        seasonData: {
+          id: "230301",
+          number: 1,
+          title: "Season 1",
+          episodes: [
+            {
+              id: "4243445",
+              number: 1,
+              title: "Welcome to the Playground",
+            },
+          ],
+        },
+      },
+    },
+  ];
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Choose media</Title>
+      {options.map((v) => (
+        <ArrowLink
+          key={v.imdbId}
+          onClick={() => props.onSelect(v)}
+          direction="right"
+          linkText={`${v.meta.title} (${v.meta.type})`}
+        />
+      ))}
+    </div>
+  );
+}
+
+function MediaScraper(props: MediaScraperProps) {
+  const [results, setResults] = useState<MWProviderScrapeResult | null>(null);
+  const [percentage, setPercentage] = useState(0);
+
+  const [scrape, loading, error] = useLoading(async (media: DetailedMeta) => {
+    const provider = getProviders().find((v) => v.id === props.id);
+    if (!provider) throw new Error("provider not found");
+    const data = await runProvider(provider, {
+      progress(num) {
+        console.log(`SCRAPING AT ${num}%`);
+        setPercentage(num);
+      },
+      media,
+      type: media.meta.type as any,
+    });
+    console.log("got data", data);
+    setResults(data);
+  });
+
+  useEffect(() => {
+    if (props.media) {
+      scrape(props.media);
+    }
+  }, [props.media, scrape]);
+
+  if (loading) return <p>Scraping... ({percentage}%)</p>;
+  if (error) return <p>Errored, check console</p>;
+
+  return (
+    <div>
+      <Title className="mb-8">Output data</Title>
+      <code>
+        <pre>{JSON.stringify(results, null, 2)}</pre>
+      </code>
+    </div>
+  );
+}
+
+function ProviderSelector(props: ProviderSelectorProps) {
+  const providers = getProviders();
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Choose provider</Title>
+      {providers.map((v) => (
+        <ArrowLink
+          key={v.id}
+          onClick={() => props.onSelect(v.id)}
+          direction="right"
+          linkText={v.displayName}
+        />
+      ))}
+    </div>
+  );
+}
+
+export function ProviderTesterView() {
+  const [media, setMedia] = useState<DetailedMeta | null>(null);
+  const [providerId, setProviderId] = useState<string | null>(null);
+
+  let content: ReactNode = null;
+  if (!providerId) {
+    content = <ProviderSelector onSelect={(id) => setProviderId(id)} />;
+  } else if (!media) {
+    content = <MediaSelector onSelect={(v) => setMedia(v)} />;
+  } else {
+    content = <MediaScraper id={providerId} media={media} />;
+  }
+
+  return (
+    <div className="py-48">
+      <Navigation />
+      <div className="mx-8 overflow-x-auto">{content}</div>
+    </div>
+  );
+}
diff --git a/src/views/developer/VideoTesterView.tsx b/src/views/developer/VideoTesterView.tsx
index 6bf34719..7681af6e 100644
--- a/src/views/developer/VideoTesterView.tsx
+++ b/src/views/developer/VideoTesterView.tsx
@@ -2,6 +2,7 @@ import { MWStreamQuality, MWStreamType } from "@/backend/helpers/streams";
 import { DetailedMeta } from "@/backend/metadata/getmeta";
 import { MWMediaType } from "@/backend/metadata/types";
 import { Button } from "@/components/Button";
+import { Dropdown } from "@/components/Dropdown";
 import { Navigation } from "@/components/layout/Navigation";
 import { ThinContainer } from "@/components/layout/ThinContainer";
 import { MetaController } from "@/video/components/controllers/MetaController";
@@ -12,11 +13,13 @@ import { Helmet } from "react-helmet";
 
 interface VideoData {
   streamUrl: string;
+  type: MWStreamType;
 }
 
 const testData: VideoData = {
   streamUrl:
     "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4",
+  type: MWStreamType.MP4,
 };
 const testMeta: DetailedMeta = {
   imdbId: "",
@@ -32,13 +35,18 @@ const testMeta: DetailedMeta = {
 
 export function VideoTesterView() {
   const [video, setVideo] = useState<VideoData | null>(null);
+  const [videoType, setVideoType] = useState<MWStreamType>(MWStreamType.MP4);
   const [url, setUrl] = useState("");
 
-  const playVideo = useCallback((streamUrl: string) => {
-    setVideo({
-      streamUrl,
-    });
-  }, []);
+  const playVideo = useCallback(
+    (streamUrl: string) => {
+      setVideo({
+        streamUrl,
+        type: videoType,
+      });
+    },
+    [videoType]
+  );
 
   if (video) {
     return (
@@ -65,18 +73,36 @@ export function VideoTesterView() {
   }
 
   return (
-    <div className="py-48">
+    <div className="py-64">
       <Navigation />
       <ThinContainer classNames="flex items-start flex-col space-y-4">
+        <div className="w-48">
+          <Dropdown
+            options={[
+              { id: MWStreamType.MP4, name: "Mp4" },
+              { id: MWStreamType.HLS, name: "hls/m3u8" },
+            ]}
+            selectedItem={{ id: videoType, name: videoType }}
+            setSelectedItem={(a) => setVideoType(a.id as MWStreamType)}
+          />
+        </div>
         <div className="mb-4 flex gap-4">
           <input
             type="text"
+            placeholder="stream url here..."
             value={url}
             onChange={(e) => setUrl(e.target.value)}
           />
           <Button onClick={() => playVideo(url)}>Play video</Button>
         </div>
-        <Button onClick={() => playVideo(testData.streamUrl)}>
+        <Button
+          onClick={() =>
+            setVideo({
+              streamUrl: testData.streamUrl,
+              type: testData.type,
+            })
+          }
+        >
           Play default video
         </Button>
       </ThinContainer>