From 088df551b050426a636e7232ffd5915af00bce4e Mon Sep 17 00:00:00 2001
From: dragongoose <19649813+dragongoose@users.noreply.github.com>
Date: Tue, 7 Mar 2023 23:31:09 -0500
Subject: [PATCH] Clean code

---
 frontend/src/views/UserView.vue       |   2 +-
 server/routes/profile/profileRoute.ts | 211 +-------------------------
 server/types/scraping/Streamer.ts     |  24 +++
 server/util/scraping/extractors.ts    | 203 +++++++++++++++++++++++++
 4 files changed, 232 insertions(+), 208 deletions(-)
 create mode 100644 server/types/scraping/Streamer.ts
 create mode 100644 server/util/scraping/extractors.ts

diff --git a/frontend/src/views/UserView.vue b/frontend/src/views/UserView.vue
index 7582349..8c4439e 100644
--- a/frontend/src/views/UserView.vue
+++ b/frontend/src/views/UserView.vue
@@ -1,7 +1,7 @@
 <script lang="ts" >
 import { ref, onMounted } from 'vue'
 import { useRoute } from "vue-router";
-import type { StreamerData } from '../../../server/routes/profile/profileRoute'
+import type { StreamerData } from '../../../server/types/scraping/Streamer'
 
 import { VideoPlayer } from '@videojs-player/vue'
 import 'video.js/dist/video-js.css'
diff --git a/server/routes/profile/profileRoute.ts b/server/routes/profile/profileRoute.ts
index 91c6ffd..a7edcfd 100644
--- a/server/routes/profile/profileRoute.ts
+++ b/server/routes/profile/profileRoute.ts
@@ -1,219 +1,16 @@
 import { Router } from 'express'
-import puppeteer from 'puppeteer-extra'
-import { Browser, Page } from 'puppeteer'
-import { PuppeteerExtraPluginAdblocker } from 'puppeteer-extra-plugin-adblocker'
-import { LooseObject } from '../../types/looseTypes'
-import { Streamlink } from '@dragongoose/streamlink'
-
-puppeteer.use(new PuppeteerExtraPluginAdblocker({ 
-    blockTrackersAndAnnoyances: true 
-}))
+import { TwitchScraper } from '../../util/scraping/extractors'
 
 const profileRouter = Router()
-
-export interface Socials {
-    type: string | null
-    text: string,
-    link: string
-}
-
-export interface StreamData {
-    tags: string[]
-    title: string
-    topic: string
-    startedAt: number
-    qualities: string[]
-}
-
-export interface StreamerData {
-    username: string,
-    followers: number,
-    followersAbbv: string,
-    isLive: boolean,
-    about: string,
-    socials?: string[],
-    pfp: string;
-    stream?: StreamData
-}
-
-
-const abbreviatedNumberToNumber = (num: string) => {
-    const base = parseFloat(num)
-    
-    const matches: {[k: string]: number} = {
-        'k': 1000,
-        'm': 1000000,
-        'b': 1000000000
-    }
-
-    const abbreviation: string = num.charAt(num.length - 1).toLowerCase()
-
-
-    if(matches[abbreviation]) {
-        const numberOnly: number = Number(num.slice(0, -1))
-        return numberOnly * matches[abbreviation]
-    } else {
-        return null
-    }
-}
-
-// https:// advancedweb.hu/how-to-speed-up-puppeteer-scraping-with-parallelization/
-const withBrowser = async (fn: Function) => {
-	const browser = await puppeteer.launch({
-        headless: true,
-        args: ['--no-sandbox']
-    });
-	try {
-		return await fn(browser);
-	} finally {
-		await browser.close();
-	}
-}
-
-const withPage = (browser: Browser) => async (fn: Function) => {
-	const page = await browser.newPage();
-    //await page.tracing.start({ path: '../profile.json', screenshots: true });
-    try {
-        return await fn(page);
-    } finally {
-        //await page.tracing.stop();
-        await page.close();
-    }
-}
-
-let isLive: boolean
-
-const getStreamData = async (page: Page) => {
-    const streamData: LooseObject = {}
-
-    if(!isLive) return null
-
-    // Get stream tags
-    const tagsSelector = '.eUxEWt * span'
-    const tags: string[] = await page.$$eval(tagsSelector, elements => elements.map(el => el.innerHTML))
-    streamData.tags = tags
-
-    // Get stream title
-    const titleSelector = 'h2.CoreText-sc-1txzju1-0'
-    const title: string = await page.$eval(titleSelector, element => element.innerText)
-    streamData.title = title
-
-    // Get topic
-    const topicSelector = '.hfMGmo'
-    const topic = await page.$eval(topicSelector, element => element.textContent)
-    streamData.topic = topic
-
-    // Get Start time
-    const liveTimeSelector = '.live-time'
-
-    // formated as HH:MM:SS
-    const liveTime = await page.$eval(liveTimeSelector, element => element.textContent)
-    if(!liveTime) return
-    const liveTimeSplit: number[] = liveTime.split(':').map(Number)
-    let date = new Date()
-    let { hours, minutes, seconds } = { hours: date.getHours(), minutes: date.getMinutes(), seconds: date.getSeconds()}
-    
-    // Subtracts current live time from current
-    // date to get the time the stream started
-    date.setHours(hours - liveTimeSplit[0])
-    date.setMinutes(minutes - liveTimeSplit[1])
-    date.setSeconds(seconds - liveTimeSplit[2])
-
-    streamData.startedAt = date.getTime()
-
-    return streamData as StreamData
-}
-
-const getAboutData = async (page: Page) => {
-        const aboutData: LooseObject = {}
-
-        if (!isLive) {
-            // Get data from about page
-            const aboutPageButtonSelector = 'li.InjectLayout-sc-1i43xsx-0:nth-child(2) > a:nth-child(1) > div:nth-child(1) > div:nth-child(1) > p:nth-child(1)'
-            await page.click(aboutPageButtonSelector)
-        }
-        await page.waitForSelector('.ccXeNc')
-    
-        const followersSelector = '.kuAEke'
-        const followers = await page.$eval(followersSelector, element => element.innerHTML)
-        aboutData.followersAbbv = followers
-        aboutData.followers = abbreviatedNumberToNumber(followers)
-    
-        const aboutSectionSelector = '.kLFSJC'
-        const aboutSection = await page.$eval(aboutSectionSelector, element => element.innerHTML)
-        aboutData.about = aboutSection
-    
-        const socialSelector = '.ccXeNc * a'
-        const socials: Socials[] = await page.$$eval(socialSelector, elements => elements.map((el) => {
-            
-            const getHostName = (url: string) => {
-                const match = url.match(/:\/\/(www[0-9]?\.)?(.[^/:]+)/i);
-                if (match != null && match.length > 2 && typeof match[2] === 'string' && match[2].length > 0) {
-                    const hostname = match[2].split(".");
-                    return hostname[0];
-                }
-                else {
-                    return null;
-                }
-            }
-
-            const validHosts = ['instagram', 'youtube', 'discord', 'tiktok','twitter']
-            const socialHost = getHostName(el.href) || el.href || ''
-            let type: string | null = socialHost
-            if(!validHosts.includes(socialHost)) 
-                type = null
-
-            return {
-                type,
-                link: el.href,
-                text: el.innerText
-            }
-        }))
-        aboutData.socials = socials
-    
-        const profilePictureSelector = 'figure.ScAvatar-sc-144b42z-0:nth-child(2) > img:nth-child(1)'
-        const profilePicutre = await page.$eval(profilePictureSelector, element => element.getAttribute('src'))
-        aboutData.pfp = profilePicutre
-
-        return aboutData as StreamerData
-}
-
-const getStreamerData = async (username: string): Promise<StreamerData> => {
-    let recoveredData: LooseObject = {}
-
-    await withBrowser(async (browser: Browser) => {
-        const result = await withPage(browser)(async (page: Page) => {
-            await page.goto(`https://twitch.tv/${username}`)    
-
-            return Promise.all([getStreamData(page), getAboutData(page)])
-        })
-        
-        recoveredData = result[1]
-        recoveredData.stream = result[0]
-        if(result[0] !== null) recoveredData.isLive = true
-
-        await browser.close()
-    })
-
-    recoveredData.username = username
-    return recoveredData as StreamerData
-}
+const scraper = new TwitchScraper()
 
 profileRouter.get('/users/:username', async (req, res, next) => {
     const username = req.params.username
-    const streamlink = new Streamlink(`https://twitch.tv/${username}`, {})
-    isLive = await streamlink.isLive()
 
-    let streamerData = await getStreamerData(username)
+    let streamerData = await scraper.getStreamerData(username)
         .catch(next)
 
-    if(streamerData && streamerData.stream && isLive)
-        streamerData.stream.qualities = await streamlink.getQualities()
-    
-    if(streamerData) {
-        streamerData.isLive = isLive
-        res.send(streamerData)
-    }
+    res.send(streamerData)
 })
 
 export default profileRouter
\ No newline at end of file
diff --git a/server/types/scraping/Streamer.ts b/server/types/scraping/Streamer.ts
new file mode 100644
index 0000000..35148ed
--- /dev/null
+++ b/server/types/scraping/Streamer.ts
@@ -0,0 +1,24 @@
+export interface Socials {
+    type: string | null
+    text: string,
+    link: string
+}
+
+export interface StreamData {
+    tags: string[]
+    title: string
+    topic: string
+    startedAt: number
+    qualities: string[]
+}
+
+export interface StreamerData {
+    username: string,
+    followers: number,
+    followersAbbv: string,
+    isLive: boolean,
+    about: string,
+    socials?: string[],
+    pfp: string;
+    stream?: StreamData
+}
\ No newline at end of file
diff --git a/server/util/scraping/extractors.ts b/server/util/scraping/extractors.ts
new file mode 100644
index 0000000..db49052
--- /dev/null
+++ b/server/util/scraping/extractors.ts
@@ -0,0 +1,203 @@
+import puppeteer from 'puppeteer-extra'
+import { Browser, Page } from 'puppeteer'
+import { PuppeteerExtraPluginAdblocker } from 'puppeteer-extra-plugin-adblocker'
+import { LooseObject } from '../../types/looseTypes'
+import { StreamData, StreamerData, Socials } from '../../types/scraping/Streamer'
+import { Streamlink } from '@dragongoose/streamlink'
+
+
+export class TwitchScraper {
+    public cache: Map<string, StreamerData> = new Map()
+
+
+    constructor() {
+        puppeteer.use(new PuppeteerExtraPluginAdblocker({ 
+            blockTrackersAndAnnoyances: true 
+        }))
+    }
+
+    private abbreviatedNumberToNumber = (num: string) => {
+        const base = parseFloat(num)
+        
+        const matches: {[k: string]: number} = {
+            'k': 1000,
+            'm': 1000000,
+            'b': 1000000000
+        }
+    
+        const abbreviation: string = num.charAt(num.length - 1).toLowerCase()
+    
+    
+        if(matches[abbreviation]) {
+            const numberOnly: number = Number(num.slice(0, -1))
+            return numberOnly * matches[abbreviation]
+        } else {
+            return null
+        }
+    }
+
+    // https:// advancedweb.hu/how-to-speed-up-puppeteer-scraping-with-parallelization/
+    private withBrowser = async (fn: Function) => {
+        const browser = await puppeteer.launch({
+            headless: true,
+            args: ['--no-sandbox']
+        });
+        try {
+            return await fn(browser);
+        } finally {
+            await browser.close();
+        }
+    }
+
+    private withPage = (browser: Browser) => async (fn: Function) => {
+        const page = await browser.newPage();
+        //await page.tracing.start({ path: '../profile.json', screenshots: true });
+        try {
+            return await fn(page);
+        } finally {
+            //await page.tracing.stop();
+            await page.close();
+        }
+    }
+
+    private getStreamData = async (page: Page, isLive: boolean) => {
+        const streamData: LooseObject = {}
+
+        if(!isLive) return null
+
+        // Get stream tags
+        const tagsSelector = '.eUxEWt * span'
+        const tags: string[] = await page.$$eval(tagsSelector, elements => elements.map(el => el.innerHTML))
+        streamData.tags = tags
+
+        // Get stream title
+        const titleSelector = 'h2.CoreText-sc-1txzju1-0'
+        const title: string = await page.$eval(titleSelector, element => element.innerText)
+        streamData.title = title
+
+        // Get topic
+        const topicSelector = '.hfMGmo'
+        const topic = await page.$eval(topicSelector, element => element.textContent)
+        streamData.topic = topic
+
+        // Get Start time
+        const liveTimeSelector = '.live-time'
+
+        // formated as HH:MM:SS
+        const liveTime = await page.$eval(liveTimeSelector, element => element.textContent)
+        if(!liveTime) return
+        const liveTimeSplit: number[] = liveTime.split(':').map(Number)
+        let date = new Date()
+        let { hours, minutes, seconds } = { hours: date.getHours(), minutes: date.getMinutes(), seconds: date.getSeconds()}
+        
+        // Subtracts current live time from current
+        // date to get the time the stream started
+        date.setHours(hours - liveTimeSplit[0])
+        date.setMinutes(minutes - liveTimeSplit[1])
+        date.setSeconds(seconds - liveTimeSplit[2])
+
+        streamData.startedAt = date.getTime()
+
+        return streamData as StreamData
+    }
+
+    private getAboutData = async (page: Page, isLive: boolean) => {
+            const aboutData: LooseObject = {}
+
+            if (!isLive) {
+                // Get data from about page
+                const aboutPageButtonSelector = 'li.InjectLayout-sc-1i43xsx-0:nth-child(2) > a:nth-child(1) > div:nth-child(1) > div:nth-child(1) > p:nth-child(1)'
+                await page.click(aboutPageButtonSelector)
+            }
+            await page.waitForSelector('.kuAEke')
+        
+            const followersSelector = '.kuAEke'
+            const followers = await page.$eval(followersSelector, element => element.innerHTML)
+            aboutData.followersAbbv = followers
+            aboutData.followers = this.abbreviatedNumberToNumber(followers)
+        
+            const aboutSectionSelector = '.kLFSJC'
+            const aboutSection = await page.$eval(aboutSectionSelector, element => element.innerHTML)
+            aboutData.about = aboutSection
+        
+            const socialSelector = '.ccXeNc * a'
+            const socials: Socials[] = await page.$$eval(socialSelector, elements => elements.map((el) => {
+                
+                const getHostName = (url: string) => {
+                    const match = url.match(/:\/\/(www[0-9]?\.)?(.[^/:]+)/i);
+                    if (match != null && match.length > 2 && typeof match[2] === 'string' && match[2].length > 0) {
+                        const hostname = match[2].split(".");
+                        return hostname[0];
+                    }
+                    else {
+                        return null;
+                    }
+                }
+
+                const validHosts = ['instagram', 'youtube', 'discord', 'tiktok','twitter']
+                const socialHost = getHostName(el.href) || el.href || ''
+                let type: string | null = socialHost
+                if(!validHosts.includes(socialHost)) 
+                    type = null
+
+                return {
+                    type,
+                    link: el.href,
+                    text: el.innerText
+                }
+            }))
+            aboutData.socials = socials
+        
+            const profilePictureSelector = 'figure.ScAvatar-sc-144b42z-0:nth-child(2) > img:nth-child(1)'
+            const profilePicutre = await page.$eval(profilePictureSelector, element => element.getAttribute('src'))
+            aboutData.pfp = profilePicutre
+
+            return aboutData as StreamerData
+    }
+
+    public getStreamerData = async (username: string): Promise<StreamerData> => {
+        let recoveredData: LooseObject = {}
+        let isLive = await this.isLive(username)
+
+        await this.withBrowser(async (browser: Browser) => {
+            const result = await this.withPage(browser)(async (page: Page) => {
+                const res = await page.goto(`https://twitch.tv/${username}`)  
+                
+                if(!res?.ok()) {
+                    return null 
+                } else {
+                    return Promise.all([this.getStreamData(page, isLive), this.getAboutData(page, isLive)])
+                }
+            })
+            
+            recoveredData = result[1]
+            recoveredData.stream = result[0]
+            if(result[0] !== null) recoveredData.isLive = true
+
+            await browser.close()
+        })
+
+
+        // add final information
+        if(recoveredData && recoveredData.stream && isLive)
+            recoveredData.stream.qualities = await this.getQualities(username)
+    
+        if(recoveredData) {
+            recoveredData.isLive = isLive
+        }
+
+        recoveredData.username = username
+        return recoveredData as StreamerData
+    }
+
+    public isLive = async (username: string) => {
+        const streamlink = new Streamlink(`https://twitch.tv/${username}`, {})
+        return await streamlink.isLive()
+    }
+
+    public getQualities = async (username: string) => {
+        const streamlink = new Streamlink(`https://twitch.tv/${username}`, {})
+        return await streamlink.getQualities()
+    }
+
+}