import { Router } from 'express' import puppeteer, { Browser, Page } from 'puppeteer' import { LooseObject } from '../../types/looseTypes' import { Streamlink } from '@dragongoose/streamlink' const profileRouter = Router() export interface Socials { type: string | null text: string, link: string } export interface StreamData { tags: string[] title: string topic: string startedAt: number qualities: string[] } export interface StreamerData { username: string, followers: number, followersAbbv: string, isLive: boolean, about: string, socials?: string[], pfp: string; stream?: StreamData } const abbreviatedNumberToNumber = (num: string) => { const base = parseFloat(num) const matches: {[k: string]: number} = { 'k': 1000, 'm': 1000000, 'b': 1000000000 } const abbreviation: string = num.charAt(num.length - 1).toLowerCase() if(matches[abbreviation]) { const numberOnly: number = Number(num.slice(0, -1)) return numberOnly * matches[abbreviation] } else { return null } } // https:// advancedweb.hu/how-to-speed-up-puppeteer-scraping-with-parallelization/ const withBrowser = async (fn: Function) => { const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox'] }); try { return await fn(browser); } finally { await browser.close(); } } const withPage = (browser: Browser) => async (fn: Function) => { const page = await browser.newPage(); //turns request interceptor on await page.setRequestInterception(true); //if the page makes a request to a resource type of image or stylesheet then abort that request page.on('request', request => { if (request.resourceType() === 'image') request.abort(); else request.continue(); }); try { return await fn(page); } finally { await page.close(); } } let isLive: boolean const getStreamData = async (page: Page) => { const streamData: LooseObject = {} if(!isLive) return null // Get stream tags const tagsSelector = '.eUxEWt * span' const tags: string[] = await page.$$eval(tagsSelector, elements => elements.map(el => el.innerHTML)) streamData.tags = tags // Get stream title const titleSelector = 'h2.CoreText-sc-1txzju1-0' const title: string = await page.$eval(titleSelector, element => element.innerText) streamData.title = title // Get topic const topicSelector = '.hfMGmo' const topic = await page.$eval(topicSelector, element => element.textContent) streamData.topic = topic // Get Start time const liveTimeSelector = '.live-time' // formated as HH:MM:SS const liveTime = await page.$eval(liveTimeSelector, element => element.textContent) if(!liveTime) return const liveTimeSplit: number[] = liveTime.split(':').map(Number) let date = new Date() let { hours, minutes, seconds } = { hours: date.getHours(), minutes: date.getMinutes(), seconds: date.getSeconds()} // Subtracts current live time from current // date to get the time the stream started date.setHours(hours - liveTimeSplit[0]) date.setMinutes(minutes - liveTimeSplit[1]) date.setSeconds(seconds - liveTimeSplit[2]) streamData.startedAt = date.getTime() return streamData as StreamData } const getAboutData = async (page: Page) => { const aboutData: LooseObject = {} if (!isLive) { // Get data from about page const aboutPageButtonSelector = 'li.InjectLayout-sc-1i43xsx-0:nth-child(2) > a:nth-child(1) > div:nth-child(1) > div:nth-child(1) > p:nth-child(1)' await page.click(aboutPageButtonSelector) } await page.waitForSelector('.ccXeNc') const followersSelector = '.kuAEke' const followers = await page.$eval(followersSelector, element => element.innerHTML) aboutData.followersAbbv = followers aboutData.followers = abbreviatedNumberToNumber(followers) const aboutSectionSelector = '.kLFSJC' const aboutSection = await page.$eval(aboutSectionSelector, element => element.innerHTML) aboutData.about = aboutSection const socialSelector = '.ccXeNc * a' const socials: Socials[] = await page.$$eval(socialSelector, elements => elements.map((el) => { const getHostName = (url: string) => { const match = url.match(/:\/\/(www[0-9]?\.)?(.[^/:]+)/i); if (match != null && match.length > 2 && typeof match[2] === 'string' && match[2].length > 0) { const hostname = match[2].split("."); return hostname[0]; } else { return null; } } const validHosts = ['instagram', 'youtube', 'discord', 'tiktok','twitter'] const socialHost = getHostName(el.href) || el.href || '' let type: string | null = socialHost if(!validHosts.includes(socialHost)) type = null return { type, link: el.href, text: el.innerText } })) aboutData.socials = socials const profilePictureSelector = 'figure.ScAvatar-sc-144b42z-0:nth-child(2) > img:nth-child(1)' const profilePicutre = await page.$eval(profilePictureSelector, element => element.getAttribute('src')) aboutData.pfp = profilePicutre return aboutData as StreamerData } const getStreamerData = async (username: string) => { let recoveredData: LooseObject = {} await withBrowser(async (browser: Browser) => { const result = await withPage(browser)(async (page: Page) => { await page.goto(`https://twitch.tv/${username}`) return Promise.all([getStreamData(page), getAboutData(page)]) }) recoveredData = result[1] recoveredData.stream = result[0] if(result[0] !== null) recoveredData.isLive = true await browser.close() }) recoveredData.username = username return recoveredData as StreamerData } profileRouter.get('/users/:username', async (req, res) => { const username = req.params.username const streamlink = new Streamlink(`https://twitch.tv/${username}`, {}) isLive = await streamlink.isLive() const qualities = await streamlink.getQualities() let streamerData = await getStreamerData(username) if(streamerData.stream) streamerData.stream.qualities = qualities res.send(streamerData) }) export default profileRouter