From 478b45977d672e111d0a645f4e429087d869e65e Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Sun, 30 Oct 2022 19:14:17 -0400 Subject: [PATCH 01/21] fix: remove double space in inspiration credit --- .gitignore | 1 + src/layouts/Header.tsx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a36f111..c8bb8d0 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ yarn-error.log* #just dev stuff dev/* +yarn.lock \ No newline at end of file diff --git a/src/layouts/Header.tsx b/src/layouts/Header.tsx index db1700a..8bfc735 100644 --- a/src/layouts/Header.tsx +++ b/src/layouts/Header.tsx @@ -68,7 +68,7 @@ const Header = (props: Props) => { nitter - ,  and  + , and  Date: Sun, 30 Oct 2022 19:18:12 -0400 Subject: [PATCH 02/21] feat: add "og:image" property for social media embeds --- src/pages/title/[titleId]/index.tsx | 72 ++++++++++++++++------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/src/pages/title/[titleId]/index.tsx b/src/pages/title/[titleId]/index.tsx index 40a4596..9c64027 100644 --- a/src/pages/title/[titleId]/index.tsx +++ b/src/pages/title/[titleId]/index.tsx @@ -1,33 +1,34 @@ // external -import { GetServerSideProps, GetStaticProps, GetStaticPaths } from 'next'; -import { useRouter } from 'next/router'; +import { GetServerSideProps, GetStaticProps, GetStaticPaths } from 'next' +import { useRouter } from 'next/router' // local -import Meta from '../../../components/Meta/Meta'; -import Layout from '../../../layouts/Layout'; -import title from '../../../utils/fetchers/title'; +import Meta from '../../../components/Meta/Meta' +import Layout from '../../../layouts/Layout' +import title from '../../../utils/fetchers/title' // components -import ErrorInfo from '../../../components/Error/ErrorInfo'; -import Basic from '../../../components/title/Basic'; -import Media from '../../../components/title/Media'; -import Cast from '../../../components/title/Cast'; -import DidYouKnow from '../../../components/title/DidYouKnow'; -import Info from '../../../components/title/Info'; -import Reviews from '../../../components/title/Reviews'; -import MoreLikeThis from '../../../components/title/MoreLikeThis'; +import ErrorInfo from '../../../components/Error/ErrorInfo' +import Basic from '../../../components/title/Basic' +import Media from '../../../components/title/Media' +import Cast from '../../../components/title/Cast' +import DidYouKnow from '../../../components/title/DidYouKnow' +import Info from '../../../components/title/Info' +import Reviews from '../../../components/title/Reviews' +import MoreLikeThis from '../../../components/title/MoreLikeThis' // misc -import Title from '../../../interfaces/shared/title'; -import { AppError } from '../../../interfaces/shared/error'; +import Title from '../../../interfaces/shared/title' +import { AppError } from '../../../interfaces/shared/error' // styles -import styles from '../../../styles/modules/pages/title/title.module.scss'; +import styles from '../../../styles/modules/pages/title/title.module.scss' +import Head from 'next/head' -type Props = { data: Title; error: null } | { error: AppError; data: null }; +type Props = { data: Title; error: null } | { error: AppError; data: null } // TO-DO: make a wrapper page component to display errors, if present in props const TitleInfo = ({ data, error }: Props) => { - const router = useRouter(); + const router = useRouter() if (error) - return ; + return const info = { meta: data.meta, @@ -36,7 +37,7 @@ const TitleInfo = ({ data, error }: Props) => { boxOffice: data.boxOffice, technicalSpecs: data.technicalSpecs, accolades: data.accolades, - }; + } return ( <> @@ -46,7 +47,12 @@ const TitleInfo = ({ data, error }: Props) => { })`} description={data.basic.plot || undefined} /> - + + + @@ -59,27 +65,27 @@ const TitleInfo = ({ data, error }: Props) => { - ); -}; + ) +} // TO-DO: make a getServerSideProps wrapper for handling errors -export const getServerSideProps: GetServerSideProps = async ctx => { - const titleId = ctx.params!.titleId as string; +export const getServerSideProps: GetServerSideProps = async (ctx) => { + const titleId = ctx.params!.titleId as string try { - const data = await title(titleId); + const data = await title(titleId) - return { props: { data, error: null } }; + return { props: { data, error: null } } } catch (error: any) { - const { message, statusCode } = error; - ctx.res.statusCode = statusCode; - ctx.res.statusMessage = message; + const { message, statusCode } = error + ctx.res.statusCode = statusCode + ctx.res.statusMessage = message - return { props: { error: { message, statusCode }, data: null } }; + return { props: { error: { message, statusCode }, data: null } } } -}; +} -export default TitleInfo; +export default TitleInfo // could've used getStaticProps instead of getServerSideProps, but meh. /* From 0c76f485f98c4b9a78d396d1fb21afae8bb7a4d0 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Sun, 30 Oct 2022 19:18:20 -0400 Subject: [PATCH 03/21] chore: add prettierrc file for future contributors --- .prettierrc | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..fbe0e55 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,6 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": false, + "singleQuote": true +} \ No newline at end of file From 261a37576b65474ef8867baa622f28a75906f1f2 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Sun, 30 Oct 2022 19:21:19 -0400 Subject: [PATCH 04/21] fix: change to poster for og:image --- src/pages/title/[titleId]/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/title/[titleId]/index.tsx b/src/pages/title/[titleId]/index.tsx index 9c64027..1cfef82 100644 --- a/src/pages/title/[titleId]/index.tsx +++ b/src/pages/title/[titleId]/index.tsx @@ -50,7 +50,7 @@ const TitleInfo = ({ data, error }: Props) => { From 59a314b2bd632faa2ceac7e430be381b23547e89 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 17:04:26 -0400 Subject: [PATCH 05/21] feat: media proxy for anonymous loads --- package.json | 3 +- src/pages/api/media_proxy.ts | 83 ++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/pages/api/media_proxy.ts diff --git a/package.json b/package.json index 932008e..9f880f6 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "axios": "^0.27.2", "cheerio": "1.0.0-rc.12", "next": "12.2.5", + "node-fetch": "^3.2.10", "react": "18.2.0", "react-dom": "18.2.0", "sharp": "^0.31.0" @@ -33,4 +34,4 @@ "node": ">=16.5.0", "pnpm": ">=7.0.0" } -} \ No newline at end of file +} diff --git a/src/pages/api/media_proxy.ts b/src/pages/api/media_proxy.ts new file mode 100644 index 0000000..98b9c93 --- /dev/null +++ b/src/pages/api/media_proxy.ts @@ -0,0 +1,83 @@ +import { NextApiRequest, NextApiResponse } from 'next' +import fetch from 'node-fetch' + +const acceptableExtensions = ['.jpg', '.png', '.gif', '.webp'] + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + // get query param + const mediaUrl = (req.query as { url: string }).url + + if (!mediaUrl) { + res.status(400) + res.send(null) + return + } + + let mediaUrlParsed: URL + + try { + mediaUrlParsed = new URL(mediaUrl) + } catch { + res.status(400) + res.send(null) + return + } + + // get media domain + const mediaDomain = mediaUrlParsed.hostname + + if (!mediaDomain.endsWith('media-amazon.com')) { + res.status(400) + res.send(null) + return + } + + if (mediaUrlParsed.protocol !== 'https:') { + res.status(400) + res.send(null) + return + } + + let validExtension = false + + for (const acceptableExtension of acceptableExtensions) { + if (mediaUrlParsed.pathname.endsWith(acceptableExtension)) { + validExtension = true + break + } + } + + if (!validExtension) { + res.status(400) + res.send(null) + return + } + + // download media + const mediaRes = await fetch(mediaUrl) + + if (!mediaRes.ok) { + res.status(mediaRes.status) + return + } + + // get media type + const mediaType = mediaRes.headers.get('content-type') + + if (!mediaType) { + res.status(500) + res.send(null) + return + } + + // set media type + res.setHeader('content-type', mediaType) + + const mediaBuffer = await mediaRes.arrayBuffer() + + // send media + res.send(Buffer.from(mediaBuffer)) +} From 2c8d138cbd7a9d040d23bbc2d209133d0e15b41b Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 17:28:18 -0400 Subject: [PATCH 06/21] feat: cache media proxy data in redis for 30 mins --- package.json | 1 + src/pages/api/media_proxy.ts | 35 ++++++++++++++++++++++------------- src/utils/redis.ts | 11 +++++++++++ 3 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 src/utils/redis.ts diff --git a/package.json b/package.json index 9f880f6..f3755ba 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "dependencies": { "axios": "^0.27.2", "cheerio": "1.0.0-rc.12", + "ioredis": "^5.2.3", "next": "12.2.5", "node-fetch": "^3.2.10", "react": "18.2.0", diff --git a/src/pages/api/media_proxy.ts b/src/pages/api/media_proxy.ts index 98b9c93..78a75c1 100644 --- a/src/pages/api/media_proxy.ts +++ b/src/pages/api/media_proxy.ts @@ -1,5 +1,7 @@ import { NextApiRequest, NextApiResponse } from 'next' import fetch from 'node-fetch' +import redis from '../../utils/redis' +import crypto from 'crypto' const acceptableExtensions = ['.jpg', '.png', '.gif', '.webp'] @@ -56,6 +58,22 @@ export default async function handler( return } + // hash mediaUrl with blake3 + const mediaUrlHash = await crypto + .createHash('sha256') + .update(mediaUrl) + .digest('base64') + + // try to find mediaUrlHash in redis + const cacheKey = `media_proxy:${mediaUrlHash}` + + const cachedMedia = await redis.get(cacheKey) + + if (cachedMedia) { + res.send(cachedMedia) + return + } + // download media const mediaRes = await fetch(mediaUrl) @@ -64,20 +82,11 @@ export default async function handler( return } - // get media type - const mediaType = mediaRes.headers.get('content-type') + const mediaBuffer = Buffer.from(await mediaRes.arrayBuffer()) - if (!mediaType) { - res.status(500) - res.send(null) - return - } - - // set media type - res.setHeader('content-type', mediaType) - - const mediaBuffer = await mediaRes.arrayBuffer() + // save in redis for 30 minutes + await redis.setex(cacheKey, 60 * 30, mediaBuffer) // send media - res.send(Buffer.from(mediaBuffer)) + res.send(mediaBuffer) } diff --git a/src/utils/redis.ts b/src/utils/redis.ts new file mode 100644 index 0000000..eb87c77 --- /dev/null +++ b/src/utils/redis.ts @@ -0,0 +1,11 @@ +import Redis from 'ioredis' + +const redisUrl = process.env.REDIS_URL + +if (!redisUrl) { + throw 'Please set the REDIS_URL environment variable.' +} + +const redis = new Redis(redisUrl) + +export default redis From b7ee6863e5536ceb48538fde9a2fc56e2f1535bb Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 17:28:27 -0400 Subject: [PATCH 07/21] feat: docker support for easy deployment --- .gitignore | 5 ++++- Dockerfile | 34 ++++++++++++++++++++++++++++++++++ docker-compose.example.yml | 23 +++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 docker-compose.example.yml diff --git a/.gitignore b/.gitignore index c8bb8d0..7888140 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,7 @@ yarn-error.log* #just dev stuff dev/* -yarn.lock \ No newline at end of file +yarn.lock + +# docker +docker-compose.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bbd5f28 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# Thanks @yordis on Github! https://github.com/vercel/next.js/discussions/16995#discussioncomment-132339 + +# Install dependencies only when needed +FROM node:lts-alpine AS deps + +WORKDIR /opt/app +COPY package.json yarn.lock ./ +RUN yarn install --frozen-lockfile + +# Rebuild the source code only when needed +# This is where because may be the case that you would try +# to build the app based on some `X_TAG` in my case (Git commit hash) +# but the code hasn't changed. +FROM node:lts-alpine AS builder + +ENV NODE_ENV=production +WORKDIR /opt/app +COPY . . +COPY --from=deps /opt/app/node_modules ./node_modules +RUN yarn build + +# Production image, copy all the files and run next +FROM node:lts-alpine AS runner + +ARG X_TAG +WORKDIR /opt/app +ENV NODE_ENV=production +COPY --from=builder /opt/app/next.config.mjs ./ +COPY --from=builder /opt/app/public ./public +COPY --from=builder /opt/app/.next ./.next +COPY --from=builder /opt/app/node_modules ./node_modules +ENV HOST=0.0.0.0 +ENV PORT=3000 +CMD ["node_modules/.bin/next", "start"] \ No newline at end of file diff --git a/docker-compose.example.yml b/docker-compose.example.yml new file mode 100644 index 0000000..ce76c57 --- /dev/null +++ b/docker-compose.example.yml @@ -0,0 +1,23 @@ +# docker-compose.yml + +version: '3' + +services: + frontend: + container_name: libremdb + build: + context: . + network: host + ports: + - "3000:3000" + env_file: .env.local + depends_on: + - redis + restart: always + redis: + container_name: libremdb_redis + image: redis + # FOR DEBUGGING ONLY + # ports: + # - "6379:6379" + restart: always \ No newline at end of file From dba2ba5aa4c04b0cb177ce058257a3a5338e7a21 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 17:37:36 -0400 Subject: [PATCH 08/21] feat: fetch images from media proxy on frontend --- src/components/title/Basic.tsx | 50 ++++++++++++++++------------- src/components/title/Media.tsx | 50 +++++++++++++++-------------- src/pages/api/media_proxy.ts | 11 ++++--- src/pages/title/[titleId]/index.tsx | 7 +++- src/utils/helpers.ts | 4 +++ 5 files changed, 70 insertions(+), 52 deletions(-) diff --git a/src/components/title/Basic.tsx b/src/components/title/Basic.tsx index acc990e..9627e6d 100644 --- a/src/components/title/Basic.tsx +++ b/src/components/title/Basic.tsx @@ -1,22 +1,27 @@ -import { Fragment } from 'react'; -import Image from 'next/future/image'; -import Link from 'next/link'; +import { Fragment } from 'react' +import Image from 'next/future/image' +import Link from 'next/link' -import { formatNumber, formatTime, modifyIMDbImg } from '../../utils/helpers'; -import { Basic } from '../../interfaces/shared/title'; -import styles from '../../styles/modules/components/title/basic.module.scss'; +import { + formatNumber, + formatTime, + getProxiedIMDbImgUrl, + modifyIMDbImg, +} from '../../utils/helpers' +import { Basic } from '../../interfaces/shared/title' +import styles from '../../styles/modules/components/title/basic.module.scss' type Props = { - className: string; - data: Basic; -}; + className: string + data: Basic +} const Basic = ({ data, className }: Props) => { - const titleType = data.type.id; + const titleType = data.type.id const releaseTime = titleType === 'tvSeries' ? `${data.releaseYear?.start}-${data.releaseYear?.end || 'present'}` - : data.releaseYear?.start; + : data.releaseYear?.start return (
{ className={styles.imageContainer} style={{ backgroundImage: - data.poster && `url(${modifyIMDbImg(data.poster.url, 300)})`, + data.poster && + `url(${getProxiedIMDbImgUrl(modifyIMDbImg(data.poster.url, 300))})`, }} > {data.poster ? ( @@ -39,11 +45,11 @@ const Basic = ({ data, className }: Props) => { alt={data.poster.caption} priority fill - sizes='300px' + sizes="300px" /> ) : ( - + )} @@ -51,7 +57,7 @@ const Basic = ({ data, className }: Props) => {

{data.title}

-
    +
      {data.status.id !== 'released' && (
    • {data.status.text}
      • )} @@ -72,7 +78,7 @@ const Basic = ({ data, className }: Props) => {

      {data.ratings.avg} - + Avg. rating

      @@ -81,7 +87,7 @@ const Basic = ({ data, className }: Props) => { {formatNumber(data.ratings.numVotes)} - + No. of votes

      @@ -93,7 +99,7 @@ const Basic = ({ data, className }: Props) => { {formatNumber(data.ranking.position)} - + {' '} @@ -130,7 +136,7 @@ const Basic = ({ data, className }: Props) => { {data.plot || '-'}

      } - {data.primaryCrew.map(crewType => ( + {data.primaryCrew.map((crewType) => (

      {`${crewType.type.category}: `} @@ -147,7 +153,7 @@ const Basic = ({ data, className }: Props) => { ))}

- ); -}; + ) +} -export default Basic; +export default Basic diff --git a/src/components/title/Media.tsx b/src/components/title/Media.tsx index 583aa17..5ef8ade 100644 --- a/src/components/title/Media.tsx +++ b/src/components/title/Media.tsx @@ -1,37 +1,39 @@ -import Image from 'next/future/image'; -import Link from 'next/link'; -import { NextRouter } from 'next/router'; -import { Media } from '../../interfaces/shared/title'; -import { modifyIMDbImg } from '../../utils/helpers'; +import Image from 'next/future/image' +import Link from 'next/link' +import { NextRouter } from 'next/router' +import { Media } from '../../interfaces/shared/title' +import { getProxiedIMDbImgUrl, modifyIMDbImg } from '../../utils/helpers' -import styles from '../../styles/modules/components/title/media.module.scss'; +import styles from '../../styles/modules/components/title/media.module.scss' type Props = { - className: string; - media: Media; - router: NextRouter; -}; + className: string + media: Media + router: NextRouter +} const Media = ({ className, media, router }: Props) => { return (
{(media.trailer || !!media.videos.total) && (
-

Videos

+

Videos

{media.trailer && (

{video.caption} ({video.runtime}s) @@ -65,16 +67,16 @@ const Media = ({ className, media, router }: Props) => { )} {!!media.images.total && (

-

Images

+

Images

- {media.images.images.map(image => ( + {media.images.images.map((image) => (
{image.caption.plainText} @@ -85,6 +87,6 @@ const Media = ({ className, media, router }: Props) => {
)}
- ); -}; -export default Media; + ) +} +export default Media diff --git a/src/pages/api/media_proxy.ts b/src/pages/api/media_proxy.ts index 78a75c1..1ec226f 100644 --- a/src/pages/api/media_proxy.ts +++ b/src/pages/api/media_proxy.ts @@ -14,7 +14,7 @@ export default async function handler( if (!mediaUrl) { res.status(400) - res.send(null) + res.end() return } @@ -24,7 +24,7 @@ export default async function handler( mediaUrlParsed = new URL(mediaUrl) } catch { res.status(400) - res.send(null) + res.end() return } @@ -33,13 +33,13 @@ export default async function handler( if (!mediaDomain.endsWith('media-amazon.com')) { res.status(400) - res.send(null) + res.end() return } if (mediaUrlParsed.protocol !== 'https:') { res.status(400) - res.send(null) + res.end() return } @@ -54,7 +54,7 @@ export default async function handler( if (!validExtension) { res.status(400) - res.send(null) + res.end() return } @@ -79,6 +79,7 @@ export default async function handler( if (!mediaRes.ok) { res.status(mediaRes.status) + res.end() return } diff --git a/src/pages/title/[titleId]/index.tsx b/src/pages/title/[titleId]/index.tsx index 1cfef82..efef1f9 100644 --- a/src/pages/title/[titleId]/index.tsx +++ b/src/pages/title/[titleId]/index.tsx @@ -20,6 +20,7 @@ import { AppError } from '../../../interfaces/shared/error' // styles import styles from '../../../styles/modules/pages/title/title.module.scss' import Head from 'next/head' +import { getProxiedIMDbImgUrl } from '../../../utils/helpers' type Props = { data: Title; error: null } | { error: AppError; data: null } @@ -50,7 +51,11 @@ const TitleInfo = ({ data, error }: Props) => { diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index 5bcea0f..a4c1acd 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -53,6 +53,10 @@ export const modifyIMDbImg = (url: string, widthInPx = 600) => { return url.replaceAll('.jpg', `UX${widthInPx}.jpg`); }; +export const getProxiedIMDbImgUrl = (url: string) => { + return `/api/media_proxy?url=${encodeURIComponent(url)}`; +} + export const AppError = class extends Error { constructor(message: string, public statusCode: number, cause?: any) { super(message, cause); From 1983f6b1fb0380642c6488a0347a7073eea20338 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 17:45:14 -0400 Subject: [PATCH 09/21] feat: proxy videos and add more descriptive error messages --- src/components/title/Media.tsx | 2 +- src/pages/api/media_proxy.ts | 40 ++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/components/title/Media.tsx b/src/components/title/Media.tsx index 5ef8ade..117ed39 100644 --- a/src/components/title/Media.tsx +++ b/src/components/title/Media.tsx @@ -37,7 +37,7 @@ const Media = ({ className, media, router }: Props) => { ))} diff --git a/src/pages/api/media_proxy.ts b/src/pages/api/media_proxy.ts index 1ec226f..6e2cb2b 100644 --- a/src/pages/api/media_proxy.ts +++ b/src/pages/api/media_proxy.ts @@ -3,7 +3,7 @@ import fetch from 'node-fetch' import redis from '../../utils/redis' import crypto from 'crypto' -const acceptableExtensions = ['.jpg', '.png', '.gif', '.webp'] +const acceptableExtensions = ['.jpg', '.png', '.gif', '.webp', '.mp4'] export default async function handler( req: NextApiRequest, @@ -14,7 +14,10 @@ export default async function handler( if (!mediaUrl) { res.status(400) - res.end() + res.json({ + success: false, + message: 'Missing query', + }) return } @@ -24,22 +27,34 @@ export default async function handler( mediaUrlParsed = new URL(mediaUrl) } catch { res.status(400) - res.end() + res.json({ + success: false, + message: 'Invalid URL', + }) return } // get media domain const mediaDomain = mediaUrlParsed.hostname - if (!mediaDomain.endsWith('media-amazon.com')) { + if ( + !mediaDomain.endsWith('media-amazon.com') && + mediaDomain !== 'imdb-video.media-imdb.com' + ) { res.status(400) - res.end() + res.json({ + success: false, + message: 'Unauthorized domain', + }) return } if (mediaUrlParsed.protocol !== 'https:') { res.status(400) - res.end() + res.json({ + success: false, + message: 'Unauthorized protocol', + }) return } @@ -54,12 +69,15 @@ export default async function handler( if (!validExtension) { res.status(400) - res.end() + res.json({ + success: false, + message: 'Unauthorized extension', + }) return } // hash mediaUrl with blake3 - const mediaUrlHash = await crypto + const mediaUrlHash = crypto .createHash('sha256') .update(mediaUrl) .digest('base64') @@ -70,6 +88,7 @@ export default async function handler( const cachedMedia = await redis.get(cacheKey) if (cachedMedia) { + res.status(302) res.send(cachedMedia) return } @@ -79,7 +98,10 @@ export default async function handler( if (!mediaRes.ok) { res.status(mediaRes.status) - res.end() + res.json({ + success: false, + message: 'Error from Amazon', + }) return } From 9bce8a2dd50736ee969da783c3b29bfb9fa215f4 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 17:46:17 -0400 Subject: [PATCH 10/21] fix: bypass response limit for media proxy endpoint --- src/pages/api/media_proxy.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pages/api/media_proxy.ts b/src/pages/api/media_proxy.ts index 6e2cb2b..4e6e2d0 100644 --- a/src/pages/api/media_proxy.ts +++ b/src/pages/api/media_proxy.ts @@ -113,3 +113,9 @@ export default async function handler( // send media res.send(mediaBuffer) } + +export const config = { + api: { + responseLimit: false, + }, +} From 720f2b6acb39fa7f6d1149f79e46c2dbc591af7a Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 18:05:19 -0400 Subject: [PATCH 11/21] feat: IP ratelimit for media proxy --- src/pages/api/media_proxy.ts | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/pages/api/media_proxy.ts b/src/pages/api/media_proxy.ts index 4e6e2d0..1cffb04 100644 --- a/src/pages/api/media_proxy.ts +++ b/src/pages/api/media_proxy.ts @@ -9,6 +9,50 @@ export default async function handler( req: NextApiRequest, res: NextApiResponse ) { + const userIp = + (req.headers['cf-connecting-ip'] as string) || + (req.headers['x-real-ip'] as string) || + req.socket.remoteAddress || + null + + if (!userIp) { + res.status(500) + res.json({ + success: false, + message: 'Unable to enforce ratelimit', + }) + return + } + + // hash ip with md5 (for speed) + const ipHash = crypto.createHash('md5').update(userIp).digest('hex') + + const key = `ip_ratelimit:${ipHash}` + + // check if ip is in redis + let ipInRedis = await redis.get(key) + + if (!ipInRedis) { + // if not, set it to 1 + await redis.setex(key, 30, '1') + ipInRedis = '1' + } + + const ipReqNumber = Number(ipInRedis) + + if (ipReqNumber > 60) { + res.status(429) + res.setHeader('x-cringe', 'stop abusing a FOSS service') + res.json({ + success: false, + message: 'Too many requests', + }) + return + } + + // increment ip in redis + await redis.set(key, String(ipReqNumber + 1)) + // get query param const mediaUrl = (req.query as { url: string }).url @@ -89,10 +133,13 @@ export default async function handler( if (cachedMedia) { res.status(302) + res.setHeader('x-cached', 'true') res.send(cachedMedia) return } + res.setHeader('x-cached', 'false') + // download media const mediaRes = await fetch(mediaUrl) From 44d3a33fb3366adafd8a629a4b11211bf7479dc8 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 18:10:34 -0400 Subject: [PATCH 12/21] feat: update information in FAQ --- src/pages/about/index.tsx | 85 ++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/src/pages/about/index.tsx b/src/pages/about/index.tsx index b0f6418..5960897 100644 --- a/src/pages/about/index.tsx +++ b/src/pages/about/index.tsx @@ -1,19 +1,19 @@ /* eslint-disable react/no-unescaped-entities */ -import Link from 'next/link'; -import Meta from '../../components/Meta/Meta'; -import Layout from '../../layouts/Layout'; +import Link from 'next/link' +import Meta from '../../components/Meta/Meta' +import Layout from '../../layouts/Layout' -import styles from '../../styles/modules/pages/about/about.module.scss'; +import styles from '../../styles/modules/pages/about/about.module.scss' const About = () => { return ( <> -
+

@@ -22,12 +22,12 @@ const About = () => {

  • {

  • {

  • {

-
+

Questions you may have

@@ -91,21 +91,21 @@ const About = () => {

Replace `imdb.com` in any IMDb URL with any of the instances. For example: ` - + imdb.com/title/tt1049413 ` to ` libremdb.iket.me/title/tt1049413 ` . To avoid changing the URLs manually, you can use extensions like{' '} LibRedirect @@ -133,12 +133,21 @@ const About = () => {

- I see connection being made to some Amazon domains. + Is content served from third-parties, like Amazon?

- For now, images and videos are directly served from Amazon. If I - have enough time in the future, I'll implement a way to serve - the images from libremdb instead. + Nope, libremdb proxies all image and video requests through the + instance to avoid exposing your IP address, browser information + and other personally identifiable metadata ( + + Contributor + + ).

@@ -146,9 +155,9 @@ const About = () => { Will Amazon track me then?

- They may log your IP address, useragent, and other such - identifiers. I'd recommend using a VPN, or accessing the website - through TOR for mitigating this risk. + Also nope. All Amazon will see is the libremdb instance making + the request, not you. IP address, browser information and other + personally identifiable metadata is hidden from Amazon.

@@ -157,7 +166,7 @@ const About = () => {

Refer to the{' '} - + features section {' '} above. @@ -187,8 +196,8 @@ const About = () => {

That's great! I've a couple of{' '} - - contact methods + + contact methods . Send your beautiful suggestions(or complaints), or just drop a hi. @@ -198,7 +207,7 @@ const About = () => {

- ); -}; + ) +} -export default About; +export default About From cdd73c612345510c9a4fd0e43711f0e5d0268586 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 18:14:28 -0400 Subject: [PATCH 13/21] chore: update env local example with redis_url --- .env.local.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env.local.example b/.env.local.example index de50d8e..e8b9e2d 100644 --- a/.env.local.example +++ b/.env.local.example @@ -7,3 +7,6 @@ NEXT_PUBLIC_URL= # AXIOS_USERAGENT= # default accept header is 'application/json, text/plain, */*' # AXIOS_ACCEPT= + +# for docker, just set the domain to the container name, default is 'libremdb_redis' +REDIS_URL=localhost:6379 From cc7968074ba63240c0ce90c2a1a40cdebdf478d7 Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 18:14:52 -0400 Subject: [PATCH 14/21] docs: update readme with redis and new local docker instructions, privacy updates and more --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5f38cd2..ae6b633 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,8 @@ Inspired by projects like [teddit](https://codeberg.org/teddit/teddit), [nitter] A key named 'theme' is stored in Local Storage provided by your browser, if you ever override the default theme. To remove it, go to site data settings, and clear the data for this website. To permamently disable libremdb from storing your theme prefrences, either turn off JavaScript or disable access to Local Storage for libremdb. - Information collected by other services: - libremdb connects to 'media-amazon.com' and 'media-imdb.com' for fetching images and videos. So, Amazon might log your IP address, and other information(such as http headers) sent by your browser. + ~~libremdb connects to 'media-amazon.com' and 'media-imdb.com' for fetching images and videos. So, Amazon might log your IP address, and other information(such as http headers) sent by your browser.~~ + None. libremdb proxies images anonymously through the instance for maximum privacy ([Contributor](https://github.com/httpjamesm)). --- @@ -114,7 +115,7 @@ Inspired by projects like [teddit](https://codeberg.org/teddit/teddit), [nitter] - [ ] use redis, or any other caching strategy - [x] implement a better installation method -- [ ] serve images and videos from libremdb itself +- [x] serve images and videos from libremdb itself --- @@ -128,7 +129,10 @@ As libremdb is made with Next.js, you can deploy it anywhere where Next.js is su for Node.js, visit [their website](https://nodejs.org/en/). for Git, run `sudo apt install git` if you're on a Debian-based distro. Else visit [their website](https://git-scm.com/). -2. Clone and set up the repo. +2. Install redis + You can install redis from [here](https://redis.io). + +3. Clone and set up the repo. ```bash git clone https://github.com/zyachel/libremdb.git # replace github.com with codeberg.org if you wish so. @@ -144,11 +148,15 @@ As libremdb is made with Next.js, you can deploy it anywhere where Next.js is su libremdb will start running at http://localhost:3000. To change port, modify the last command like this: `pnpm start -- -p `. -### Docker +### Docker (Local & Recommended) -There's a [docker image](https://github.com/PussTheCat-org/docker-libremdb-quay) made by [@TheFrenchGhosty](https://github.com/TheFrenchGhosty) for [PussTheCat.org's instance](https://libremdb.pussthecat.org). You can use that in case you wish to use docker. +You can build the docker image using the provided Dockerfile and set it up using the example docker-compose file. ---- +Change the docker-compose file to your liking and run `docker-compose up -d` to start the container, that's all! + +### Docker (Built) + +## There's a [docker image](https://github.com/PussTheCat-org/docker-libremdb-quay) made by [@TheFrenchGhosty](https://github.com/TheFrenchGhosty) for [PussTheCat.org's instance](https://libremdb.pussthecat.org). You can use that in case you wish to use docker. ## Miscellaneous From eac51d2465f4464a23b5bc9712629332c28866bc Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 18:21:49 -0400 Subject: [PATCH 15/21] docs: correct two FAQ questions --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ae6b633..6101aec 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,11 @@ Inspired by projects like [teddit](https://codeberg.org/teddit/teddit), [nitter] - It doesn't have all routes. I'll implement more with time :) -- I see connection being made to some Amazon domains. - For now, images and videos are directly served from Amazon. If I have enough time in the future, I'll implement a way to serve the images from libremdb instead. +- Is content served from third-parties, like Amazon? + Nope, libremdb proxies all image and video requests through the instance to avoid exposing your IP address, browser information and other personally identifiable metadata ([Contributor](https://github.com/httpjamesm)). - Will Amazon track me then? - They may log your IP address, useragent, and other such - identifiers. I'd recommend using a VPN, or accessing the website through TOR for mitigating this risk. + Also nope. All Amazon will see is the libremdb instance making the request, not you. IP address, browser information and other personally identifiable metadata is hidden from Amazon. - Why not just use IMDb? Refer to the [features section](#some-features) above. @@ -88,6 +87,7 @@ Inspired by projects like [teddit](https://codeberg.org/teddit/teddit), [nitter] - Information collected by other services: ~~libremdb connects to 'media-amazon.com' and 'media-imdb.com' for fetching images and videos. So, Amazon might log your IP address, and other information(such as http headers) sent by your browser.~~ + None. libremdb proxies images anonymously through the instance for maximum privacy ([Contributor](https://github.com/httpjamesm)). --- From 6ae71d7907f3634773d973c7840b4bfb6aa7ea4d Mon Sep 17 00:00:00 2001 From: httpjamesm Date: Mon, 31 Oct 2022 18:23:05 -0400 Subject: [PATCH 16/21] fix: remove "information collected by other services" in privacy --- src/pages/privacy/index.tsx | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/src/pages/privacy/index.tsx b/src/pages/privacy/index.tsx index f78b9c8..4f40577 100644 --- a/src/pages/privacy/index.tsx +++ b/src/pages/privacy/index.tsx @@ -1,14 +1,14 @@ -import Meta from '../../components/Meta/Meta'; -import Layout from '../../layouts/Layout'; +import Meta from '../../components/Meta/Meta' +import Layout from '../../layouts/Layout' -import styles from '../../styles/modules/pages/privacy/privacy.module.scss'; +import styles from '../../styles/modules/pages/privacy/privacy.module.scss' const Privacy = () => { return ( <>
@@ -41,24 +41,11 @@ const Privacy = () => { Local Storage for libremdb.

-
-

- Information collected by other services -

-

- libremdb connects to 'media-amazon.com' and 'media-imdb.com' for - fetching images and videos. So, Amazon might log your IP - address, and other information(such as http headers) sent by - your browser. -

-