apply patches from the old upstream and cleanup

Signed-off-by: ngn <ngn@ngn.tf>

.dockerignore

@@ -1,7 +1,6 @@
 *.png
 *.md
-accounts.*
-LICENSE.txt
-compose.yml
+LICENSE
+docker-compose.yml
 Dockerfile
 tests/

.gitea/workflows/build-get-account.yml

@@ -1,28 +0,0 @@
-name: Build the docker image for the get_account.py script
-
-on:
-  push:
-    branches: ["custom"]
-
-env:
-  REGISTRY: git.ngn.tf
-  IMAGE: ${{gitea.repository}}/get-account
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: "https://github.com/actions/checkout@v4"
-
-      - name: Login to container repo
-        uses: "https://github.com/docker/login-action@v1"
-        with:
-          registry: ${{env.REGISTRY}}
-          username: ${{gitea.actor}}
-          password: ${{secrets.PACKAGES_TOKEN}}
-
-      - name: Build and push the image
-        run: |
-          docker build . -f docker/get_account/Dockerfile --tag ${{env.REGISTRY}}/${{env.IMAGE}}:latest
-          docker push ${{env.REGISTRY}}/${{env.IMAGE}}:latest

.gitea/workflows/build-web.yml

@@ -1,28 +0,0 @@
-name: Build the docker image for the web server
-
-on:
-  push:
-    branches: ["custom"]
-
-env:
-  REGISTRY: git.ngn.tf
-  IMAGE: ${{gitea.repository}}/web
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: "https://github.com/actions/checkout@v4"
-
-      - name: Login to container repo
-        uses: "https://github.com/docker/login-action@v1"
-        with:
-          registry: ${{env.REGISTRY}}
-          username: ${{gitea.actor}}
-          password: ${{secrets.PACKAGES_TOKEN}}
-
-      - name: Build and push the image
-        run: |
-          docker build . -f docker/nitter/Dockerfile --tag ${{env.REGISTRY}}/${{env.IMAGE}}:latest
-          docker push ${{env.REGISTRY}}/${{env.IMAGE}}:latest

.gitea/workflows/docker.yml

@@ -0,0 +1,48 @@
+name: docker
+
+on:
+  push:
+    branches: ["main"]
+
+env:
+  REGISTRY: git.ngn.tf
+  IMAGE: ${{gitea.repository}}
+
+jobs:
+  nitter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Login to container repo
+        uses: docker/login-action@v3
+        with:
+          registry: ${{env.REGISTRY}}
+          username: ${{gitea.actor}}
+          password: ${{secrets.PACKAGES_TOKEN}}
+
+      - name: Build and push the image
+        run: |
+          docker build . -f docker/nitter.Dockerfile \
+            --tag ${{env.REGISTRY}}/${{env.IMAGE}}:latest
+          docker push ${{env.REGISTRY}}/${{env.IMAGE}}:latest
+
+  session:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Login to container repo
+        uses: docker/login-action@v3
+        with:
+          registry: ${{env.REGISTRY}}
+          username: ${{gitea.actor}}
+          password: ${{secrets.PACKAGES_TOKEN}}
+
+      - name: Build and push the image
+        run: |
+          docker build . -f docker/session.Dockerfile \
+            --tag ${{env.REGISTRY}}/${{env.IMAGE}}/session:latest
+          docker push ${{env.REGISTRY}}/${{env.IMAGE}}/session:latest

.gitignore

@@ -1,16 +1,17 @@
-./nitter
+nitter
 *.html
 *.db
-data
-tests/__pycache__
-tests/geckodriver.log
-tests/downloaded_files
-tests/latest_logs
-tools/gencss
-tools/rendermd
-public/css/style.css
-public/md/*.html
+/tests/__pycache__
+/tests/geckodriver.log
+/tests/downloaded_files
+/tests/latest_logs
+/tools/gencss
+/tools/rendermd
+/public/css/style.css
+/public/md/*.html
 nitter.conf
-compose.yml
-accounts.*
+guest_accounts.json*
+sessions.json*
 dump.rdb
+docker-compose.yml
+compose.yml

.travis.yml

@@ -0,0 +1,51 @@
+jobs:
+  include:
+    - stage: test
+      if: (NOT type IN (pull_request)) AND (branch = master)
+      dist: bionic
+      language: python
+      python:
+        - 3.6
+      services:
+        - docker
+        - xvfb
+      script:
+        - sudo apt update
+        - sudo apt install --force-yes chromium-chromedriver
+        - wget https://www.dropbox.com/s/ckuoaubd1crrj2k/Linux_x64_737173_chrome-linux.zip -O chrome.zip
+        - unzip chrome.zip
+        - cd chrome-linux
+        - sudo rm /usr/bin/google-chrome
+        - sudo ln -s chrome /usr/bin/google-chrome
+        - cd ..
+        - pip3 install --upgrade pip
+        - pip3 install -U seleniumbase pytest
+        - docker run -d -p 127.0.0.1:8080:8080/tcp $IMAGE_NAME:$TRAVIS_COMMIT
+        - sleep 10
+        - cd tests
+        - pytest --headless -n 8 --reruns 10 --reruns-delay 2
+    - stage: pr
+      if: type IN (pull_request)
+      dist: bionic
+      language: python
+      python:
+        - 3.6
+      services:
+        - docker
+        - xvfb
+      script:
+        - sudo apt update
+        - sudo apt install --force-yes chromium-chromedriver
+        - wget https://www.dropbox.com/s/ckuoaubd1crrj2k/Linux_x64_737173_chrome-linux.zip -O chrome.zip
+        - unzip chrome.zip
+        - cd chrome-linux
+        - sudo rm /usr/bin/google-chrome
+        - sudo ln -s chrome /usr/bin/google-chrome
+        - cd ..
+        - pip3 install --upgrade pip
+        - pip3 install -U seleniumbase pytest
+        - docker build -t $IMAGE_NAME:$TRAVIS_COMMIT .
+        - docker run -d -p 127.0.0.1:8080:8080/tcp $IMAGE_NAME:$TRAVIS_COMMIT
+        - sleep 10
+        - cd tests
+        - pytest --headless -n 8 --reruns 3 --reruns-delay 2

README.md

@@ -1,8 +1,7 @@
 # nitter - alternative Twitter frontend
 
-![](https://git.ngn.tf/ngn/nitter/actions/workflows/build-web.yml/badge.svg)
-![](https://git.ngn.tf/ngn/nitter/actions/workflows/build-get-account.yml/badge.svg)
+![](https://git.ngn.tf/ngn/nitter/actions/workflows/build.yml/badge.svg)
 ![](https://git.ngn.tf/ngn/nitter/actions/workflows/ups.yml/badge.svg)
 
-A fork of the [nitter](https://github.com/PrivacyDevel/nitter) project, with my
+A fork of the [nitter](https://github.com/zedeus/nitter) project, with my
 personal changes.

compose.example.yml

@@ -1,20 +1,20 @@
 services:
   nitter:
     container_name: nitter
-    image: git.ngn.tf/ngn/nitter/web
+    image: git.ngn.tf/ngn/nitter
     ports:
-    - 80:8080
+      - 80:8080
     volumes:
-    - ./nitter.conf:/srv/nitter.conf:Z,ro
-    - ./accounts.jsonl:/srv/accounts.jsonl:Z,ro
+      - ./nitter.conf:/srv/nitter.conf:Z,ro
+      - ./sessions.jsonl:/srv/sessions.jsonl:Z,ro
     depends_on:
-    - nitter_redis
+      - nitter_redis
     restart: unless-stopped
     user: 998:998
     security_opt:
-    - no-new-privileges:true
+      - no-new-privileges:true
     cap_drop:
-    - ALL
+      - ALL
     read_only: true
 
   nitter_redis:
@@ -22,11 +22,11 @@ services:
     image: redis:6-alpine
     command: redis-server --save 60 1 --loglevel warning
     volumes:
-    - ./data:/data
+      - ./data:/data
     restart: unless-stopped
     user: 999:1000
     security_opt:
-    - no-new-privileges:true
+      - no-new-privileges:true
     cap_drop:
-    - ALL
+      - ALL
     read_only: true

docker/get_account/Dockerfile

@@ -1,6 +0,0 @@
-FROM python
-
-RUN pip install pyotp requests
-COPY ./tools/get_account.py /get_account.py
-
-ENTRYPOINT ["python3", "/get_account.py"]

docker/nitter.Dockerfile

@@ -1,26 +1,30 @@
-FROM nimlang/nim:2.0.0-alpine-regular as build
+# builds nitter
+FROM nimlang/nim:2.2.0-alpine-regular as build
 
 RUN apk --no-cache add libsass-dev pcre
 
-WORKDIR /src
+WORKDIR /src/nitter
 
 COPY nitter.nimble .
 RUN nimble install -y --depsOnly
 
 COPY . .
-RUN nimble build -d:danger -d:lto -d:strip
-RUN nimble scss
+RUN nimble build -d:danger -d:lto -d:strip --mm:refc && \
+    nimble scss
 
+# runs nitter
 FROM alpine:latest
 
+WORKDIR /src/
+
 RUN apk --no-cache add pcre ca-certificates
 
 WORKDIR /srv
 COPY --from=build /src/nitter ./
 COPY --from=build /src/public ./public
 
-RUN adduser -h /srv -D -s /bin/sh -u 1001 runner
-RUN chown runner:runner -R /srv
+RUN adduser -h /srv -D -s /bin/sh -u 1001 runner && \
+    chown runner:runner -R /srv
 
 USER runner
 CMD ./nitter

docker/session.Dockerfile

@@ -0,0 +1,6 @@
+FROM python
+
+RUN pip install pyotp requests
+COPY ./tools/get_session.py /get_session.py
+
+ENTRYPOINT ["python3", "/get_session.py"]

nitter.example.conf

@@ -1,4 +1,4 @@
-[server]
+[Server]
 hostname = "nitter.net"  # for generating links, change this to your own domain/ip
 title = "nitter"
 address = "0.0.0.0"
@@ -6,9 +6,8 @@ port = 8080
 https = false  # disable to enable cookies when not using https
 httpMaxConnections = 100
 staticDir = "./public"
-accountsFile = "./accounts.jsonl"
 
-[cache]
+[Cache]
 listMinutes = 240  # how long to cache list info (not the tweets, so keep it high)
 rssMinutes = 10  # how long to cache rss queries
 redisHost = "localhost"  # Change to "nitter_redis" if using docker-compose
@@ -20,22 +19,16 @@ redisMaxConnections = 30
 # goes above this, they're closed when released. don't worry about this unless
 # you receive tons of requests per second
 
-[config]
+[Config]
 hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
-enableDebug = false  # enable request logs and debug endpoints (/.accounts)
+enableDebug = false  # enable request logs and debug endpoints (/.sessions)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
-tokenCount = 10
-# minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
-# the limits reset every 15 minutes, and the pool is filled up so there's
-# always at least `tokenCount` usable tokens. only increase this if you receive
-# major bursts all the time and don't have a rate limiting setup via e.g. nginx
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
-[preferences]
+[Preferences]
 theme = "Nitter"
 replaceTwitter = "nitter.net"
 replaceYouTube = "piped.video"

nitter.nimble

@@ -15,6 +15,7 @@ requires "jester#baca3f"
 requires "karax#5cf360c"
 requires "sass#7dfdd03"
 requires "nimcrypto#a079df9"
+requires "markdown#158efe3"
 requires "packedjson#9e6fbb6"
 requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"

renovate.json

@@ -1,5 +0,0 @@
-{
-  "extends": ["config:recommended"],
-  "timezone": "Europe/Istanbul",
-  "prHourlyLimit": 20
-}

src/api.nim

@@ -69,23 +69,6 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
   let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
   result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
-proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} =
-  if id.len == 0: return
-  var
-    variables = %*{
-      "userId": id,
-      "includePromotedContent":false,
-      "withClientEventToken":false,
-      "withBirdwatchNotes":false,
-      "withVoice":true,
-      "withV2Timeline":false
-    }
-  if after.len > 0:
-    variables["cursor"] = % after
-  let  
-    url = consts.favorites ? {"variables": $variables, "features": gqlFeatures}
-  result = parseGraphTimeline(await fetch(url, Api.favorites), after)
-
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
@@ -103,42 +86,6 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
     js = await fetch(graphTweet ? params, Api.tweetDetail)
   result = parseGraphConversation(js, id)
 
-proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = reactorsVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphFavoriters ? params, Api.favoriters)
-  result = parseGraphFavoritersTimeline(js, id)
-
-proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = reactorsVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphRetweeters ? params, Api.retweeters)
-  result = parseGraphRetweetersTimeline(js, id)
-
-proc getGraphFollowing*(id: string; after=""): Future[UsersTimeline] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = followVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphFollowing ? params, Api.following)
-  result = parseGraphFollowTimeline(js, id)
-
-proc getGraphFollowers*(id: string; after=""): Future[UsersTimeline] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = followVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphFollowers ? params, Api.followers)
-  result = parseGraphFollowTimeline(js, id)
-
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
   result = (await getGraphTweet(id, after)).replies
   result.beginning = after.len == 0
@@ -189,13 +136,13 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   result = parseGraphSearch[User](await fetch(url, Api.search), after)
   result.query = query
 
-proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
+proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+  if id.len == 0: return
   let
-    ps = genParams({"screen_name": name, "trim_user": "true"},
-                    count="18", ext=false)
-    url = photoRail ? ps
-  result = parsePhotoRail(await fetch(url, Api.photoRail))
+    variables = userTweetsVariables % [id, ""]
+    params = {"variables": variables, "features": gqlFeatures}
+    url = graphUserMedia ? params
+  result = parseGraphPhotoRail(await fetch(url, Api.userMedia))
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)

src/apiutils.nim

@@ -3,33 +3,14 @@ import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
 import jsony, packedjson, zippy, oauth1
 import types, auth, consts, parserutils, http_pool
 import experimental/types/common
-import config
 
 const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
+  errorsToSkip = {doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
 var pool: HttpPool
 
-proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
-                count="20"; ext=true): seq[(string, string)] =
-  result = timelineParams
-  for p in pars:
-    result &= p
-  if ext:
-    result &= ("include_ext_alt_text", "1")
-    result &= ("include_ext_media_stats", "1")
-    result &= ("include_ext_media_availability", "1")
-  if count.len > 0:
-    result &= ("count", count)
-  if cursor.len > 0:
-    # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to turn them back into a plus
-    if " " in cursor:
-      result &= ("cursor", cursor.replace(" ", "+"))
-    else:
-      result &= ("cursor", cursor)
-
 proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
   let
     encodedUrl = url.replace(",", "%2C").replace("+", "%20")
@@ -55,44 +36,31 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
     "authorization": header,
     "content-type": "application/json",
     "x-twitter-active-user": "yes",
-    "authority": "api.twitter.com",
+    "authority": "api.x.com",
     "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.9",
     "accept": "*/*",
     "DNT": "1"
   })
 
-template updateAccount() =
-  if resp.headers.hasKey(rlRemaining):
-    let
-      remaining = parseInt(resp.headers[rlRemaining])
-      reset = parseInt(resp.headers[rlReset])
-    account.setRateLimit(api, remaining, reset)
-
-template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
+template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
-  var account = await getGuestAccount(api)
-  if account.oauthToken.len == 0:
-    echo "[accounts] Empty oauth token, account: ", account.id
+  var session = await getSession(api)
+  if session.oauthToken.len == 0:
+    echo "[sessions] Empty oauth token, session: ", session.id
     raise rateLimitError()
 
   try:
     var resp: AsyncResponse
-    var headers = genHeaders($url, account.oauthToken, account.oauthSecret)
-    for key, value in additional_headers.pairs():
-      headers.add(key, value)
-    pool.use(headers):
+    pool.use(genHeaders($url, session.oauthToken, session.oauthSecret)):
       template getContent =
         resp = await c.get($url)
         result = await resp.body
 
       getContent()
 
-      if resp.status == $Http429:
-        raise rateLimitError()
-
       if resp.status == $Http503:
         badClient = true
         raise newException(BadClientError, "Bad client")
@@ -101,7 +69,7 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
       let
         remaining = parseInt(resp.headers[rlRemaining])
         reset = parseInt(resp.headers[rlReset])
-      account.setRateLimit(api, remaining, reset)
+      session.setRateLimit(api, remaining, reset)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -109,23 +77,25 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
 
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
-        if errors in {expiredToken, badToken}:
-          echo "fetch error: ", errors
-          invalidate(account)
-          raise rateLimitError()
-        elif errors in {rateLimited}:
-          # rate limit hit, resets after 24 hours
-          setLimited(account, api)
-          raise rateLimitError()
+        if errors notin errorsToSkip:
+          echo "Fetch error, API: ", api, ", errors: ", errors
+          if errors in {expiredToken, badToken, locked}:
+            invalidate(session)
+            raise rateLimitError()
+          elif errors in {rateLimited}:
+            # rate limit hit, resets after 24 hours
+            setLimited(session, api)
+            raise rateLimitError()
       elif result.startsWith("429 Too Many Requests"):
-        echo "[accounts] 429 error, API: ", api, ", account: ", account.id
-        account.apis[api].remaining = 0
+        echo "[sessions] 429 error, API: ", api, ", session: ", session.id
+        session.apis[api].remaining = 0
         # rate limit hit, resets after the 15 minute window
         raise rateLimitError()
 
     fetchBody
 
     if resp.status == $Http400:
+      echo "ERROR 400, ", api, ": ", result
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
@@ -134,24 +104,23 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
   except OSError as e:
     raise e
   except Exception as e:
-    let id = if account.isNil: "null" else: $account.id
-    echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
+    let id = if session.isNil: "null" else: $session.id
+    echo "error: ", e.name, ", msg: ", e.msg, ", sessionId: ", id, ", url: ", url
     raise rateLimitError()
   finally:
-    release(account)
+    release(session)
 
 template retry(bod) =
   try:
     bod
   except RateLimitError:
-    echo "[accounts] Rate limited, retrying ", api, " request..."
+    echo "[sessions] Rate limited, retrying ", api, " request..."
     bod
 
-proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
-
+proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
   retry:
     var body: string
-    fetchImpl(body, additional_headers):
+    fetchImpl body:
       if body.startsWith('{') or body.startsWith('['):
         result = parseJson(body)
       else:
@@ -159,14 +128,15 @@ proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders
         result = newJNull()
 
       let error = result.getError
-      if error in {expiredToken, badToken}:
-        echo "fetchBody error: ", error
-        invalidate(account)
-        raise rateLimitError()
+      if error != null and error notin errorsToSkip:
+        echo "Fetch error, API: ", api, ", error: ", error
+        if error in {expiredToken, badToken, locked}:
+          invalidate(session)
+          raise rateLimitError()
 
-proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
+proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
   retry:
-    fetchImpl(result, additional_headers):
+    fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):
         echo resp.status, ": ", result, " --- url: ", url
         result.setLen(0)

src/auth.nim

@@ -1,16 +1,15 @@
 #SPDX-License-Identifier: AGPL-3.0-only
 import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
 import types
-import experimental/parser/guestaccount
+import experimental/parser/session
 
-# max requests at a time per account to avoid race conditions
+# max requests at a time per session to avoid race conditions
 const
   maxConcurrentReqs = 2
-  dayInSeconds = 24 * 60 * 60
+  hourInSeconds = 60 * 60
   apiMaxReqs: Table[Api, int] = {
     Api.search: 50,
-    Api.tweetDetail: 150,
-    Api.photoRail: 180,
+    Api.tweetDetail: 500,
     Api.userTweets: 500,
     Api.userTweetsAndReplies: 500,
     Api.userMedia: 500,
@@ -24,23 +23,16 @@ const
   }.toTable
 
 var
-  accountPool: seq[GuestAccount]
+  sessionPool: seq[Session]
   enableLogging = false
 
 template log(str: varargs[string, `$`]) =
-  if enableLogging: echo "[accounts] ", str.join("")
+  echo "[sessions] ", str.join("")
 
 proc snowflakeToEpoch(flake: int64): int64 =
   int64(((flake shr 22) + 1288834974657) div 1000)
 
-proc hasExpired(account: GuestAccount): bool =
-  let
-    created = snowflakeToEpoch(account.id)
-    now = epochTime().int64
-    daysOld = int(now - created) div dayInSeconds
-  return daysOld > 30
-
-proc getAccountPoolHealth*(): JsonNode =
+proc getSessionPoolHealth*(): JsonNode =
   let now = epochTime().int
 
   var
@@ -51,38 +43,38 @@ proc getAccountPoolHealth*(): JsonNode =
     newest = 0'i64
     average = 0'i64
 
-  for account in accountPool:
-    let created = snowflakeToEpoch(account.id)
+  for session in sessionPool:
+    let created = snowflakeToEpoch(session.id)
     if created > newest:
       newest = created
     if created < oldest:
       oldest = created
     average += created
 
-    for api in account.apis.keys:
+    if session.limited:
+      limited.incl session.id
+
+    for api in session.apis.keys:
       let
-        apiStatus = account.apis[api]
+        apiStatus = session.apis[api]
         reqs = apiMaxReqs[api] - apiStatus.remaining
 
-      if apiStatus.limited:
-        limited.incl account.id
-
-      # no requests made with this account and endpoint since the limit reset
+      # no requests made with this session and endpoint since the limit reset
       if apiStatus.reset < now:
         continue
 
       reqsPerApi.mgetOrPut($api, 0).inc reqs
       totalReqs.inc reqs
 
-  if accountPool.len > 0:
-    average = average div accountPool.len
+  if sessionPool.len > 0:
+    average = average div sessionPool.len
   else:
     oldest = 0
     average = 0
 
   return %*{
-    "accounts": %*{
-      "total": accountPool.len,
+    "sessions": %*{
+      "total": sessionPool.len,
       "limited": limited.card,
       "oldest": $fromUnix(oldest),
       "newest": $fromUnix(newest),
@@ -94,116 +86,117 @@ proc getAccountPoolHealth*(): JsonNode =
     }
   }
 
-proc getAccountPoolDebug*(): JsonNode =
+proc getSessionPoolDebug*(): JsonNode =
   let now = epochTime().int
   var list = newJObject()
 
-  for account in accountPool:
-    let accountJson = %*{
+  for session in sessionPool:
+    let sessionJson = %*{
       "apis": newJObject(),
-      "pending": account.pending,
+      "pending": session.pending,
     }
 
-    for api in account.apis.keys:
+    if session.limited:
+      sessionJson["limited"] = %true
+
+    for api in session.apis.keys:
       let
-        apiStatus = account.apis[api]
+        apiStatus = session.apis[api]
         obj = %*{}
 
       if apiStatus.reset > now.int:
         obj["remaining"] = %apiStatus.remaining
+        obj["reset"] = %apiStatus.reset
 
-      if "remaining" notin obj and not apiStatus.limited:
+      if "remaining" notin obj:
         continue
 
-      if apiStatus.limited:
-        obj["limited"] = %true
-
-      accountJson{"apis", $api} = obj
-      list[$account.id] = accountJson
+      sessionJson{"apis", $api} = obj
+      list[$session.id] = sessionJson
 
   return %list
 
 proc rateLimitError*(): ref RateLimitError =
   newException(RateLimitError, "rate limited")
 
-proc isLimited(account: GuestAccount; api: Api): bool =
-  if account.isNil:
+proc noSessionsError*(): ref NoSessionsError =
+  newException(NoSessionsError, "no sessions available")
+
+proc isLimited(session: Session; api: Api): bool =
+  if session.isNil:
     return true
 
-  if api in account.apis:
-    let limit = account.apis[api]
+  if session.limited and api != Api.userTweets:
+    if (epochTime().int - session.limitedAt) > hourInSeconds:
+      session.limited = false
+      log "resetting limit: ", session.id
+      return false
+    else:
+      return true
 
-    if limit.limited and (epochTime().int - limit.limitedAt) > dayInSeconds:
-      account.apis[api].limited = false
-      log "resetting limit, api: ", api, ", id: ", account.id
-
-    return limit.limited or (limit.remaining <= 10 and limit.reset > epochTime().int)
+  if api in session.apis:
+    let limit = session.apis[api]
+    return limit.remaining <= 10 and limit.reset > epochTime().int
   else:
     return false
 
-proc isReady(account: GuestAccount; api: Api): bool =
-  not (account.isNil or account.pending > maxConcurrentReqs or account.isLimited(api))
+proc isReady(session: Session; api: Api): bool =
+  not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
 
-proc invalidate*(account: var GuestAccount) =
-  if account.isNil: return
-  log "invalidating expired account: ", account.id
+proc invalidate*(session: var Session) =
+  if session.isNil: return
+  log "invalidating: ", session.id
 
   # TODO: This isn't sufficient, but it works for now
-  let idx = accountPool.find(account)
-  if idx > -1: accountPool.delete(idx)
-  account = nil
+  let idx = sessionPool.find(session)
+  if idx > -1: sessionPool.delete(idx)
+  session = nil
 
-proc release*(account: GuestAccount) =
-  if account.isNil: return
-  dec account.pending
+proc release*(session: Session) =
+  if session.isNil: return
+  dec session.pending
 
-proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
-  for i in 0 ..< accountPool.len:
+proc getSession*(api: Api): Future[Session] {.async.} =
+  for i in 0 ..< sessionPool.len:
     if result.isReady(api): break
-    result = accountPool.sample()
+    result = sessionPool.sample()
 
   if not result.isNil and result.isReady(api):
     inc result.pending
   else:
-    log "no accounts available for API: ", api
-    raise rateLimitError()
+    log "no sessions available for API: ", api
+    raise noSessionsError()
 
-proc setLimited*(account: GuestAccount; api: Api) =
-  account.apis[api].limited = true
-  account.apis[api].limitedAt = epochTime().int
-  log "rate limited, api: ", api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id
+proc setLimited*(session: Session; api: Api) =
+  session.limited = true
+  session.limitedAt = epochTime().int
+  log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id
 
-proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) =
+proc setRateLimit*(session: Session; api: Api; remaining, reset: int) =
   # avoid undefined behavior in race conditions
-  if api in account.apis:
-    let limit = account.apis[api]
+  if api in session.apis:
+    let limit = session.apis[api]
     if limit.reset >= reset and limit.remaining < remaining:
       return
     if limit.reset == reset and limit.remaining >= remaining:
-      account.apis[api].remaining = remaining
+      session.apis[api].remaining = remaining
       return
 
-  account.apis[api] = RateLimit(remaining: remaining, reset: reset)
+  session.apis[api] = RateLimit(remaining: remaining, reset: reset)
 
-proc initAccountPool*(cfg: Config) =
-  let path = cfg.accountsFile
+proc initSessionPool*(cfg: Config; path: string) =
   enableLogging = cfg.enableDebug
 
-  if not path.endswith(".jsonl"):
-    log "Accounts file should be formated with JSONL"
+  if path.endsWith(".json"):
+    log "ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
     quit 1
 
   if not fileExists(path):
-    log "Failed to access the accounts file (", path, ")"
+    log "ERROR: ", path, " not found. This file is required to authenticate API requests."
     quit 1
 
-  log "Parsing JSONL accounts file: ", path
+  log "parsing JSONL account sessions file: ", path
   for line in path.lines:
-    accountPool.add parseGuestAccount(line)
+    sessionPool.add parseSession(line)
 
-  let accountsPrePurge = accountPool.len
-  #accountPool.keepItIf(not it.hasExpired)
-
-  log "Successfully added ", accountPool.len, " valid accounts."
-  if accountsPrePurge > accountPool.len:
-    log "Purged ", accountsPrePurge - accountPool.len, " expired accounts."
+  log "successfully added ", sessionPool.len, " valid account sessions"

src/config.nim

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import parsecfg except Config
 import types, strutils
-from os import getEnv
 
 proc get*[T](config: parseCfg.Config; section, key: string; default: T): T =
   let val = config.getSectionValue(section, key)
@@ -16,37 +15,32 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
 
   let conf = Config(
     # Server
-    address: cfg.get("server", "address", "0.0.0.0"),
-    port: cfg.get("server", "port", 8080),
-    useHttps: cfg.get("server", "https", true),
-    httpMaxConns: cfg.get("server", "httpMaxConnections", 100),
-    staticDir: cfg.get("server", "staticDir", "./public"),
-    accountsFile: cfg.get("server", "accountsFile", "./accounts.jsonl"),
-    title: cfg.get("server", "title", "Nitter"),
-    hostname: cfg.get("server", "hostname", "nitter.net"),
+    address: cfg.get("Server", "address", "0.0.0.0"),
+    port: cfg.get("Server", "port", 8080),
+    useHttps: cfg.get("Server", "https", true),
+    httpMaxConns: cfg.get("Server", "httpMaxConnections", 100),
+    staticDir: cfg.get("Server", "staticDir", "./public"),
+    title: cfg.get("Server", "title", "Nitter"),
+    hostname: cfg.get("Server", "hostname", "nitter.net"),
 
     # Cache
-    listCacheTime: cfg.get("cache", "listMinutes", 120),
-    rssCacheTime: cfg.get("cache", "rssMinutes", 10),
+    listCacheTime: cfg.get("Cache", "listMinutes", 120),
+    rssCacheTime: cfg.get("Cache", "rssMinutes", 10),
 
-    redisHost: cfg.get("cache", "redisHost", "localhost"),
-    redisPort: cfg.get("cache", "redisPort", 6379),
-    redisConns: cfg.get("cache", "redisConnections", 20),
-    redisMaxConns: cfg.get("cache", "redisMaxConnections", 30),
-    redisPassword: cfg.get("cache", "redisPassword", ""),
+    redisHost: cfg.get("Cache", "redisHost", "localhost"),
+    redisPort: cfg.get("Cache", "redisPort", 6379),
+    redisConns: cfg.get("Cache", "redisConnections", 20),
+    redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30),
+    redisPassword: cfg.get("Cache", "redisPassword", ""),
 
     # Config
-    hmacKey: cfg.get("config", "hmacKey", "secretkey"),
-    base64Media: cfg.get("config", "base64Media", false),
-    minTokens: cfg.get("config", "tokenCount", 10),
-    enableRss: cfg.get("config", "enableRSS", true),
-    enableDebug: cfg.get("config", "enableDebug", false),
-    proxy: cfg.get("config", "proxy", ""),
-    proxyAuth: cfg.get("config", "proxyAuth", "")
+    hmacKey: cfg.get("Config", "hmacKey", "secretkey"),
+    base64Media: cfg.get("Config", "base64Media", false),
+    minTokens: cfg.get("Config", "tokenCount", 10),
+    enableRss: cfg.get("Config", "enableRSS", true),
+    enableDebug: cfg.get("Config", "enableDebug", false),
+    proxy: cfg.get("Config", "proxy", ""),
+    proxyAuth: cfg.get("Config", "proxyAuth", "")
   )
 
   return (conf, cfg)
-
-
-let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
-let (cfg*, fullCfg*) = getConfig(configPath)

src/consts.nim

@@ -1,56 +1,28 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils, strutils
+import uri, strutils
 
 const
   consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
   consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
 
-  api = parseUri("https://api.twitter.com")
-  activate* = $(api / "1.1/guest/activate.json")
+  gql = parseUri("https://api.x.com") / "graphql"
 
-  photoRail* = api / "1.1/statuses/media_timeline.json"
-
-  timelineApi = api / "2/timeline"
-
-  graphql = api / "graphql"
-  graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
-  graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
-  graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
-  graphTweet* = graphql / "q94uRCEn65LZThakYcPT6g/TweetDetail"
-  graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
-  graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
-  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
-  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
-  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
-  graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
-  graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters"
-  graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters"
-  graphFollowers* = graphql / "EAqBhgcGr_qPOzhS4Q3scQ/Followers"
-  graphFollowing* = graphql / "JPZiqKjET7_M1r5Tlr8pyA/Following"
-  favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes"
-
-  timelineParams* = {
-    "include_can_media_tag": "1",
-    "include_cards": "1",
-    "include_entities": "1",
-    "include_profile_interstitial_type": "0",
-    "include_quote_count": "0",
-    "include_reply_count": "0",
-    "include_user_entities": "0",
-    "include_ext_reply_count": "0",
-    "include_ext_media_color": "0",
-    "cards_platform": "Web-13",
-    "tweet_mode": "extended",
-    "send_error_codes": "1",
-    "simple_quoted_tweet": "1"
-  }.toSeq
+  graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
+  graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
+  graphUserTweets* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndReplies* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserMedia* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
+  graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2"
+  graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
+  graphSearchTimeline* = gql / "KI9jCXUx3Ymt-hDKLOZb9Q/SearchTimeline"
+  graphListById* = gql / "oygmAig8kjn0pKsx_bUadQ/ListByRestId"
+  graphListBySlug* = gql / "88GTz-IPPWLn1EiU8XoNVg/ListBySlug"
+  graphListMembers* = gql / "kSmxeqEeelqdHSR7jMnb_w/ListMembers"
+  graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
 
   gqlFeatures* = """{
   "android_graphql_skip_api_media_color_palette": false,
   "blue_business_profile_image_shape_enabled": false,
-  "c9s_tweet_anatomy_moderator_badge_enabled": false,
   "creator_subscriptions_subscription_count_enabled": false,
   "creator_subscriptions_tweet_preview_api_enabled": true,
   "freedom_of_speech_not_reach_fetch_enabled": false,
@@ -72,7 +44,6 @@ const
   "responsive_web_twitter_article_tweet_consumption_enabled": false,
   "responsive_web_twitter_blue_verified_badge_is_enabled": true,
   "rweb_lists_timeline_redesign_enabled": true,
-  "rweb_video_timestamps_enabled": true,
   "spaces_2022_h2_clipping": true,
   "spaces_2022_h2_spaces_communities": true,
   "standardized_nudges_misinfo": false,
@@ -89,7 +60,23 @@ const
   "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
   "verified_phone_label_enabled": false,
   "vibe_api_enabled": false,
-  "view_counts_everywhere_api_enabled": false
+  "view_counts_everywhere_api_enabled": false,
+  "premium_content_api_read_enabled": false,
+  "communities_web_enable_tweet_community_results_fetch": false,
+  "responsive_web_jetfuel_frame": false,
+  "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
+  "responsive_web_grok_image_annotation_enabled": false,
+  "rweb_tipjar_consumption_enabled": false,
+  "profile_label_improvements_pcf_label_in_post_enabled": false,
+  "creator_subscriptions_quote_tweet_preview_enabled": false,
+  "c9s_tweet_anatomy_moderator_badge_enabled": false,
+  "responsive_web_grok_analyze_post_followups_enabled": false,
+  "rweb_video_timestamps_enabled": false,
+  "responsive_web_grok_share_attachment_enabled": false,
+  "articles_preview_enabled": false,
+  "immersive_video_status_linkable_timestamps": false,
+  "articles_api_enabled": false,
+  "responsive_web_grok_analysis_button_from_backend": false
 }""".replace(" ", "").replace("\n", "")
 
   tweetVariables* = """{
@@ -123,15 +110,3 @@ const
   "rest_id": "$1", $2
   "count": 20
 }"""
-
-  reactorsVariables* = """{
-  "tweetId" : "$1", $2
-  "count" : 20,
-  "includePromotedContent": false
-}"""
-
-  followVariables* = """{
-  "userId" : "$1", $2
-  "count" : 20,
-  "includePromotedContent": false
-}"""

src/experimental/parser/guestaccount.nim

@@ -1,21 +0,0 @@
-import std/strutils
-import jsony
-import ../types/guestaccount
-from ../../types import GuestAccount
-
-proc toGuestAccount(account: RawAccount): GuestAccount =
-  let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
-  result = GuestAccount(
-    id: parseBiggestInt(id),
-    oauthToken: account.oauthToken,
-    oauthSecret: account.oauthTokenSecret
-  )
-
-proc parseGuestAccount*(raw: string): GuestAccount =
-  let rawAccount = raw.fromJson(RawAccount)
-  result = rawAccount.toGuestAccount
-
-proc parseGuestAccounts*(path: string): seq[GuestAccount] =
-  let rawAccounts = readFile(path).fromJson(seq[RawAccount])
-  for account in rawAccounts:
-    result.add account.toGuestAccount

src/experimental/parser/session.nim

@@ -0,0 +1,15 @@
+import std/strutils
+import jsony
+import ../types/session
+from ../../types import Session
+
+proc parseSession*(raw: string): Session =
+  let 
+    session = raw.fromJson(RawSession)
+    id = session.oauthToken[0 ..< session.oauthToken.find('-')]
+
+  result = Session(
+    id: parseBiggestInt(id),
+    oauthToken: session.oauthToken,
+    oauthSecret: session.oauthTokenSecret
+  )

src/experimental/types/session.nim

@@ -1,4 +1,4 @@
 type
-  RawAccount* = object
+  RawSession* = object
     oauthToken*: string
     oauthTokenSecret*: string

src/formatters.nim

@@ -11,6 +11,8 @@ const
 let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
+  xRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?x\.com"
+  xLinkRegex = re"""<a href="https:\/\/x.com([^"]+)">x\.com(\S+)</a>"""
 
   ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
 
@@ -56,12 +58,18 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
     result = result.replace(ytRegex, prefs.replaceYouTube)
 
-  if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
-    result = result.replace(cards, prefs.replaceTwitter & "/cards")
-    result = result.replace(twRegex, prefs.replaceTwitter)
-    result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
+  if prefs.replaceTwitter.len > 0:
+    if tco in result:
+      result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
+    if "x.com" in result:
+      result = result.replace(xRegex, prefs.replaceTwitter)
+      result = result.replacef(xLinkRegex, a(
+        prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
+    if "twitter.com" in result:
+      result = result.replace(cards, prefs.replaceTwitter & "/cards")
+      result = result.replace(twRegex, prefs.replaceTwitter)
+      result = result.replacef(twLinkRegex, a(
+        prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
     result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")

src/nitter.nim

@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, strformat, logging
-import config
 from net import Port
 from htmlgen import a
 from os import getEnv
@@ -13,7 +12,13 @@ import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
   unsupported, embed, resolver, router_utils]
 
-initAccountPool(cfg)
+let
+  configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
+  (cfg, fullCfg) = getConfig(configPath)
+
+  sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl")
+
+initSessionPool(cfg, sessionsPath)
 
 if not cfg.enableDebug:
   # Silence Jester's query warning
@@ -66,7 +71,7 @@ routes:
 
   error InternalError:
     echo error.exc.name, ": ", error.exc.msg
-    const link = a("https://git.ngn.tf/ngn/nitter", href = "https://git.ngn.tf/ngn/nitter")
+    const link = a("ngn@ngn.tf", href = "mailto:ngn@ngn.tf")
     resp Http500, showError(
       &"An error occurred, please report to {link}", cfg)
 
@@ -78,6 +83,11 @@ routes:
     resp Http429, showError(
       &"Instance has been rate limited.", cfg)
 
+  error NoSessionsError:
+    const link = a("another instance", href = instancesUrl)
+    resp Http429, showError(
+      &"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
+
   extend rss, ""
   extend status, ""
   extend search, ""

src/parser.nim

@@ -3,7 +3,6 @@ import strutils, options, times, math
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
-import std/tables
 
 proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet
 
@@ -33,8 +32,7 @@ proc parseGraphUser(js: JsonNode): User =
   var user = js{"user_result", "result"}
   if user.isNull:
     user = ? js{"user_results", "result"}
-  
-  result = parseUser(user{"legacy"})
+  result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
 
   if result.verifiedType == VerifiedType.none and user{"is_blue_verified"}.getBool(false):
     result.verifiedType = blue
@@ -238,11 +236,8 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   # graphql
   with rt, js{"retweeted_status_result", "result"}:
     # needed due to weird edgecase where the actual tweet data isn't included
-    var rt_tweet = rt
-    if "tweet" in rt:
-      rt_tweet = rt{"tweet"}
-    if "legacy" in rt_tweet:
-      result.retweet = some parseGraphTweet(rt_tweet)
+    if "legacy" in rt:
+      result.retweet = some parseGraphTweet(rt)
       return
 
   if jsCard.kind != JNull:
@@ -294,138 +289,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseLegacyTweet(js: JsonNode): Tweet =
-  result = parseTweet(js, js{"card"})
-  if not result.isNil and result.available:
-    result.user = parseUser(js{"user"})
-
-    if result.quote.isSome:
-      result.quote = some parseLegacyTweet(js{"quoted_status"})
-
-proc parseTweetSearch*(js: JsonNode; after=""): Timeline =
-  result.beginning = after.len == 0
-
-  if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0:
-    return
-
-  for item in js{"modules"}:
-    with tweet, item{"status", "data"}:
-      let parsed = parseLegacyTweet(tweet)
-
-      if parsed.retweet.isSome:
-        parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})
-
-      result.content.add @[parsed]
-
-  if result.content.len > 0:
-    result.bottom = $(result.content[^1][0].id - 1)
-
-proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
-  let intId = if id.len > 0: parseBiggestInt(id) else: 0
-  result = global.tweets.getOrDefault(id, Tweet(id: intId))
-
-  if result.quote.isSome:
-    let quote = get(result.quote).id
-    if $quote in global.tweets:
-      result.quote = some global.tweets[$quote]
-    else:
-      result.quote = some Tweet()
-
-  if result.retweet.isSome:
-    let rt = get(result.retweet).id
-    if $rt in global.tweets:
-      result.retweet = some finalizeTweet(global, $rt)
-    else:
-      result.retweet = some Tweet()
-
-proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
-  let pin = js{"pinEntry", "entry", "entryId"}.getStr
-  if pin.len == 0: return
-
-  let id = pin.getId
-  if id notin global.tweets: return
-
-  global.tweets[id].pinned = true
-  return finalizeTweet(global, id)
-
-proc parseGlobalObjects(js: JsonNode): GlobalObjects =
-  result = GlobalObjects()
-  let
-    tweets = ? js{"globalObjects", "tweets"}
-    users = ? js{"globalObjects", "users"}
-
-  for k, v in users:
-    result.users[k] = parseUser(v, k)
-
-  for k, v in tweets:
-    var tweet = parseTweet(v, v{"card"})
-    if tweet.user.id in result.users:
-      tweet.user = result.users[tweet.user.id]
-    result.tweets[k] = tweet
-
-proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
-  if js.kind != JArray or js.len == 0:
-    return
-
-  for i in js:
-    if res.tweets.beginning and i{"pinEntry"}.notNull:
-      with pin, parsePin(i, global):
-        res.pinned = some pin
-
-    with r, i{"replaceEntry", "entry"}:
-      if "top" in r{"entryId"}.getStr:
-        res.tweets.top = r.getCursor
-      elif "bottom" in r{"entryId"}.getStr:
-        res.tweets.bottom = r.getCursor
-
-proc parseTimeline*(js: JsonNode; after=""): Profile =
-  result = Profile(tweets: Timeline(beginning: after.len == 0))
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0: return
-
-  result.parseInstructions(global, instructions)
-
-  var entries: JsonNode
-  for i in instructions:
-    if "addEntries" in i:
-      entries = i{"addEntries", "entries"}
-
-  for e in ? entries:
-    let entry = e{"entryId"}.getStr
-    if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
-      let tweet = finalizeTweet(global, e.getEntryId)
-      if not tweet.available: continue
-      result.tweets.content.add tweet
-    elif "cursor-top" in entry:
-      result.tweets.top = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.tweets.bottom = e.getCursor
-    elif entry.startsWith("sq-cursor"):
-      with cursor, e{"content", "operation", "cursor"}:
-        if cursor{"cursorType"}.getStr == "Bottom":
-          result.tweets.bottom = cursor{"value"}.getStr
-        else:
-          result.tweets.top = cursor{"value"}.getStr
-
-proc parsePhotoRail*(js: JsonNode): PhotoRail =
-  with error, js{"error"}:
-    if error.getStr == "Not authorized.":
-      return
-
-  for tweet in js:
-    let
-      t = parseTweet(tweet, js{"tweet_card"})
-      url = if t.photos.len > 0: t.photos[0]
-            elif t.video.isSome: get(t.video).thumb
-            elif t.gif.isSome: get(t.gif).thumb
-            elif t.card.isSome: get(t.card).image
-            else: ""
-
-    if url.len == 0: continue
-    result.add GalleryPhoto(url: url, tweetId: $t.id)
-
 proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
   if js.kind == JNull:
     return Tweet()
@@ -473,7 +336,7 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
       let cursor = t{"item", "content", "value"}
       result.thread.cursor = cursor.getStr
       result.thread.hasMore = true
-    elif "tweet" in entryId:
+    elif "tweet" in entryId and "promoted" notin entryId:
       let
         isLegacy = t{"item"}.hasKey("itemContent")
         (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
@@ -489,54 +352,60 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet =
   with tweet, js{"data", "tweet_result", "result"}:
     result = parseGraphTweet(tweet, false)
 
-proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
+proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
 
-  let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
+  let
+    rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2"
+    contentKey = if v2: "content" else: "itemContent"
+    resultKey = if v2: "tweetResult" else: "tweet_results"
+
+  let instructions = ? js{"data", rootKey, "instructions"}
   if instructions.len == 0:
     return
 
-  for e in instructions[0]{"entries"}:
-    let entryId = e{"entryId"}.getStr
-    if entryId.startsWith("tweet"):
-      with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
-        let tweet = parseGraphTweet(tweetResult, true)
+  for i in instructions:
+    if i{"__typename"}.getStr == "TimelineAddEntries":
+      for e in i{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", contentKey, resultKey, "result"}:
+            let tweet = parseGraphTweet(tweetResult, not v2)
 
-        if not tweet.available:
-          tweet.id = parseBiggestInt(entryId.getId())
+            if not tweet.available:
+              tweet.id = parseBiggestInt(entryId.getId())
 
-        if $tweet.id == tweetId:
-          result.tweet = tweet
-        else:
-          result.before.content.add tweet
-    elif entryId.startsWith("tombstone"):
-      let id = entryId.getId()
-      let tweet = Tweet(
-        id: parseBiggestInt(id),
-        available: false,
-        text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
-      )
+            if $tweet.id == tweetId:
+              result.tweet = tweet
+            else:
+              result.before.content.add tweet
+        elif entryId.startsWith("conversationthread"):
+          let (thread, self) = parseGraphThread(e)
+          if self:
+            result.after = thread
+          elif thread.content.len > 0:
+            result.replies.content.add thread
+        elif entryId.startsWith("tombstone"):
+          let id = entryId.getId()
+          let tweet = Tweet(
+            id: parseBiggestInt(id),
+            available: false,
+            text: e{"content", contentKey, "tombstoneInfo", "richText"}.getTombstone
+          )
 
-      if id == tweetId:
-        result.tweet = tweet
-      else:
-        result.before.content.add tweet
-    elif entryId.startsWith("conversationthread"):
-      let (thread, self) = parseGraphThread(e)
-      if self:
-        result.after = thread
-      else:
-        result.replies.content.add thread
-    elif entryId.startsWith("cursor-bottom"):
-      result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+          if id == tweetId:
+            result.tweet = tweet
+          else:
+            result.before.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          result.replies.bottom = e{"content", contentKey, "value"}.getStr
 
 proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
   result = Profile(tweets: Timeline(beginning: after.len == 0))
 
   let instructions =
     if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"}
-      elif root == "user": ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-    else: ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
+    else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
 
   if instructions.len == 0:
     return
@@ -556,21 +425,6 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
           result.tweets.content.add thread.content
         elif entryId.startsWith("cursor-bottom"):
           result.tweets.bottom = e{"content", "value"}.getStr
-    # TODO cleanup
-    if i{"type"}.getStr == "TimelineAddEntries":
-      for e in i{"entries"}:
-        let entryId = e{"entryId"}.getStr
-        if entryId.startsWith("tweet"):
-          with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
-            let tweet = parseGraphTweet(tweetResult, false)
-            if not tweet.available:
-              tweet.id = parseBiggestInt(entryId.getId())
-            result.tweets.content.add tweet
-        elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
-          let (thread, self) = parseGraphThread(e)
-          result.tweets.content.add thread.content
-        elif entryId.startsWith("cursor-bottom"):
-          result.tweets.bottom = e{"content", "value"}.getStr
     if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
       with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
         let tweet = parseGraphTweet(tweetResult, false)
@@ -581,35 +435,34 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
             tweet.id = parseBiggestInt(entryId)
         result.pinned = some tweet
 
-proc parseGraphUsersTimeline(timeline: JsonNode; after=""): UsersTimeline =
-  result = UsersTimeline(beginning: after.len == 0)
+proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
+  result = @[]
 
-  let instructions = ? timeline{"instructions"}
-
-  if instructions.len == 0:
-    return
+  let instructions =
+    ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
 
   for i in instructions:
-    if i{"type"}.getStr == "TimelineAddEntries":
+    if i{"__typename"}.getStr == "TimelineAddEntries":
       for e in i{"entries"}:
         let entryId = e{"entryId"}.getStr
-        if entryId.startsWith("user"):
-          with graphUser, e{"content", "itemContent"}:
-            let user = parseGraphUser(graphUser)
-            result.content.add user
-        elif entryId.startsWith("cursor-bottom"):
-          result.bottom = e{"content", "value"}.getStr
-        elif entryId.startsWith("cursor-top"):
-          result.top = e{"content", "value"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "content", "tweetResult", "result"}:
+            let t = parseGraphTweet(tweetResult, false)
+            if not t.available:
+              t.id = parseBiggestInt(entryId.getId())
 
-proc parseGraphFavoritersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline =
-  return parseGraphUsersTimeline(js{"data", "favoriters_timeline", "timeline"}, after)
+            let url = 
+              if t.photos.len > 0: t.photos[0]
+              elif t.video.isSome: get(t.video).thumb
+              elif t.gif.isSome: get(t.gif).thumb
+              elif t.card.isSome: get(t.card).image
+              else: ""
 
-proc parseGraphRetweetersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline =
-  return parseGraphUsersTimeline(js{"data", "retweeters_timeline", "timeline"}, after)
+            if url.len > 0:
+              result.add GalleryPhoto(url: url, tweetId: $t.id)
 
-proc parseGraphFollowTimeline*(js: JsonNode; root: string; after=""): UsersTimeline =
-  return parseGraphUsersTimeline(js{"data", "user", "result", "timeline", "timeline"}, after)
+            if result.len == 16:
+              break
 
 proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
   result = Result[T](beginning: after.len == 0)

src/prefs_impl.nim

@@ -125,7 +125,7 @@ macro genDefaultPrefs*(): untyped =
         of input: newLit(pref.defaultInput)
 
     result.add quote do:
-      defaultPrefs.`ident` = cfg.get("preferences", `name`, `default`)
+      defaultPrefs.`ident` = cfg.get("Preferences", `name`, `default`)
 
 macro genCookiePrefs*(cookies): untyped =
   result = nnkStmtList.newTree()

src/query.nim

@@ -40,13 +40,6 @@ proc getMediaQuery*(name: string): Query =
     sep: "OR"
   )
 
-
-proc getFavoritesQuery*(name: string): Query =
-  Query(
-    kind: favorites,
-    fromUser: @[name]
-  )
-
 proc getReplyQuery*(name: string): Query =
   Query(
     kind: replies,

src/redis_cache.nim

@@ -86,7 +86,7 @@ proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: PhotoRail; name: string) {.async.} =
-  await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
+  await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
 
 proc cache*(data: User) {.async.} =
   if data.username.len == 0: return
@@ -158,14 +158,14 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
 #     if not result.isNil:
 #       await cache(result)
 
-proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
-  let rail = await get("pr:" & toLower(name))
+proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+  if id.len == 0: return
+  let rail = await get("pr2:" & toLower(id))
   if rail != redisNil:
     rail.deserialize(PhotoRail)
   else:
-    result = await getPhotoRail(name)
-    await cache(result, name)
+    result = await getPhotoRail(id)
+    await cache(result, id)
 
 proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
   let list = if id.len == 0: redisNil

src/routes/debug.nim

@@ -6,8 +6,8 @@ import ".."/[auth, types]
 proc createDebugRouter*(cfg: Config) =
   router debug:
     get "/.health":
-      respJson getAccountPoolHealth()
+      respJson getSessionPoolHealth()
 
-    get "/.accounts":
+    get "/.sessions":
       cond cfg.enableDebug
-      respJson getAccountPoolDebug()
+      respJson getSessionPoolDebug()

src/routes/media.nim

@@ -37,7 +37,8 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
   try:
     let res = await client.get(url)
     if res.status != "200 OK":
-      echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
+      if res.status != "404 Not Found":
+        echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
       return Http404
 
     let hashed = $hash(url)
@@ -122,7 +123,7 @@ proc createMediaRouter*(cfg: Config) =
       cond "http" in url
 
       if getHmac(url) != request.matches[1]:
-        resp showError("Failed to verify signature", cfg)
+        resp Http403, showError("Failed to verify signature", cfg)
 
       if ".mp4" in url or ".ts" in url or ".m4s" in url:
         let code = await proxyMedia(request, url)

src/routes/rss.nim

@@ -23,7 +23,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
     names = getNames(name)
 
   if names.len == 1:
-    profile = await fetchProfile(after, query, cfg, skipRail=true, skipPinned=true)
+    profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
   else:
     var q = query
     q.fromUser = names
@@ -102,7 +102,7 @@ proc createRssRouter*(cfg: Config) =
     get "/@name/@tab/rss":
       cond cfg.enableRss
       cond '.' notin @"name"
-      cond @"tab" in ["with_replies", "media", "favorites", "search"]
+      cond @"tab" in ["with_replies", "media", "search"]
       let
         name = @"name"
         tab = @"tab"
@@ -110,7 +110,6 @@ proc createRssRouter*(cfg: Config) =
           case tab
           of "with_replies": getReplyQuery(name)
           of "media": getMediaQuery(name)
-          of "favorites": getFavoritesQuery(name)
           of "search": initQuery(params(request), name=name)
           else: Query(fromUser: @[name])
 

src/routes/status.nim

@@ -5,7 +5,7 @@ import jester, karax/vdom
 
 import router_utils
 import ".."/[types, formatters, api]
-import ../views/[general, status, search]
+import ../views/[general, status]
 
 export uri, sequtils, options, sugar
 export router_utils
@@ -14,29 +14,6 @@ export status
 
 proc createStatusRouter*(cfg: Config) =
   router status:
-    get "/@name/status/@id/@reactors":
-      cond '.' notin @"name"
-      let id = @"id"
-
-      if id.len > 19 or id.any(c => not c.isDigit):
-        resp Http404, showError("Invalid tweet ID", cfg)
-
-      let prefs = cookiePrefs()
-
-      # used for the infinite scroll feature
-      if @"scroll".len > 0:
-        let replies = await getReplies(id, getCursor())
-        if replies.content.len == 0:
-          resp Http404, ""
-        resp $renderReplies(replies, prefs, getPath())
-
-      if @"reactors" == "favoriters":
-        resp renderMain(renderUserList(await getGraphFavoriters(id, getCursor()), prefs),
-                        request, cfg, prefs)
-      elif @"reactors" == "retweeters":
-        resp renderMain(renderUserList(await getGraphRetweeters(id, getCursor()), prefs),
-                        request, cfg, prefs)
-
     get "/@name/status/@id/?":
       cond '.' notin @"name"
       let id = @"id"

src/routes/timeline.nim

@@ -16,7 +16,6 @@ proc getQuery*(request: Request; tab, name: string): Query =
   case tab
   of "with_replies": getReplyQuery(name)
   of "media": getMediaQuery(name)
-  of "favorites": getFavoritesQuery(name)
   of "search": initQuery(params(request), name=name)
   else: Query(fromUser: @[name])
 
@@ -28,7 +27,7 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] =
   else:
     body
 
-proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
+proc fetchProfile*(after: string; query: Query; skipRail=false;
                    skipPinned=false): Future[Profile] {.async.} =
   let
     name = query.fromUser[0]
@@ -48,7 +47,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
   let
     rail =
       skipIf(skipRail or query.kind == media, @[]):
-        getCachedPhotoRail(name)
+        getCachedPhotoRail(userId)
 
     user = getCachedUser(name)
 
@@ -57,7 +56,6 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
     of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after)
     of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
     of media: await getGraphUserTweets(userId, TimelineKind.media, after)
-    of favorites: await getFavorites(userId, cfg, after)
     else: Profile(tweets: await getGraphTweetSearch(query, after))
 
   result.user = await user
@@ -73,7 +71,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
-  var profile = await fetchProfile(after, query, cfg, skipPinned=prefs.hidePins)
+  var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
   template u: untyped = profile.user
 
   if u.suspended:
@@ -81,7 +79,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
 
   if profile.user.id.len == 0: return
 
-  let pHtml = renderProfile(profile, cfg, prefs, getPath())
+  let pHtml = renderProfile(profile, prefs, getPath())
   result = renderMain(pHtml, request, cfg, prefs, pageTitle(u), pageDesc(u),
                       rss=rss, images = @[u.getUserPic("_400x400")],
                       banner=u.banner)
@@ -111,42 +109,35 @@ proc createTimelineRouter*(cfg: Config) =
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
       cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
-      cond @"tab" in ["with_replies", "media", "search", "favorites", "following", "followers", ""]
+      cond @"tab" in ["with_replies", "media", "search", ""]
       let
         prefs = cookiePrefs()
         after = getCursor()
         names = getNames(@"name")
-        tab = @"tab"
 
-      case tab:
-        of "followers":
-          resp renderMain(renderUserList(await getGraphFollowers(await getUserId(@"name"), getCursor()), prefs), request, cfg, prefs)
-        of "following":
-          resp renderMain(renderUserList(await getGraphFollowing(await getUserId(@"name"), getCursor()), prefs), request, cfg, prefs)
+      var query = request.getQuery(@"tab", @"name")
+      if names.len != 1:
+        query.fromUser = names
+
+      # used for the infinite scroll feature
+      if @"scroll".len > 0:
+        if query.fromUser.len != 1:
+          var timeline = await getGraphTweetSearch(query, after)
+          if timeline.content.len == 0: resp Http404
+          timeline.beginning = true
+          resp $renderTweetSearch(timeline, prefs, getPath())
         else:
-          var query = request.getQuery(@"tab", @"name")
-          if names.len != 1:
-            query.fromUser = names
+          var profile = await fetchProfile(after, query, skipRail=true)
+          if profile.tweets.content.len == 0: resp Http404
+          profile.tweets.beginning = true
+          resp $renderTimelineTweets(profile.tweets, prefs, getPath())
 
-          # used for the infinite scroll feature
-          if @"scroll".len > 0:
-            if query.fromUser.len != 1:
-              var timeline = await getGraphTweetSearch(query, after)
-              if timeline.content.len == 0: resp Http404
-              timeline.beginning = true
-              resp $renderTweetSearch(timeline, prefs, getPath())
-            else:
-              var profile = await fetchProfile(after, query, cfg, skipRail=true)
-              if profile.tweets.content.len == 0: resp Http404
-              profile.tweets.beginning = true
-              resp $renderTimelineTweets(profile.tweets, prefs, getPath())
+      let rss =
+        if @"tab".len == 0:
+          "/$1/rss" % @"name"
+        elif @"tab" == "search":
+          "/$1/search/rss?$2" % [@"name", genQueryUrl(query)]
+        else:
+          "/$1/$2/rss" % [@"name", @"tab"]
 
-          let rss =
-            if @"tab".len == 0:
-              "/$1/rss" % @"name"
-            elif @"tab" == "search":
-              "/$1/search/rss?$2" % [@"name", genQueryUrl(query)]
-            else:
-              "/$1/$2/rss" % [@"name", @"tab"]
-
-          respTimeline(await showTimeline(request, query, cfg, prefs, rss, after))
+      respTimeline(await showTimeline(request, query, cfg, prefs, rss, after))

src/sass/tweet/_base.scss

@@ -207,7 +207,6 @@
     padding-top: 5px;
     min-width: 1em;
     margin-right: 0.8em;
-    pointer-events: all;
 }
 
 .show-thread {

src/sass/tweet/video.scss

@@ -16,6 +16,8 @@ video {
 }
 
 .video-container {
+    min-height: 80px;
+    min-width: 200px;
     max-height: 530px;
     margin: 0;
     display: flex;

src/types.nim

@@ -6,6 +6,7 @@ genPrefsType()
 
 type
   RateLimitError* = object of CatchableError
+  NoSessionsError* = object of CatchableError
   InternalError* = object of CatchableError
   BadClientError* = object of CatchableError
 
@@ -15,7 +16,6 @@ type
   Api* {.pure.} = enum
     tweetDetail
     tweetResult
-    photoRail
     search
     list
     listBySlug
@@ -23,26 +23,21 @@ type
     listTweets
     userRestId
     userScreenName
-    favorites
     userTweets
     userTweetsAndReplies
     userMedia
-    favoriters
-    retweeters
-    following
-    followers
 
   RateLimit* = object
     remaining*: int
     reset*: int
-    limited*: bool
-    limitedAt*: int
 
-  GuestAccount* = ref object
+  Session* = ref object
     id*: int64
     oauthToken*: string
     oauthSecret*: string
     pending*: int
+    limited*: bool
+    limitedAt*: int
     apis*: Table[Api, RateLimit]
 
   Error* = enum
@@ -50,8 +45,10 @@ type
     noUserMatches = 17
     protectedUser = 22
     missingParams = 25
+    timeout = 29
     couldntAuth = 32
     doesntExist = 34
+    unauthorized = 37
     invalidParam = 47
     userNotFound = 50
     suspended = 63
@@ -61,7 +58,9 @@ type
     tweetNotFound = 144
     tweetNotAuthorized = 179
     forbidden = 200
+    badRequest = 214
     badToken = 239
+    locked = 326
     noCsrf = 353
     tweetUnavailable = 421
     tweetCensored = 422
@@ -116,7 +115,7 @@ type
     variants*: seq[VideoVariant]
 
   QueryKind* = enum
-    posts, replies, media, users, tweets, userList, favorites
+    posts, replies, media, users, tweets, userList
 
   Query* = object
     kind*: QueryKind
@@ -236,7 +235,6 @@ type
     replies*: Result[Chain]
 
   Timeline* = Result[Tweets]
-  UsersTimeline* = Result[User]
 
   Profile* = object
     user*: User
@@ -265,7 +263,6 @@ type
     title*: string
     hostname*: string
     staticDir*: string
-    accountsFile*: string
 
     hmacKey*: string
     base64Media*: bool
@@ -283,7 +280,6 @@ type
     redisConns*: int
     redisMaxConns*: int
     redisPassword*: string
-    redisDb*: int
 
   Rss* = object
     feed*, cursor*: string

src/views/profile.nim

@@ -58,14 +58,10 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
 
       tdiv(class="profile-card-extra-links"):
         ul(class="profile-statlist"):
-          a(href="/" & user.username):
-            renderStat(user.tweets, "posts", text="Tweets")
-          a(href="/" & user.username & "/following"):
-            renderStat(user.following, "following")
-          a(href="/" & user.username & "/followers"):
-            renderStat(user.followers, "followers")
-          a(href="/" & user.username & "/favorites"):
-            renderStat(user.likes, "likes")
+          renderStat(user.tweets, "posts", text="Tweets")
+          renderStat(user.following, "following")
+          renderStat(user.followers, "followers")
+          renderStat(user.likes, "likes")
 
 proc renderPhotoRail(profile: Profile): VNode =
   let count = insertSep($profile.user.media, ',')
@@ -103,7 +99,7 @@ proc renderProtected(username: string): VNode =
       h2: text "This account's tweets are protected."
       p: text &"Only confirmed followers have access to @{username}'s tweets."
 
-proc renderProfile*(profile: var Profile; cfg: Config; prefs: Prefs; path: string): VNode =
+proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
   profile.tweets.query.fromUser = @[profile.user.username]
 
   buildHtml(tdiv(class="profile-tabs")):

src/views/renderutils.nim

@@ -91,7 +91,7 @@ proc genDate*(pref, state: string): VNode =
 
 proc genImg*(url: string; class=""): VNode =
   buildHtml():
-    img(src=getPicUrl(url), class=class, alt="")
+    img(src=getPicUrl(url), class=class, alt="", loading="lazy")
 
 proc getTabClass*(query: Query; tab: QueryKind): string =
   if query.kind == tab: "tab-item active"

src/views/search.nim

@@ -3,7 +3,7 @@ import strutils, strformat, sequtils, unicode, tables, options
 import karax/[karaxdsl, vdom]
 
 import renderutils, timeline
-import ".."/[types, query, config]
+import ".."/[types, query]
 
 const toggles = {
   "nativeretweets": "Retweets",
@@ -24,12 +24,12 @@ proc renderSearch*(): VNode =
   buildHtml(tdiv(class="panel-container")):
     tdiv(class="search-bar"):
       form(`method`="get", action="/search", autocomplete="off"):
-        hiddenField("f", "users")
+        hiddenField("f", "tweets")
         input(`type`="text", name="q", autofocus="",
-              placeholder="Enter username...", dir="auto")
+              placeholder="Search...", dir="auto")
         button(`type`="submit"): icon "search"
 
-proc renderProfileTabs*(query: Query; username: string; cfg: Config): VNode =
+proc renderProfileTabs*(query: Query; username: string): VNode =
   let link = "/" & username
   buildHtml(ul(class="tab")):
     li(class=query.getTabClass(posts)):
@@ -38,8 +38,6 @@ proc renderProfileTabs*(query: Query; username: string; cfg: Config): VNode =
       a(href=(link & "/with_replies")): text "Tweets & Replies"
     li(class=query.getTabClass(media)):
       a(href=(link & "/media")): text "Media"
-    li(class=query.getTabClass(favorites)):
-      a(href=(link & "/favorites")): text "Likes"
     li(class=query.getTabClass(tweets)):
       a(href=(link & "/search")): text "Search"
 
@@ -99,7 +97,7 @@ proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
         text query.fromUser.join(" | ")
 
     if query.fromUser.len > 0:
-      renderProfileTabs(query, query.fromUser.join(","), cfg)
+      renderProfileTabs(query, query.fromUser.join(","))
 
     if query.fromUser.len == 0 or query.kind == tweets:
       tdiv(class="timeline-header"):
@@ -120,8 +118,3 @@ proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode =
 
     renderSearchTabs(results.query)
     renderTimelineUsers(results, prefs)
-
-proc renderUserList*(results: Result[User]; prefs: Prefs): VNode =
-  buildHtml(tdiv(class="timeline-container")):
-    tdiv(class="timeline-header")
-    renderTimelineUsers(results, prefs)

src/views/tweet.nim

@@ -10,9 +10,7 @@ import general
 const doctype = "<!DOCTYPE html>\n"
 
 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
-  let url = getPicUrl(user.getUserPic("_mini"))
-  buildHtml():
-    img(class=(prefs.getAvatarClass & " mini"), src=url)
+  genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini"))
 
 proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
   buildHtml(tdiv):
@@ -92,10 +90,10 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
       tdiv(class="attachment video-container"):
         let thumb = getSmallPic(video.thumb)
         if not video.available:
-          img(src=thumb)
+          img(src=thumb, loading="lazy")
           renderVideoUnavailable(video)
         elif not prefs.isPlaybackEnabled(playbackType):
-          img(src=thumb)
+          img(src=thumb, loading="lazy")
           renderVideoDisabled(playbackType, path)
         else:
           let
@@ -144,7 +142,7 @@ proc renderPoll(poll: Poll): VNode =
 proc renderCardImage(card: Card): VNode =
   buildHtml(tdiv(class="card-image-container")):
     tdiv(class="card-image"):
-      img(src=getPicUrl(card.image), alt="")
+      genImg(card.image)
       if card.kind == player:
         tdiv(class="card-overlay"):
           tdiv(class="overlay-circle"):
@@ -180,19 +178,14 @@ func formatStat(stat: int): string =
   if stat > 0: insertSep($stat, ',')
   else: ""
 
-proc renderStats(stats: TweetStats; views: string; tweet: Tweet): VNode =
+proc renderStats(stats: TweetStats; views: string): VNode =
   buildHtml(tdiv(class="tweet-stats")):
-    a(href=getLink(tweet)):
-      span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
-    a(href=getLink(tweet, false) & "/retweeters"):
-      span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
-    a(href="/search?q=quoted_tweet_id:" & $tweet.id):
-      span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
-    a(href=getLink(tweet, false) & "/favoriters"):
-      span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
-    a(href=getLink(tweet)):
-      if views.len > 0:
-        span(class="tweet-stat"): icon "play", insertSep(views, ',')
+    span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
+    span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
+    span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
+    span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
+    if views.len > 0:
+      span(class="tweet-stat"): icon "play", insertSep(views, ',')
 
 proc renderReply(tweet: Tweet): VNode =
   buildHtml(tdiv(class="replying-to")):
@@ -350,7 +343,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderMediaTags(tweet.mediaTags)
 
       if not prefs.hideTweetStats:
-        renderStats(tweet.stats, views, tweet)
+        renderStats(tweet.stats, views)
 
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):

tests/base.py

@@ -0,0 +1,102 @@
+from seleniumbase import BaseCase
+
+
+class Card(object):
+    def __init__(self, tweet=''):
+        card = tweet + '.card '
+        self.link = card + 'a'
+        self.title = card + '.card-title'
+        self.description = card + '.card-description'
+        self.destination = card + '.card-destination'
+        self.image = card + '.card-image'
+
+
+class Quote(object):
+    def __init__(self, tweet=''):
+        quote = tweet + '.quote '
+        namerow = quote + '.fullname-and-username '
+        self.link = quote + '.quote-link'
+        self.fullname = namerow + '.fullname'
+        self.username = namerow + '.username'
+        self.text = quote + '.quote-text'
+        self.media = quote + '.quote-media-container'
+        self.unavailable = quote + '.quote.unavailable'
+
+
+class Tweet(object):
+    def __init__(self, tweet=''):
+        namerow = tweet + '.tweet-header '
+        self.fullname = namerow + '.fullname'
+        self.username = namerow + '.username'
+        self.date = namerow + '.tweet-date'
+        self.text = tweet + '.tweet-content.media-body'
+        self.retweet = tweet + '.retweet-header'
+        self.reply = tweet + '.replying-to'
+
+
+class Profile(object):
+    fullname = '.profile-card-fullname'
+    username = '.profile-card-username'
+    protected = '.icon-lock'
+    verified = '.verified-icon'
+    banner = '.profile-banner'
+    bio = '.profile-bio'
+    location = '.profile-location'
+    website = '.profile-website'
+    joinDate = '.profile-joindate'
+    mediaCount = '.photo-rail-header'
+
+
+class Timeline(object):
+    newest = 'div[class="timeline-item show-more"]'
+    older = 'div[class="show-more"]'
+    end = '.timeline-end'
+    none = '.timeline-none'
+    protected = '.timeline-protected'
+    photo_rail = '.photo-rail-grid'
+
+
+class Conversation(object):
+    main = '.main-tweet'
+    before = '.before-tweet'
+    after = '.after-tweet'
+    replies = '.replies'
+    thread = '.reply'
+    tweet = '.timeline-item'
+    tweet_text = '.tweet-content'
+
+
+class Poll(object):
+    votes = '.poll-info'
+    choice = '.poll-meter'
+    value = 'poll-choice-value'
+    option = 'poll-choice-option'
+    leader = 'leader'
+
+
+class Media(object):
+    container = '.attachments'
+    row = '.gallery-row'
+    image = '.still-image'
+    video = '.gallery-video'
+    gif = '.gallery-gif'
+
+
+class BaseTestCase(BaseCase):
+    def setUp(self):
+        super(BaseTestCase, self).setUp()
+
+    def tearDown(self):
+        super(BaseTestCase, self).tearDown()
+
+    def open_nitter(self, page=''):
+        self.open(f'http://localhost:8080/{page}')
+
+    def search_username(self, username):
+        self.open_nitter()
+        self.update_text('.search-bar input[type=text]', username)
+        self.submit('.search-bar form')
+
+
+def get_timeline_tweet(num=1):
+    return Tweet(f'.timeline > div:nth-child({num}) ')

tests/requirements.txt

@@ -0,0 +1 @@
+seleniumbase

tests/test_card.py

@@ -0,0 +1,89 @@
+from base import BaseTestCase, Card, Conversation
+from parameterized import parameterized
+
+
+card = [
+    ['nim_lang/status/1136652293510717440',
+     'Version 0.20.0 released',
+     'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
+     'nim-lang.org', True],
+
+    ['voidtarget/status/1094632512926605312',
+     'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
+     'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
+     'gist.github.com', True],
+
+    ['nim_lang/status/1082989146040340480',
+     'Nim in 2018: A short recap',
+     'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
+     'nim-lang.org', True]
+]
+
+no_thumb = [
+    ['FluentAI/status/1116417904831029248',
+     'LinkedIn',
+     'This link will take you to a page that’s not on LinkedIn',
+     'lnkd.in'],
+
+    ['Thom_Wolf/status/1122466524860702729',
+     'GitHub - NVIDIA/Megatron-LM: Ongoing research training transformer models at scale',
+     'Ongoing research training transformer models at scale - NVIDIA/Megatron-LM',
+     'github.com'],
+
+    ['brent_p/status/1088857328680488961',
+     'Hts Nim Sugar',
+     'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',
+     'brentp.github.io'],
+
+    ['voidtarget/status/1133028231672582145',
+     'sinkingsugar/nimqt-example',
+     'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.',
+     'github.com']
+]
+
+playable = [
+    ['nim_lang/status/1118234460904919042',
+     'Nim development blog 2019-03',
+     'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...',
+     'youtube.com'],
+
+    ['nim_lang/status/1121090879823986688',
+     'Nim - First natively compiled language w/ hot code-reloading at...',
+     '#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...',
+     'youtube.com']
+]
+
+class CardTest(BaseTestCase):
+    @parameterized.expand(card)
+    def test_card(self, tweet, title, description, destination, large):
+        self.open_nitter(tweet)
+        c = Card(Conversation.main + " ")
+        self.assert_text(title, c.title)
+        self.assert_text(destination, c.destination)
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
+        if len(description) > 0:
+            self.assert_text(description, c.description)
+        if large:
+            self.assert_element_visible('.card.large')
+        else:
+            self.assert_element_not_visible('.card.large')
+
+    @parameterized.expand(no_thumb)
+    def test_card_no_thumb(self, tweet, title, description, destination):
+        self.open_nitter(tweet)
+        c = Card(Conversation.main + " ")
+        self.assert_text(title, c.title)
+        self.assert_text(destination, c.destination)
+        if len(description) > 0:
+            self.assert_text(description, c.description)
+
+    @parameterized.expand(playable)
+    def test_card_playable(self, tweet, title, description, destination):
+        self.open_nitter(tweet)
+        c = Card(Conversation.main + " ")
+        self.assert_text(title, c.title)
+        self.assert_text(destination, c.destination)
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
+        self.assert_element_visible('.card-overlay')
+        if len(description) > 0:
+            self.assert_text(description, c.description)

tests/test_profile.py

@@ -0,0 +1,76 @@
+from base import BaseTestCase, Profile
+from parameterized import parameterized
+
+profiles = [
+        ['mobile_test', 'Test account',
+         'Test Account. test test Testing username with @mobile_test_2 and a #hashtag',
+         'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '97'],
+        ['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13']
+]
+
+verified = [['jack'], ['elonmusk']]
+
+protected = [
+    ['mobile_test_7', 'mobile test 7', ''],
+    ['Poop', 'Randy', 'Social media fanatic.']
+]
+
+invalid = [['thisprofiledoesntexist'], ['%']]
+
+banner_image = [
+    ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
+]
+
+
+class ProfileTest(BaseTestCase):
+    @parameterized.expand(profiles)
+    def test_data(self, username, fullname, bio, location, website, joinDate, mediaCount):
+        self.open_nitter(username)
+        self.assert_exact_text(fullname, Profile.fullname)
+        self.assert_exact_text(f'@{username}', Profile.username)
+
+        tests = [
+            (bio, Profile.bio),
+            (location, Profile.location),
+            (website, Profile.website),
+            (joinDate, Profile.joinDate),
+            (mediaCount + " Photos and videos", Profile.mediaCount)
+        ]
+
+        for text, selector in tests:
+            if len(text) > 0:
+                self.assert_exact_text(text, selector)
+            else:
+                self.assert_element_absent(selector)
+
+    @parameterized.expand(verified)
+    def test_verified(self, username):
+        self.open_nitter(username)
+        self.assert_element_visible(Profile.verified)
+
+    @parameterized.expand(protected)
+    def test_protected(self, username, fullname, bio):
+        self.open_nitter(username)
+        self.assert_element_visible(Profile.protected)
+        self.assert_exact_text(fullname, Profile.fullname)
+        self.assert_exact_text(f'@{username}', Profile.username)
+
+        if len(bio) > 0:
+            self.assert_text(bio, Profile.bio)
+        else:
+            self.assert_element_absent(Profile.bio)
+
+    @parameterized.expand(invalid)
+    def test_invalid_username(self, username):
+        self.open_nitter(username)
+        self.assert_text(f'User "{username}" not found')
+
+    def test_suspended(self):
+        self.open_nitter('suspendme')
+        self.assert_text('User "suspendme" has been suspended')
+
+    @parameterized.expand(banner_image)
+    def test_banner_image(self, username, url):
+        self.open_nitter(username)
+        banner = self.find_element(Profile.banner + ' img')
+        self.assertIn(url, banner.get_attribute('src'))

tests/test_quote.py

@@ -0,0 +1,63 @@
+from base import BaseTestCase, Quote, Conversation
+from parameterized import parameterized
+
+text = [
+    ['elonmusk/status/1138136540096319488',
+     'TREV PAGE', '@Model3Owners',
+     """As of March 58.4% of new car sales in Norway are electric. 
+
+What are we doing wrong? reuters.com/article/us-norwa…"""],
+
+    ['nim_lang/status/1491461266849808397#m',
+     'Nim', '@nim_lang',
+     """What's better than Nim 1.6.0?
+
+Nim 1.6.2 :)
+
+nim-lang.org/blog/2021/12/17…"""]
+]
+
+image = [
+    ['elonmusk/status/1138827760107790336', 'D83h6Y8UIAE2Wlz'],
+    ['SpaceX/status/1067155053461426176', 'Ds9EYfxXoAAPNmx']
+]
+
+gif = [
+    ['SpaceX/status/747497521593737216', 'Cl-R5yFWkAA_-3X'],
+    ['nim_lang/status/1068099315074248704', 'DtJSqP9WoAAKdRC']
+]
+
+video = [
+    ['bkuensting/status/1067316003200217088', 'IyCaQlzF0q8u9vBd']
+]
+
+
+class QuoteTest(BaseTestCase):
+    @parameterized.expand(text)
+    def test_text(self, tweet, fullname, username, text):
+        self.open_nitter(tweet)
+        quote = Quote(Conversation.main + " ")
+        self.assert_text(fullname, quote.fullname)
+        self.assert_text(username, quote.username)
+        self.assert_text(text, quote.text)
+
+    @parameterized.expand(image)
+    def test_image(self, tweet, url):
+        self.open_nitter(tweet)
+        quote = Quote(Conversation.main + " ")
+        self.assert_element_visible(quote.media)
+        self.assertIn(url, self.get_image_url(quote.media + ' img'))
+
+    @parameterized.expand(gif)
+    def test_gif(self, tweet, url):
+        self.open_nitter(tweet)
+        quote = Quote(Conversation.main + " ")
+        self.assert_element_visible(quote.media)
+        self.assertIn(url, self.get_attribute(quote.media + ' source', 'src'))
+
+    @parameterized.expand(video)
+    def test_video(self, tweet, url):
+        self.open_nitter(tweet)
+        quote = Quote(Conversation.main + " ")
+        self.assert_element_visible(quote.media)
+        self.assertIn(url, self.get_image_url(quote.media + ' img'))

tests/test_search.py

@@ -0,0 +1,9 @@
+from base import BaseTestCase
+from parameterized import parameterized
+
+
+#class SearchTest(BaseTestCase):
+    #@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
+    #def test_username_search(self, username):
+        #self.search_username(username)
+        #self.assert_text(f'{username}')

tests/test_thread.py

@@ -0,0 +1,52 @@
+from base import BaseTestCase, Conversation
+from parameterized import parameterized
+
+thread = [
+    ['octonion/status/975253897697611777', [], 'Based', ['Crystal', 'Julia'], [
+        ['For', 'Then', 'Okay,', 'Python', 'Speed', 'Java', 'Coding', 'I', 'You'],
+        ['yeah,']
+    ]],
+
+    ['octonion/status/975254452625002496', ['Based'], 'Crystal', ['Julia'], []],
+
+    ['octonion/status/975256058384887808', ['Based', 'Crystal'], 'Julia', [], []],
+
+    ['gauravssnl/status/975364889039417344',
+     ['Based', 'For', 'Then', 'Okay,', 'Python'], 'Speed', [], [
+         ['Java', 'Coding', 'I', 'You'], ['JAVA!']
+     ]],
+
+    ['d0m96/status/1141811379407425537', [], 'I\'m',
+     ['The', 'The', 'Today', 'Some', 'If', 'There', 'Above'],
+     [['Thank', 'Also,']]],
+
+    ['gmpreussner/status/999766552546299904', [], 'A', [],
+     [['I', 'Especially'], ['I']]]
+]
+
+
+class ThreadTest(BaseTestCase):
+    def find_tweets(self, selector):
+        return self.find_elements(f"{selector} {Conversation.tweet_text}")
+
+    def compare_first_word(self, tweets, selector):
+        if len(tweets) > 0:
+            self.assert_element_visible(selector)
+            for i, tweet in enumerate(self.find_tweets(selector)):
+                text = tweet.text.split(" ")[0]
+                self.assert_equal(tweets[i], text)
+
+    @parameterized.expand(thread)
+    def test_thread(self, tweet, before, main, after, replies):
+        self.open_nitter(tweet)
+        self.assert_element_visible(Conversation.main)
+
+        self.assert_text(main, Conversation.main)
+        self.assert_text(main, Conversation.main)
+
+        self.compare_first_word(before, Conversation.before)
+        self.compare_first_word(after, Conversation.after)
+
+        for i, reply in enumerate(self.find_elements(Conversation.thread)):
+            selector = Conversation.replies + f" > div:nth-child({i + 1})"
+            self.compare_first_word(replies[i], selector)

tests/test_timeline.py

@@ -0,0 +1,64 @@
+from base import BaseTestCase, Timeline
+from parameterized import parameterized
+
+normal = [['jack'], ['elonmusk']]
+
+after = [['jack', '1681686036294803456'],
+         ['elonmusk', '1681686036294803456']]
+
+no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
+
+empty = [['emptyuser'], ['mobile_test_10']]
+
+protected = [['mobile_test_7'], ['Empty_user']]
+
+photo_rail = [['mobile_test', ['Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG']]]
+
+
+class TweetTest(BaseTestCase):
+    @parameterized.expand(normal)
+    def test_timeline(self, username):
+        self.open_nitter(username)
+        self.assert_element_present(Timeline.older)
+        self.assert_element_absent(Timeline.newest)
+        self.assert_element_absent(Timeline.end)
+        self.assert_element_absent(Timeline.none)
+
+    @parameterized.expand(after)
+    def test_after(self, username, cursor):
+        self.open_nitter(f'{username}?cursor={cursor}')
+        self.assert_element_present(Timeline.newest)
+        self.assert_element_present(Timeline.older)
+        self.assert_element_absent(Timeline.end)
+        self.assert_element_absent(Timeline.none)
+
+    @parameterized.expand(no_more)
+    def test_no_more(self, username):
+        self.open_nitter(username)
+        self.assert_text('No more items', Timeline.end)
+        self.assert_element_present(Timeline.newest)
+        self.assert_element_absent(Timeline.older)
+
+    @parameterized.expand(empty)
+    def test_empty(self, username):
+        self.open_nitter(username)
+        self.assert_text('No items found', Timeline.none)
+        self.assert_element_absent(Timeline.newest)
+        self.assert_element_absent(Timeline.older)
+        self.assert_element_absent(Timeline.end)
+
+    @parameterized.expand(protected)
+    def test_protected(self, username):
+        self.open_nitter(username)
+        self.assert_text('This account\'s tweets are protected.', Timeline.protected)
+        self.assert_element_absent(Timeline.newest)
+        self.assert_element_absent(Timeline.older)
+        self.assert_element_absent(Timeline.end)
+
+    #@parameterized.expand(photo_rail)
+    #def test_photo_rail(self, username, images):
+        #self.open_nitter(username)
+        #self.assert_element_visible(Timeline.photo_rail)
+        #for i, url in enumerate(images):
+            #img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
+            #self.assertIn(url, img)

tests/test_tweet.py

@@ -0,0 +1,149 @@
+from base import BaseTestCase, Tweet, Conversation, get_timeline_tweet
+from parameterized import parameterized
+
+# image = tweet + 'div.attachments.media-body > div > div > a > div > img'
+# self.assert_true(self.get_image_url(image).split('/')[0] == 'http')
+
+timeline = [
+    [1, 'Test account', 'mobile_test', '10 Aug 2016', '763483571793174528',
+     '.'],
+
+    [3, 'Test account', 'mobile_test', '3 Mar 2016', '705522133443571712',
+     'LIVE on #Periscope pscp.tv/w/aadiTzF6dkVOTXZSbX…'],
+
+    [6, 'mobile test 2', 'mobile_test_2', '1 Oct 2014', '517449200045277184',
+     'Testing. One two three four. Test.']
+]
+
+status = [
+    [20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'],
+    [134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'],
+    [105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'],
+    [572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test']
+]
+
+invalid = [
+    ['mobile_test/status/120938109238'],
+    ['TheTwoffice/status/8931928312']
+]
+
+multiline = [
+    [400897186990284800, 'mobile_test_3',
+     """
+♔
+  KEEP
+ CALM
+   AND
+CLICHÉ
+    ON"""],
+    [1718660434457239868, 'WebDesignMuseum',
+     """
+Happy 32nd Birthday HTML tags! 
+
+On October 29, 1991, the internet pioneer, Tim Berners-Lee, published a document entitled HTML Tags. 
+
+The document contained a description of the first 18 HTML tags: <title>, <nextid>, <a>, <isindex>, <plaintext>, <listing>, <p>, <h1>…<h6>, <address>, <hp1>, <hp2>…, <dl>, <dt>, <dd>, <ul>, <li>,<menu> and <dir>. The design of the first version of HTML language was influenced by the SGML universal markup language.
+
+#WebDesignHistory"""]
+]
+
+link = [
+    ['nim_lang/status/1110499584852353024', [
+        'nim-lang.org/araq/ownedrefs.…',
+        'news.ycombinator.com/item?id…',
+        'teddit.net/r/programming…'
+    ]],
+    ['nim_lang/status/1125887775151140864', [
+        'en.wikipedia.org/wiki/Nim_(p…'
+    ]],
+    ['hiankun_taioan/status/1086916335215341570', [
+        '(hackernoon.com/interview-wit…)'
+    ]],
+    ['archillinks/status/1146302618223951873', [
+        'flickr.com/photos/87101284@N…',
+        'hisafoto.tumblr.com/post/176…'
+    ]],
+    ['archillinks/status/1146292551936335873', [
+        'flickr.com/photos/michaelrye…',
+        'furtho.tumblr.com/post/16618…'
+    ]]
+]
+
+username = [
+    ['Bountysource/status/1094803522053320705', ['nim_lang']],
+    ['leereilly/status/1058464250098704385', ['godotengine', 'unity3d', 'nim_lang']]
+]
+
+emoji = [
+    ['Tesla/status/1134850442511257600', '🌈❤️🧡💛💚💙💜']
+]
+
+retweet = [
+    [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
+    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
+]
+
+
+class TweetTest(BaseTestCase):
+    @parameterized.expand(timeline)
+    def test_timeline(self, index, fullname, username, date, tid, text):
+        self.open_nitter(username)
+        tweet = get_timeline_tweet(index)
+        self.assert_exact_text(fullname, tweet.fullname)
+        self.assert_exact_text('@' + username, tweet.username)
+        self.assert_exact_text(date, tweet.date)
+        self.assert_text(text, tweet.text)
+        permalink = self.find_element(tweet.date + ' a')
+        self.assertIn(tid, permalink.get_attribute('href'))
+
+    @parameterized.expand(status)
+    def test_status(self, tid, fullname, username, date, text):
+        tweet = Tweet()
+        self.open_nitter(f'{username}/status/{tid}')
+        self.assert_exact_text(fullname, tweet.fullname)
+        self.assert_exact_text('@' + username, tweet.username)
+        self.assert_exact_text(date, tweet.date)
+        self.assert_text(text, tweet.text)
+
+    @parameterized.expand(multiline)
+    def test_multiline_formatting(self, tid, username, text):
+        self.open_nitter(f'{username}/status/{tid}')
+        self.assert_text(text.strip('\n'), Conversation.main)
+
+    @parameterized.expand(emoji)
+    def test_emoji(self, tweet, text):
+        self.open_nitter(tweet)
+        self.assert_text(text, Conversation.main)
+
+    @parameterized.expand(link)
+    def test_link(self, tweet, links):
+        self.open_nitter(tweet)
+        for link in links:
+            self.assert_text(link, Conversation.main)
+
+    @parameterized.expand(username)
+    def test_username(self, tweet, usernames):
+        self.open_nitter(tweet)
+        for un in usernames:
+            link = self.find_link_text(f'@{un}')
+            self.assertIn(f'/{un}', link.get_property('href'))
+
+    @parameterized.expand(retweet)
+    def test_retweet(self, index, url, retweet_by, fullname, username, text):
+        self.open_nitter(url)
+        tweet = get_timeline_tweet(index)
+        self.assert_text(f'{retweet_by} retweeted', tweet.retweet)
+        self.assert_text(text, tweet.text)
+        self.assert_exact_text(fullname, tweet.fullname)
+        self.assert_exact_text(username, tweet.username)
+
+    @parameterized.expand(invalid)
+    def test_invalid_id(self, tweet):
+        self.open_nitter(tweet)
+        self.assert_text('Tweet not found', '.error-panel')
+
+    #@parameterized.expand(reply)
+    #def test_thread(self, tweet, num):
+        #self.open_nitter(tweet)
+        #thread = self.find_element(f'.timeline > div:nth-child({num})')
+        #self.assertIn(thread.get_attribute('class'), 'thread-line')

tests/test_tweet_media.py

@@ -0,0 +1,113 @@
+from base import BaseTestCase, Poll, Media
+from parameterized import parameterized
+from selenium.webdriver.common.by import By
+
+poll = [
+    ['nim_lang/status/1064219801499955200', 'Style insensitivity', '91', 1, [
+        ('47%', 'Yay'), ('53%', 'Nay')
+    ]],
+
+    ['polls/status/1031986180622049281', 'What Tree Is Coolest?', '3,322', 1, [
+        ('30%', 'Oak'), ('42%', 'Bonsai'), ('5%', 'Hemlock'), ('23%', 'Apple')
+    ]]
+]
+
+image = [
+    ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
+    #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
+]
+
+gif = [
+    ['elonmusk/status/1141367104702038016', 'D9bzUqoUcAAfUgf'],
+    ['Proj_Borealis/status/1136595194621677568', 'D8X_PJAXUAAavPT']
+]
+
+video_m3u8 = [
+    ['d0m96/status/1078373829917974528', '9q1-v9w8-ft3awgD.jpg'],
+    ['SpaceX/status/1138474014152712192', 'ocJJj2uu4n1kyD2Y.jpg']
+]
+
+gallery = [
+    # ['mobile_test/status/451108446603980803', [
+    #     ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
+    # ]],
+
+    # ['mobile_test/status/471539824713691137', [
+    #     ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
+    #     ['Bos--IqIQAAav23']
+    # ]],
+
+    ['mobile_test/status/469530783384743936', [
+        ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],
+        ['BoQbwarIAAAlaE-', 'BoQbwh_IEAA27ef']
+    ]]
+]
+
+
+class MediaTest(BaseTestCase):
+    @parameterized.expand(poll)
+    def test_poll(self, tweet, text, votes, leader, choices):
+        self.open_nitter(tweet)
+        self.assert_text(text, '.main-tweet')
+        self.assert_text(votes, Poll.votes)
+
+        poll_choices = self.find_elements(Poll.choice)
+        for i, (v, o) in enumerate(choices):
+            choice = poll_choices[i]
+            value = choice.find_element(By.CLASS_NAME, Poll.value)
+            option = choice.find_element(By.CLASS_NAME, Poll.option)
+            choice_class = choice.get_attribute('class')
+
+            self.assert_equal(v, value.text)
+            self.assert_equal(o, option.text)
+
+            if i == leader:
+                self.assertIn(Poll.leader, choice_class)
+            else:
+                self.assertNotIn(Poll.leader, choice_class)
+
+    @parameterized.expand(image)
+    def test_image(self, tweet, url):
+        self.open_nitter(tweet)
+        self.assert_element_visible(Media.container)
+        self.assert_element_visible(Media.image)
+
+        image_url = self.get_image_url(Media.image + ' img')
+        self.assertIn(url, image_url)
+
+    @parameterized.expand(gif)
+    def test_gif(self, tweet, gif_id):
+        self.open_nitter(tweet)
+        self.assert_element_visible(Media.container)
+        self.assert_element_visible(Media.gif)
+
+        url = self.get_attribute('source', 'src')
+        thumb = self.get_attribute('video', 'poster')
+        self.assertIn(gif_id + '.mp4', url)
+        self.assertIn(gif_id + '.jpg', thumb)
+
+    @parameterized.expand(video_m3u8)
+    def test_video_m3u8(self, tweet, thumb):
+        # no url because video playback isn't supported yet
+        self.open_nitter(tweet)
+        self.assert_element_visible(Media.container)
+        self.assert_element_visible(Media.video)
+
+        video_thumb = self.get_attribute(Media.video + ' img', 'src')
+        self.assertIn(thumb, video_thumb)
+
+    @parameterized.expand(gallery)
+    def test_gallery(self, tweet, rows):
+        self.open_nitter(tweet)
+        self.assert_element_visible(Media.container)
+        self.assert_element_visible(Media.row)
+        self.assert_element_visible(Media.image)
+
+        gallery_rows = self.find_elements(Media.row)
+        self.assert_equal(len(rows), len(gallery_rows))
+
+        for i, row in enumerate(gallery_rows):
+            images = row.find_elements(By.CSS_SELECTOR, 'img')
+            self.assert_equal(len(rows[i]), len(images))
+            for j, image in enumerate(images):
+                self.assertIn(rows[i][j], image.get_attribute('src'))

tools/get_session.py

@@ -1,59 +1,56 @@
 #!/usr/bin/env python3
-from sys import argv
 import requests
-import pyotp
 import json
+import sys
+import pyotp
 
-TW_CONSUMER_KEY = "3nVuSoBZnx6U4vzUxf5w"
-TW_CONSUMER_SECRET = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
+# NOTE: pyotp and requests are dependencies
+# > pip install pyotp requests
 
+TW_CONSUMER_KEY = '3nVuSoBZnx6U4vzUxf5w'
+TW_CONSUMER_SECRET = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys'
 
 def auth(username, password, otp_secret):
-    bearer_token_req = requests.post(
-        "https://api.twitter.com/oauth2/token",
+    bearer_token_req = requests.post("https://api.twitter.com/oauth2/token",
         auth=(TW_CONSUMER_KEY, TW_CONSUMER_SECRET),
         headers={"Content-Type": "application/x-www-form-urlencoded"},
-        data="grant_type=client_credentials",
+        data='grant_type=client_credentials'
     ).json()
-    bearer_token = " ".join(str(x) for x in bearer_token_req.values())
+    bearer_token = ' '.join(str(x) for x in bearer_token_req.values())
 
-    guest_token = (
-        requests.post(
-            "https://api.twitter.com/1.1/guest/activate.json",
-            headers={"Authorization": bearer_token},
-        )
-        .json()
-        .get("guest_token")
-    )
+    guest_token = requests.post(
+        "https://api.twitter.com/1.1/guest/activate.json",
+        headers={'Authorization': bearer_token}
+    ).json().get('guest_token')
 
     if not guest_token:
         print("Failed to obtain guest token.")
-        exit(1)
+        sys.exit(1)
 
     twitter_header = {
-        "Authorization": bearer_token,
+        'Authorization': bearer_token,
         "Content-Type": "application/json",
         "User-Agent": "TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)",
-        "X-Twitter-API-Version": "5",
+        "X-Twitter-API-Version": '5',
         "X-Twitter-Client": "TwitterAndroid",
         "X-Twitter-Client-Version": "10.21.0-release.0",
         "OS-Version": "28",
         "System-User-Agent": "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)",
         "X-Twitter-Active-User": "yes",
         "X-Guest-Token": guest_token,
-        "X-Twitter-Client-DeviceID": "",
+        "X-Twitter-Client-DeviceID": ""
     }
 
     session = requests.Session()
     session.headers = twitter_header
 
     task1 = session.post(
-        "https://api.twitter.com/1.1/onboarding/task.json",
+        'https://api.twitter.com/1.1/onboarding/task.json',
         params={
-            "flow_name": "login",
-            "api_version": "1",
-            "known_device_token": "",
-            "sim_country_code": "us",
+            'flow_name': 'login',
+            'api_version': '1',
+            'known_device_token': '',
+            'sim_country_code': 'us'
         },
         json={
             "flow_token": None,
@@ -62,49 +59,50 @@ def auth(username, password, otp_secret):
                 "flow_context": {
                     "referrer_context": {
                         "referral_details": "utm_source=google-play&utm_medium=organic",
-                        "referrer_url": "",
+                        "referrer_url": ""
                     },
-                    "start_location": {"location": "deeplink"},
+                    "start_location": {
+                        "location": "deeplink"
+                    }
                 },
                 "requested_variant": None,
-                "target_user_id": 0,
-            },
-        },
+                "target_user_id": 0
+            }
+        }
     )
 
-    session.headers["att"] = task1.headers.get("att")
+    session.headers['att'] = task1.headers.get('att')
 
     task2 = session.post(
-        "https://api.twitter.com/1.1/onboarding/task.json",
+        'https://api.twitter.com/1.1/onboarding/task.json',
         json={
-            "flow_token": task1.json().get("flow_token"),
-            "subtask_inputs": [
-                {
-                    "enter_text": {
-                        "suggestion_id": None,
-                        "text": username,
-                        "link": "next_link",
-                    },
-                    "subtask_id": "LoginEnterUserIdentifier",
-                }
-            ],
-        },
+            "flow_token": task1.json().get('flow_token'),
+            "subtask_inputs": [{
+                "enter_text": {
+                    "suggestion_id": None,
+                    "text": username,
+                    "link": "next_link"
+                },
+                "subtask_id": "LoginEnterUserIdentifier"
+            }]
+        }
     )
 
     task3 = session.post(
-        "https://api.twitter.com/1.1/onboarding/task.json",
+        'https://api.twitter.com/1.1/onboarding/task.json',
         json={
-            "flow_token": task2.json().get("flow_token"),
-            "subtask_inputs": [
-                {
-                    "enter_password": {"password": password, "link": "next_link"},
-                    "subtask_id": "LoginEnterPassword",
-                }
-            ],
-        },
+            "flow_token": task2.json().get('flow_token'),
+            "subtask_inputs": [{
+                "enter_password": {
+                    "password": password,
+                    "link": "next_link"
+                },
+                "subtask_id": "LoginEnterPassword"
+            }],
+        }
     )
 
-    for t3_subtask in task3.json().get("subtasks", []):
+    for t3_subtask in task3.json().get('subtasks', []):
         if "open_account" in t3_subtask:
             return t3_subtask["open_account"]
         elif "enter_text" in t3_subtask:
@@ -125,7 +123,7 @@ def auth(username, password, otp_secret):
                             "subtask_id": "LoginTwoFactorAuthChallenge",
                         }
                     ],
-                },
+                }
             )
             task4 = task4resp.json()
             for t4_subtask in task4.get("subtasks", []):
@@ -134,26 +132,21 @@ def auth(username, password, otp_secret):
 
     return None
 
-
 if __name__ == "__main__":
-    if len(argv) != 4:
-        print("Usage: %s <username> <password> <2fa secret>" % argv[0])
-        exit(1)
+    if len(sys.argv) != 4:
+        print("Usage: %s <username> <password> <2fa secret>" % sys.argv[0])
+        sys.exit(1)
 
-    username = argv[1]
-    password = argv[2]
-    otp_secret = argv[3]
+    username = sys.argv[1]
+    password = sys.argv[2]
+    otp_secret = sys.argv[3]
 
     result = auth(username, password, otp_secret)
     if result is None:
         print("Authentication failed.")
-        exit(1)
+        sys.exit(1)
 
-    print(
-        json.dumps(
-            {
-                "oauth_token": result.get("oauth_token"),
-                "oauth_token_secret": result.get("oauth_token_secret"),
-            }
-        )
-    )
+    print(json.dumps({
+        "oauth_token": result.get("oauth_token"),
+        "oauth_token_secret": result.get("oauth_token_secret"),
+    }, indent=2))

tools/rendermd.nim

@@ -1,10 +0,0 @@
-import std/[os, strutils]
-import markdown
-
-for file in walkFiles("public/md/*.md"):
-  let
-    html = markdown(readFile(file))
-    output = file.replace(".md", ".html")
-
-  output.writeFile(html)
-  echo "Rendered ", output

ups.json

@@ -1,5 +1,5 @@
 {
-  "upstream": "https://github.com/PrivacyDevel/nitter",
+  "upstream": "https://github.com/zedeus/nitter",
   "provider": "github",
-  "commit": "7350155d2d3be85e4c60ad39e744f9125505a668"
+  "commit": "e40c61a6ae76431c570951cc4925f38523b00a82"
 }