From af3c43dbf542117c248b096b33d19afeeb19b2f3 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Tue, 2 Nov 2021 15:30:25 +0530 Subject: [PATCH] Cache post data using sled Each post fetch was taking 800ms TAT, so I'm using sled to fetch and cache post data. This reduced TAT down to 2ms. However, this could cause storage issues. I must design some sort of resource manager to clean up cache. --- .gitignore | 1 + Cargo.lock | 78 +++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 ++ sailfish.yml | 2 ++ src/data.rs | 37 +++++++++++++++++++++ src/proxy.rs | 28 ++++------------ templates/p.html | 3 -- templates/post.html | 5 --- 8 files changed, 127 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index ea8c4bf..6c19a0e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +posts_cache diff --git a/Cargo.lock b/Cargo.lock index c51139e..57c141e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -304,6 +304,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde 1.0.130", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -480,6 +489,29 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if", + "lazy_static", +] + [[package]] name = "derive_more" version = "0.99.16" @@ -619,6 +651,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures-channel" version = "0.3.17" @@ -660,6 +702,15 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -983,6 +1034,7 @@ dependencies = [ "actix-rt", "actix-web", "actix-web-codegen 0.5.0-beta.4", + "bincode", "chrono", "config", "derive_more", @@ -995,6 +1047,7 @@ dependencies = [ "sailfish", "serde 1.0.130", "serde_json", + "sled", "url", ] @@ -1052,6 +1105,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -1700,6 +1762,22 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot", +] + [[package]] name = "smallvec" version = "1.7.0" diff --git a/Cargo.toml b/Cargo.toml index 5cfbb28..5a03bec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,3 +38,6 @@ reqwest = { version = "0.11.6", features = ["json"] } graphql_client = { version = "0.10.0", features = ["reqwest"]} chrono = "0.4.19" + +sled = "0.34.7" +bincode = "1.3.3" diff --git a/sailfish.yml b/sailfish.yml index 5f64209..0d394a3 100644 --- a/sailfish.yml +++ b/sailfish.yml @@ -1 +1,3 @@ delimiter: "." +optimization: + rm_whitespace: true diff --git a/src/data.rs b/src/data.rs index 2eef40b..83dcc27 100644 --- a/src/data.rs +++ b/src/data.rs @@ -15,19 +15,56 @@ * along with this program. If not, see . */ use actix_web::web; +use graphql_client::{reqwest::post_graphql, GraphQLQuery}; use reqwest::Client; +use sled::{Db, Tree}; #[derive(Clone)] pub struct Data { pub client: Client, + cache: Db, + pub posts: Tree, } +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "schemas/schema.graphql", + query_path = "schemas/query.graphql", + response_derives = "Debug, Serialize, Deserialize, Clone" +)] +pub struct GetPost; + +pub type PostResp = get_post::GetPostPost; + pub type AppData = web::Data; impl Data { pub fn new() -> AppData { + let cache = sled::open("posts_cache").unwrap(); + let posts = cache.open_tree("posts").unwrap(); AppData::new(Self { client: Client::new(), + cache, + posts, }) } + + pub async fn get_post(&self, id: &str) -> PostResp { + match self.posts.get(id) { + Ok(Some(v)) => bincode::deserialize(&v[..]).unwrap(), + _ => { + let vars = get_post::Variables { id: id.to_owned() }; + const URL: &str = "https://medium.com/_/graphql"; + + let res = post_graphql::(&self.client, URL, vars) + .await + .unwrap(); + let res = res.data.expect("missing response data").post.unwrap(); + self.posts + .insert(id, bincode::serialize(&res).unwrap()) + .unwrap(); + res + } + } + } } diff --git a/src/proxy.rs b/src/proxy.rs index ab047b2..0ba73d2 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -17,9 +17,9 @@ use std::ops::{Bound, RangeBounds}; use actix_web::{web, HttpResponse, Responder}; -use graphql_client::{reqwest::post_graphql, GraphQLQuery}; use sailfish::TemplateOnce; +use crate::data::PostResp; use crate::AppData; pub mod routes { @@ -95,18 +95,11 @@ impl StringUtils for str { } } -#[derive(GraphQLQuery)] -#[graphql( - schema_path = "schemas/schema.graphql", - query_path = "schemas/query.graphql", - response_derives = "Debug" -)] -struct GetPost; - #[derive(TemplateOnce)] #[template(path = "post.html")] +#[template(rm_whitespace = true)] pub struct Post { - pub data: get_post::GetPostPost, + pub data: PostResp, pub id: String, } @@ -125,20 +118,11 @@ async fn page(path: web::Path<(String, String)>, data: AppData) -> impl Responde if post_id.is_none() { return HttpResponse::BadRequest().finish(); } - let id = post_id.unwrap().to_string(); - - let vars = get_post::Variables { id: id.clone() }; - - const URL: &str = "https://medium.com/_/graphql"; - - let res = post_graphql::(&data.client, URL, vars) - .await - .unwrap(); - let response_data: get_post::ResponseData = res.data.expect("missing response data"); + let id = post_id.unwrap(); let page = Post { - id, - data: response_data.post.unwrap(), + id: id.to_owned(), + data: data.get_post(&id).await, } .render_once() .unwrap(); diff --git a/templates/p.html b/templates/p.html index 3e4db31..87fd397 100644 --- a/templates/p.html +++ b/templates/p.html @@ -34,11 +34,8 @@ <.= text .> <.}.> <.}.> - <. if cur < p.text.len() {.> <.= p.text.slice(cur..) .> <.}.> - <.}.>

- diff --git a/templates/post.html b/templates/post.html index 2198b9d..ad6f9f3 100644 --- a/templates/post.html +++ b/templates/post.html @@ -10,7 +10,6 @@ <. use chrono::{TimeZone, Utc}; .> <. let dt = Utc.timestamp_millis(data.created_at); .> <. let date = dt.format("%b %e, %Y").to_string(); .> -

<.= data.title .>

@@ -29,8 +28,6 @@ <. if pindex == 0 && p.type_ == "H3" {.> <. continue; .> <.}.> - - <. if p.type_ == "IMG" {.> <. include!("./img.html"); .> <.} else if p.type_ == "P" {.> @@ -46,8 +43,6 @@ <.} else if p.type_ == "H6" {.>

<.= p.text .>
<.}.> - - <.}.>