diff --git a/.gitignore b/.gitignore index ed4ff25..10de716 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /compose.yml /docker-compose.yml -/banners -/captcha +/banner +/favicon.ico /config.php diff --git a/compose.example.yml b/compose.example.yml index 92e5cd2..c4eebb2 100644 --- a/compose.example.yml +++ b/compose.example.yml @@ -6,8 +6,8 @@ services: - 80:8080 volumes: - ./config.php:/srv/4get/data/config.php:ro - - ./banners:/srv/4get/banner:ro - - ./captcha:/srv/4get/data/captcha:ro + - ./favicon.ico:/srv/4get/favicon.ico:ro + - ./banner:/srv/4get/banner:ro - type: tmpfs target: /tmp/icons cap_drop: diff --git a/src/.gitignore b/src/.gitignore index c4c987c..270c426 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -21,9 +21,8 @@ scraper/soundcloud.json scraper/mp3-pm.html scraper/curlie.html -icons/* banner/* -!banner/*default* +data/* -data/captcha -data/config.php +!banner/*default* +!data/*.def.* diff --git a/src/data/config.def.php b/src/data/config.def.php index b502248..7eb82b4 100644 --- a/src/data/config.def.php +++ b/src/data/config.def.php @@ -37,8 +37,8 @@ class config{ // Proxy pool assignments for each scraper // false = Use server's raw IP - // string = will load a proxy list from data/proxies - // Eg. "onion" will load data/proxies/onion.txt + // string = will load a proxy list from data directory + // Eg. "tor" will load data/tor.txt const PROXY_DDG = false; // duckduckgo const PROXY_BRAVE = false; const PROXY_FB = false; // facebook diff --git a/src/data/fonts/captcha.ttf b/src/data/fonts/captcha.ttf deleted file mode 100644 index 13f5dc7..0000000 Binary files a/src/data/fonts/captcha.ttf and /dev/null differ diff --git a/src/data/proxies/.gitignore b/src/data/proxies/.gitignore deleted file mode 100644 index 68f8e48..0000000 --- a/src/data/proxies/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore -!onion.txt diff --git a/src/data/proxies/onion.txt b/src/data/tor.def.txt similarity index 100% rename from src/data/proxies/onion.txt rename to src/data/tor.def.txt diff --git a/src/icons/lolcat.ca.png b/src/icons/lolcat.ca.png deleted file mode 100644 index bf47a44..0000000 Binary files a/src/icons/lolcat.ca.png and /dev/null differ diff --git a/src/lib/backend.php b/src/lib/backend.php index 66e78a1..51c3a47 100644 --- a/src/lib/backend.php +++ b/src/lib/backend.php @@ -1,27 +1,27 @@ scraper = $scraper; } - + /* Proxy stuff */ public function get_ip(){ - + $pool = constant("config::PROXY_" . strtoupper($this->scraper)); if($pool === false){ - + // we don't want a proxy, fuck off! return 'raw_ip::::'; } - + // indent $proxy_index_raw = apcu_inc("p." . $this->scraper); - - $proxylist = file_get_contents("data/proxies/" . $pool . ".txt"); + + $proxylist = file_get_contents("data/" . $pool . ".txt"); $proxylist = explode("\n", $proxylist); // ignore empty or commented lines @@ -29,15 +29,15 @@ class backend{ $entry = ltrim($entry); return strlen($entry) > 0 && substr($entry, 0, 1) != "#"; }); - + $proxylist = array_values($proxylist); - + return $proxylist[$proxy_index_raw % count($proxylist)]; } - + // this function is also called directly on nextpage public function assign_proxy(&$curlproc, string $ip){ - + // parse proxy line [ $type, @@ -46,34 +46,34 @@ class backend{ $username, $password ] = explode(":", $ip, 5); - + switch($type){ - + case "raw_ip": return; break; - + case "http": case "https": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); curl_setopt($curlproc, CURLOPT_PROXY, $type . "://" . $address . ":" . $port); break; - + case "socks4": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4); curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; - + case "socks5": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; - + case "socks4a": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A); curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; - + case "socks5_hostname": case "socks5h": case "socks5a": @@ -81,25 +81,25 @@ class backend{ curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; } - + if($username != ""){ - + curl_setopt($curlproc, CURLOPT_PROXYUSERPWD, $username . ":" . $password); } } - - - + + + /* Next page stuff */ public function store(string $payload, string $page, string $proxy){ - + $key = sodium_crypto_secretbox_keygen(); $nonce = random_bytes(SODIUM_CRYPTO_SECRETBOX_NONCEBYTES); - + $requestid = apcu_inc("requestid"); - + apcu_store( $page[0] . "." . // first letter of page name $this->scraper . // scraper name @@ -117,31 +117,31 @@ class backend{ 900 // cache information for 15 minutes ); - return + return $this->scraper . $requestid . "." . rtrim(strtr(base64_encode($key), '+/', '-_'), '='); } - + public function get(string $npt, string $page){ - + $page = $page[0]; $explode = explode(".", $npt, 2); - + if(count($explode) !== 2){ - + throw new Exception("Malformed nextPageToken!"); } - + $apcu = $page . "." . $explode[0]; $key = $explode[1]; - + $payload = apcu_fetch($apcu); - + if($payload === false){ - + throw new Exception("The next page token is invalid or has expired!"); } - + $key = base64_decode( str_pad( @@ -151,7 +151,7 @@ class backend{ STR_PAD_RIGHT ) ); - + // decrypt and decompress data $payload[2] = gzinflate( @@ -161,15 +161,15 @@ class backend{ $key ) ); - + if($payload[2] === false){ - + throw new Exception("The next page token is invalid or has expired!"); } - + // remove the key after using successfully apcu_delete($apcu); - + return [ $payload[2], // data $payload[1] // proxy