include_once 'vendor/autoload.php'; use \Curl\Curl; use Symfony\Component\DomCrawler\Crawler; $config = include_once __DIR__."/config.php"; $container = new \Slim\Container($config); $container['cache_key'] = function ($container) { return md5($container["request"]->getUri()->getHost().$container["request"]->getUri()->getPath()); }; $container['replace'] = function ($container) { return explode(PHP_EOL, file_get_contents(__DIR__.'/replace.txt')); }; $container['host'] = function ($container) { return str_replace('www.', '', $_SERVER['HTTP_HOST']); }; $container['cache'] = $container->factory( function ($c) { $cache = null; $type = $c->settings['cache']['type']; switch ($type) { /* * Array */ case 'array_cache': $cache = new \Doctrine\Common\Cache\ArrayCache(); break; /* * APCu */ case 'apcu_cache': $cache = new \Doctrine\Common\Cache\ApcuCache(); break; /* * Xcache */ case 'xcache_cache': $cache = new \Doctrine\Common\Cache\XcacheCache(); break; /* * File */ case 'file_cache': $cache = new \Doctrine\Common\Cache\FilesystemCache($c->settings['cache']['file_cache']); break; /* * SQLite3 */ case 'sqlite_cache': $cache = new \Doctrine\Common\Cache\SQLite3Cache(new SQLite3($c->settings['cache']['sqlite_cache']), 'cache'); break; /* * Memcache */ case 'memcache_cache': $memcache = new Memcache(); $memcache->connect($c->settings['cache']['memcache_cache']['host'], $c->settings['cache']['memcache_cache']['port']); $cache = new \Doctrine\Common\Cache\MemcacheCache(); $cache->setMemcache($memcache); break; /* * Memcached */ case 'memcached_cache': $memcached = new Memcached(); $memcached->addServer($c->settings['cache']['memcached_cache']['host'], $c->settings['cache']['memcached_cache']['port']); $cache = new \Doctrine\Common\Cache\MemcachedCache(); $cache->setMemcached($memcached); break; /* * Redis */ case 'redis_cache': $redis = new Redis(); $redis->connect($c->settings['cache']['redis_cache']['host'], $c->settings['cache']['redis_cache']['port']); $cache = new \Doctrine\Common\Cache\RedisCache(); $cache->setRedis($redis); break; /* * MongoDB */ case 'mongo_cache': $mongo = new Mongo("mongodb://{$c->settings['cache']['mongo_cache']['username']}:{$c->settings['cache']['mongo_cache']['password']}@{$c->settings['cache']['mongo_cache']['host']}:{$c->settings['cache']['mongo_cache']['port']}/{$c->settings['cache']['mongo_cache']['dbname']}"); $collection = $mongo->$c->settings['cache']['mongo_cache']['dbname']->$mongo->$c->settings['cache']['mongo_cache']['collection']; $cache = new \Doctrine\Common\Cache\MongoDBCache($collection); break; /* * File */ default: $cache = new \Doctrine\Common\Cache\FilesystemCache($c->settings['cache']['file_cache']); } return $cache; } ); $container['notFoundHandler'] = function ($container) { return function ($request, $response) use ($container) { return $response ->withStatus(404) ->withHeader('Content-Type', 'text/html') ->write('Нет такой страницы'); }; }; $app = new \Slim\App($container); $trustedIpMw = function ($request, $response, $next) { $flags = FILTER_FLAG_IPV4 | FILTER_FLAG_IPV6; $serverParams = $request->getServerParams(); $settings = $this->get('settings'); if ( isset($serverParams['REMOTE_ADDR']) && filter_var($serverParams['REMOTE_ADDR'], FILTER_VALIDATE_IP, $flags)) { $ipAddress = $serverParams['REMOTE_ADDR']; } if(isset($ipAddress) && in_array($ipAddress, $settings['trustedIp'])){ return $next($request, $response); }else{ return $response->withStatus(403) ->withHeader('Content-Type', 'text/xml') ->write('You are forbidden'); } }; $app->get('/msp_cache_delete/{cache}', function ($request, $response, $args) { $links_file = __DIR__.'/data/'.$args['cache'].'-links.txt'; $host = $args['cache']; if(!empty($host) && file_exists($links_file)){ try { if(!is_writable($links_file)) throw new Exception('File not writable'); unlink($links_file); } catch(Exception $e) { die($e); } } if($this->cache->contains($host.'-proxy')){ $proxy_host = $this->cache->fetch($host.'-proxy'); $this->cache->delete($host.'-proxy'); } if(isset($proxy_host) && $this->cache->contains($proxy_host.'-baseurl')){ $this->cache->delete($proxy_host.'-baseurl'); } return $response->withRedirect('/'); })->setName('msp_cache_delete')->add($trustedIpMw); $app->get('/list.txt', function ($request, $response) { $host = $this->host; $keywords = explode(PHP_EOL, file_get_contents(__DIR__.'/data/'.$host.'-links.txt')); foreach (array_unique($keywords) as $keyword) { echo $keyword; echo "
"; } })->setName('list'); $app->get('/yandex_{id}.html', function ($request, $response, $args) { ob_start(); include __DIR__.'/templates/yandex.php'; $template = ob_get_clean(); return $response->withStatus(200) ->withHeader('Content-Type', 'text/html') ->write($template); })->setName('yandex'); $app->get('/google{id}.html', function ($request, $response, $args) { ob_start(); include __DIR__.'/templates/google.php'; $template = ob_get_clean(); return $response->withStatus(200) ->withHeader('Content-Type', 'text/html') ->write($template); })->setName('google'); $app->get('/[{keyword:.*}]', function ($request, $response, $args) { $url_request_url = $_SERVER['REQUEST_URI']; $lifeTime = $this->settings['lifeTime']; $host = $this->host; if($this->cache->contains($host.'-proxy')){ $proxy_host = $this->cache->fetch($host.'-proxy'); }else{ $proxies = explode(PHP_EOL, file_get_contents(__DIR__.'/proxy.txt')); shuffle($proxies); $proxy_host = trim($proxies[0]); $this->cache->save($host.'-proxy', $proxy_host); } if($this->cache->contains($proxy_host.'-baseurl')){ $proxy_absolute_url = $this->cache->fetch($proxy_host.'-baseurl'); }else{ $proxy_absolute_url = $proxy_host; } $proxy_domain = parse_url($proxy_absolute_url, PHP_URL_HOST); if($this->cache->contains($proxy_host.$url_request_url.'-page')){ $html = $this->cache->fetch($proxy_host.$url_request_url.'-page'); }else{ $curl = new Curl(); $curl->setCookieFile(__DIR__.'/cookie.txt'); $curl->setOpt(CURLOPT_USERAGENT, $this->settings['userAgent']); $curl->setOpt(CURLOPT_REFERER, $proxy_host); $curl->setOpt(CURLOPT_FOLLOWLOCATION, true); $curl->setOpt(CURLOPT_SSL_VERIFYPEER, false); $curl->setOpt(CURLOPT_SSL_VERIFYHOST, false); $curl->setOpt(CURLOPT_RETURNTRANSFER, true); $curl->setOpt(CURLOPT_HEADER, true); /** * Настройки для граббинга через прокси */ $proxies = $this->settings['proxies']; if($proxies['isEnable']) { if(endsWith($proxies['proxyPath'], '.txt') and file_exists($proxies['proxyPath'])){ $proxyList = explode(PHP_EOL, file_get_contents($proxies['proxyPath'])); shuffle($proxyList); $currentProxy = $proxyList[0]; }else{ $currentProxy = $proxies['proxyPath']; } $curl->setOpt( CURLOPT_PROXY, $currentProxy); $curl->setOpt( CURLOPT_PROXYTYPE, $proxies['proxyType']); } $curl->get($proxy_host.$url_request_url); if ($curl->error) { return $response ->withStatus(503) ->withHeader('Content-Type', 'text/html') ->write('Service Unavailable'); } else { preg_match('#Location: (.*)#', $curl->response, $redirect_url); if(isset($redirect_url[1])) { $proxy_absolute_url = str_replace('index.php', '', trim($redirect_url[1])); $this->cache->save($proxy_host.'-baseurl',$proxy_absolute_url); } } $html = substr($curl->response, $curl->getInfo(CURLINFO_HEADER_SIZE)); $this->cache->save($proxy_host.$url_request_url.'-page', $html, $lifeTime); } /** * Определяет тип контента */ if (endsWith($proxy_host.$url_request_url, '.txt')) { $html = str_replace($proxy_domain, $host, $html); $contentType = 'text/plain'; }elseif(endsWith($proxy_host.$url_request_url, '.xml')){ $html = str_replace($proxy_domain, $host, $html); $contentType = 'text/xml'; }else{ $contentType = 'text/html'; $crawler = new Crawler($html, $proxy_host); $links_file = __DIR__.'/data/'.$host.'-links.txt'; $links_file_exists = false; $file = null; if(!file_exists($links_file)){ $file = new SplFileObject($links_file, 'w+'); $links_file_exists = true; } $crawler->filter('a')->each(function ($node, $i) use ($proxy_absolute_url, $proxy_domain, $host, $file, $links_file_exists) { $url = str_replace($proxy_domain, $host, convertRelativeToAbsoluteURL($node->attr('href'),$proxy_absolute_url)); $url = str_replace('www.'.$host, $host, $url); $url = str_replace('https://', 'http://', $url); if($links_file_exists ) { $file->fwrite($url.PHP_EOL); } $node->getNode(0)->setAttribute('href', $url); }); $crawler->filter('img')->each(function ($node, $i) use ($proxy_host ) { if (strpos($node->attr('src'), '//') === false) { $node->getNode(0)->setAttribute('src', $proxy_host .$node->attr('src')); } }); $crawler->filter('link')->each(function ($node, $i) use ($proxy_host ) { if (strpos($node->attr('href'), '//') === false) { $node->getNode(0)->setAttribute('href', $proxy_host.$node->attr('href')); } }); $crawler->filter('script')->each(function ($node, $i) use ($proxy_host) { if (strpos($node->attr('src'), '//') === false) { $node->getNode(0)->setAttribute('src', $proxy_host.$node->attr('src')); } }); $title = $crawler->filterXPath('//title')->text(); $html = $crawler->html(); } if($this->settings['convertCharset']){ $html = iconv(mb_detect_encoding($html, mb_detect_order(), true), "UTF-8", $html ); } if($this->settings['isRedirect']){ if(!empty($title)) { $redirect_url = str_replace('{keyword}', urlencode($title), $this->settings['redirectUrl']); return $response->withRedirect($redirect_url); }else{ return $response->withRedirect($this->settings['redirectUrlReserve']); } } if($this->settings['isWritingTitle'] && file_exists($this->settings['writingTitleFile']) && is_writable($this->settings['writingTitleFile'])){ if(!empty($title)) { file_put_contents($this->settings['writingTitleFile'], $title.PHP_EOL, FILE_APPEND); } } foreach ($this->replace as $replaceLine) { list($search, $replace) = explode("|", trim($replaceLine)); $html = str_replace($search, $replace, $html); } require(__DIR__.'/replace.php'); return $response->withStatus(200) ->withHeader('Content-Type', $contentType) ->write($html); })->setName('copy'); $app->run();