diff --git a/code/nel/include/nel/gui/http_cache.h b/code/nel/include/nel/gui/http_cache.h new file mode 100644 index 000000000..0921b2f64 --- /dev/null +++ b/code/nel/include/nel/gui/http_cache.h @@ -0,0 +1,77 @@ +// Ryzom - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef CL_HTTP_CACHE_H +#define CL_HTTP_CACHE_H + +#include "nel/misc/types_nl.h" + +namespace NLGUI +{ + struct CHttpCacheObject + { + CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "") + : Expires(expires) + , LastModified(lastModified) + , Etag(etag){}; + + uint32 Expires; + std::string LastModified; + std::string Etag; + + void serial(NLMISC::IStream& f); + }; + + /** + * Keeping track of downloaded files cache related headers + * \author Meelis Mägi (nimetu) + * \date 2017 + */ + class CHttpCache + { + typedef std::map THttpCacheMap; + + public: + static CHttpCache* getInstance(); + static void release(); + + public: + void setCacheIndex(const std::string& fname); + void init(); + + CHttpCacheObject lookup(const std::string& fname); + void store(const std::string& fname, const CHttpCacheObject& data); + + void flushCache(); + + void serial(NLMISC::IStream& f); + + private: + CHttpCache(); + ~CHttpCache(); + + void pruneCache(); + + static CHttpCache* instance; + + THttpCacheMap _List; + + std::string _IndexFilename; + bool _Initialized; + size_t _MaxObjects; + }; +} +#endif diff --git a/code/nel/src/gui/group_html.cpp b/code/nel/src/gui/group_html.cpp index acc33357a..0c1fba8d0 100644 --- a/code/nel/src/gui/group_html.cpp +++ b/code/nel/src/gui/group_html.cpp @@ -45,6 +45,7 @@ #include "nel/3d/texture_file.h" #include "nel/misc/big_file.h" #include "nel/gui/url_parser.h" +#include "nel/gui/http_cache.h" using namespace std; using namespace NLMISC; @@ -87,6 +88,15 @@ namespace NLGUI curl_slist_free_all(HeadersSent); } + void sendHeaders(const std::vector headers) + { + for(uint i = 0; i < headers.size(); ++i) + { + HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str()); + } + curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent); + } + void setRecvHeader(const std::string &header) { size_t pos = header.find(": "); @@ -110,12 +120,42 @@ namespace NLGUI return ""; } + const uint32 getExpires() + { + time_t ret = 0; + if (HeadersRecv.count("expires") > 0) + ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL); + + return ret > -1 ? ret : 0; + } + + const std::string getLastModified() + { + if (HeadersRecv.count("last-modified") > 0) + { + return HeadersRecv["last-modified"]; + } + + return ""; + } + + const std::string getEtag() + { + if (HeadersRecv.count("etag") > 0) + { + return HeadersRecv["etag"]; + } + + return ""; + } + public: CURL *Request; std::string Url; std::string Content; + private: // headers sent with curl request, must be released after transfer curl_slist * HeadersSent; @@ -274,19 +314,16 @@ namespace NLGUI return false; } - // TODO: replace with expire and etag headers - if (CFile::fileExists(download.dest)) + time_t currentTime; + time(¤tTime); + + CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest); + if (cache.Expires > currentTime) { - time_t currentTime; - time(¤tTime); - uint32 mtime = CFile::getFileModificationDate(download.dest); - if (mtime + 3600 > currentTime) - { #ifdef LOG_DL - nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime); + nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime); #endif - return false; - } + return false; } string tmpdest = download.dest + ".tmp"; @@ -322,6 +359,16 @@ namespace NLGUI curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + std::vector headers; + if (!cache.Etag.empty()) + headers.push_back("If-None-Match: " + cache.Etag); + + if (!cache.LastModified.empty()) + headers.push_back("If-Modified-Since: " + cache.LastModified); + + if (headers.size() > 0) + download.data->sendHeaders(headers); + // catch headers curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback); curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data); @@ -601,9 +648,27 @@ namespace NLGUI if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString()))) { NLMISC::CFile::deleteFile(tmpfile.c_str()); + + // 304 Not Modified + if (res == CURLE_OK && r == 304) + { + CHttpCacheObject obj; + obj.Expires = it->data->getExpires(); + obj.Etag = it->data->getEtag(); + obj.LastModified = it->data->getLastModified(); + + CHttpCache::getInstance()->store(it->dest, obj); + } } else { + CHttpCacheObject obj; + obj.Expires = it->data->getExpires(); + obj.Etag = it->data->getEtag(); + obj.LastModified = it->data->getLastModified(); + + CHttpCache::getInstance()->store(it->dest, obj); + string finalUrl; if (it->type == ImgType) { @@ -5179,11 +5244,7 @@ namespace NLGUI std::vector headers; headers.push_back("Accept-Language: "+options.languageCode); headers.push_back("Accept-Charset: utf-8"); - for(uint i=0; i< headers.size(); ++i) - { - _CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str()); - } - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent); + _CurlWWW->sendHeaders(headers); // catch headers for redirect curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback); diff --git a/code/nel/src/gui/http_cache.cpp b/code/nel/src/gui/http_cache.cpp new file mode 100644 index 000000000..f0839752c --- /dev/null +++ b/code/nel/src/gui/http_cache.cpp @@ -0,0 +1,200 @@ +// Ryzom - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdpch.h" +#include "nel/gui/http_cache.h" + +using namespace std; +using namespace NLMISC; + +#ifdef DEBUG_NEW +#define new DEBUG_NEW +#endif + +namespace NLGUI +{ + CHttpCache* CHttpCache::instance = NULL; + + CHttpCache* CHttpCache::getInstance() + { + if (!instance) + { + instance = new CHttpCache(); + } + + return instance; + } + + void CHttpCache::release() + { + delete instance; + instance = NULL; + } + + CHttpCache::CHttpCache() + : _Initialized(false) + , _MaxObjects(100) + { }; + + CHttpCache::~CHttpCache() + { + flushCache(); + } + + void CHttpCache::setCacheIndex(const std::string& fname) + { + _IndexFilename = fname; + _Initialized = false; + } + + CHttpCacheObject CHttpCache::lookup(const std::string& fname) + { + if (!_Initialized) + init(); + + if (_List.count(fname) > 0) + return _List[fname]; + + return CHttpCacheObject(); + } + + void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data) + { + if (!_Initialized) + init(); + + _List[fname] = data; + } + + void CHttpCache::init() + { + if (_Initialized) + return; + + _Initialized = true; + + if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename)) + return; + + CIFile in; + if (!in.open(_IndexFilename)) { + nlwarning("Unable to open %s for reading", _IndexFilename.c_str()); + return; + } + + serial(in); + } + + void CHttpCacheObject::serial(NLMISC::IStream& f) + { + f.serialVersion(1); + f.serial(Expires); + f.serial(LastModified); + f.serial(Etag); + } + + void CHttpCache::serial(NLMISC::IStream& f) + { + // saved state is ignored when version checks fail + try { + f.serialVersion(1); + + // CacheIdx + f.serialCheck(NELID("hcaC")); + f.serialCheck(NELID("xdIe")); + + if (f.isReading()) + { + uint32 numFiles; + f.serial(numFiles); + + _List.clear(); + for (uint k = 0; k < numFiles; ++k) + { + std::string fname; + f.serial(fname); + + CHttpCacheObject obj; + obj.serial(f); + + _List[fname] = obj; + } + } + else + { + uint32 numFiles = _List.size(); + f.serial(numFiles); + + for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it) + { + std::string fname(it->first); + f.serial(fname); + + (*it).second.serial(f); + } + } + } catch (...) { + _List.clear(); + nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str()); + return; + } + } + + void CHttpCache::pruneCache() + { + if (_List.size() < _MaxObjects) + return; + + size_t mustDrop = _List.size() - _MaxObjects; + + time_t currentTime; + time(¤tTime); + + // if we over object limit, then start removing expired objects + // this does not guarantee that max limit is reached + for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) { + if (it->second.Expires <= currentTime) { + it = _List.erase(it); + + --mustDrop; + if (mustDrop == 0) + break; + } + else + { + ++it; + } + } + } + + void CHttpCache::flushCache() + { + if (_IndexFilename.empty()) + return; + + pruneCache(); + + COFile out; + if (!out.open(_IndexFilename)) + { + nlwarning("Unable to open %s for writing", _IndexFilename.c_str()); + return; + } + + serial(out); + out.close(); + } +} diff --git a/code/nel/src/gui/stdpch.h b/code/nel/src/gui/stdpch.h index e0be5837e..a4ba0ecac 100644 --- a/code/nel/src/gui/stdpch.h +++ b/code/nel/src/gui/stdpch.h @@ -26,6 +26,7 @@ #include #include +#include #include "nel/misc/types_nl.h" #include "nel/misc/algo.h" diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index 267042b7e..900f3c2c1 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -68,6 +68,7 @@ #include "interface_v3/sbrick_manager.h" #include "nel/gui/widget_manager.h" +#include "nel/gui/http_cache.h" // #include "gabarit.h" #include "hair_set.h" @@ -1365,6 +1366,10 @@ void prelogInit() //nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000); } + // Initialize HTTP cache + CHttpCache::getInstance()->setCacheIndex("cache/cache.index"); + CHttpCache::getInstance()->init(); + // Register the reflected classes registerInterfaceElements(); diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index 689f0bf56..a1f3a29c2 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -94,7 +94,7 @@ #include "bg_downloader_access.h" #include "nel/gui/lua_manager.h" #include "item_group_manager.h" - +#include "nel/gui/http_cache.h" /////////// // USING // @@ -687,6 +687,7 @@ void release() CWidgetManager::release(); CViewRenderer::release(); CIXml::releaseLibXml(); + CHttpCache::release(); #if FINAL_VERSION // openURL ("http://www.ryzomcore.org/exit/");