Changed: Use http cache headers for image cache

--HG--
branch : develop
This commit is contained in:
Nimetu 2017-04-20 21:19:51 +03:00
parent 3d09dd2094
commit 108759952e
6 changed files with 361 additions and 16 deletions

View file

@ -0,0 +1,77 @@
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef CL_HTTP_CACHE_H
#define CL_HTTP_CACHE_H
#include "nel/misc/types_nl.h"
namespace NLGUI
{
struct CHttpCacheObject
{
CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "")
: Expires(expires)
, LastModified(lastModified)
, Etag(etag){};
uint32 Expires;
std::string LastModified;
std::string Etag;
void serial(NLMISC::IStream& f);
};
/**
* Keeping track of downloaded files cache related headers
* \author Meelis Mägi (nimetu)
* \date 2017
*/
class CHttpCache
{
typedef std::map<std::string, CHttpCacheObject> THttpCacheMap;
public:
static CHttpCache* getInstance();
static void release();
public:
void setCacheIndex(const std::string& fname);
void init();
CHttpCacheObject lookup(const std::string& fname);
void store(const std::string& fname, const CHttpCacheObject& data);
void flushCache();
void serial(NLMISC::IStream& f);
private:
CHttpCache();
~CHttpCache();
void pruneCache();
static CHttpCache* instance;
THttpCacheMap _List;
std::string _IndexFilename;
bool _Initialized;
size_t _MaxObjects;
};
}
#endif

View file

@ -45,6 +45,7 @@
#include "nel/3d/texture_file.h"
#include "nel/misc/big_file.h"
#include "nel/gui/url_parser.h"
#include "nel/gui/http_cache.h"
using namespace std;
using namespace NLMISC;
@ -87,6 +88,15 @@ namespace NLGUI
curl_slist_free_all(HeadersSent);
}
void sendHeaders(const std::vector<std::string> headers)
{
for(uint i = 0; i < headers.size(); ++i)
{
HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str());
}
curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent);
}
void setRecvHeader(const std::string &header)
{
size_t pos = header.find(": ");
@ -110,12 +120,42 @@ namespace NLGUI
return "";
}
const uint32 getExpires()
{
time_t ret = 0;
if (HeadersRecv.count("expires") > 0)
ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL);
return ret > -1 ? ret : 0;
}
const std::string getLastModified()
{
if (HeadersRecv.count("last-modified") > 0)
{
return HeadersRecv["last-modified"];
}
return "";
}
const std::string getEtag()
{
if (HeadersRecv.count("etag") > 0)
{
return HeadersRecv["etag"];
}
return "";
}
public:
CURL *Request;
std::string Url;
std::string Content;
private:
// headers sent with curl request, must be released after transfer
curl_slist * HeadersSent;
@ -274,20 +314,17 @@ namespace NLGUI
return false;
}
// TODO: replace with expire and etag headers
if (CFile::fileExists(download.dest))
{
time_t currentTime;
time(&currentTime);
uint32 mtime = CFile::getFileModificationDate(download.dest);
if (mtime + 3600 > currentTime)
CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest);
if (cache.Expires > currentTime)
{
#ifdef LOG_DL
nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime);
nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime);
#endif
return false;
}
}
string tmpdest = download.dest + ".tmp";
@ -322,6 +359,16 @@ namespace NLGUI
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
std::vector<std::string> headers;
if (!cache.Etag.empty())
headers.push_back("If-None-Match: " + cache.Etag);
if (!cache.LastModified.empty())
headers.push_back("If-Modified-Since: " + cache.LastModified);
if (headers.size() > 0)
download.data->sendHeaders(headers);
// catch headers
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data);
@ -601,9 +648,27 @@ namespace NLGUI
if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString())))
{
NLMISC::CFile::deleteFile(tmpfile.c_str());
// 304 Not Modified
if (res == CURLE_OK && r == 304)
{
CHttpCacheObject obj;
obj.Expires = it->data->getExpires();
obj.Etag = it->data->getEtag();
obj.LastModified = it->data->getLastModified();
CHttpCache::getInstance()->store(it->dest, obj);
}
}
else
{
CHttpCacheObject obj;
obj.Expires = it->data->getExpires();
obj.Etag = it->data->getEtag();
obj.LastModified = it->data->getLastModified();
CHttpCache::getInstance()->store(it->dest, obj);
string finalUrl;
if (it->type == ImgType)
{
@ -5179,11 +5244,7 @@ namespace NLGUI
std::vector<std::string> headers;
headers.push_back("Accept-Language: "+options.languageCode);
headers.push_back("Accept-Charset: utf-8");
for(uint i=0; i< headers.size(); ++i)
{
_CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str());
}
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent);
_CurlWWW->sendHeaders(headers);
// catch headers for redirect
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);

View file

@ -0,0 +1,200 @@
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#include "stdpch.h"
#include "nel/gui/http_cache.h"
using namespace std;
using namespace NLMISC;
#ifdef DEBUG_NEW
#define new DEBUG_NEW
#endif
namespace NLGUI
{
CHttpCache* CHttpCache::instance = NULL;
CHttpCache* CHttpCache::getInstance()
{
if (!instance)
{
instance = new CHttpCache();
}
return instance;
}
void CHttpCache::release()
{
delete instance;
instance = NULL;
}
CHttpCache::CHttpCache()
: _Initialized(false)
, _MaxObjects(100)
{ };
CHttpCache::~CHttpCache()
{
flushCache();
}
void CHttpCache::setCacheIndex(const std::string& fname)
{
_IndexFilename = fname;
_Initialized = false;
}
CHttpCacheObject CHttpCache::lookup(const std::string& fname)
{
if (!_Initialized)
init();
if (_List.count(fname) > 0)
return _List[fname];
return CHttpCacheObject();
}
void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data)
{
if (!_Initialized)
init();
_List[fname] = data;
}
void CHttpCache::init()
{
if (_Initialized)
return;
_Initialized = true;
if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename))
return;
CIFile in;
if (!in.open(_IndexFilename)) {
nlwarning("Unable to open %s for reading", _IndexFilename.c_str());
return;
}
serial(in);
}
void CHttpCacheObject::serial(NLMISC::IStream& f)
{
f.serialVersion(1);
f.serial(Expires);
f.serial(LastModified);
f.serial(Etag);
}
void CHttpCache::serial(NLMISC::IStream& f)
{
// saved state is ignored when version checks fail
try {
f.serialVersion(1);
// CacheIdx
f.serialCheck(NELID("hcaC"));
f.serialCheck(NELID("xdIe"));
if (f.isReading())
{
uint32 numFiles;
f.serial(numFiles);
_List.clear();
for (uint k = 0; k < numFiles; ++k)
{
std::string fname;
f.serial(fname);
CHttpCacheObject obj;
obj.serial(f);
_List[fname] = obj;
}
}
else
{
uint32 numFiles = _List.size();
f.serial(numFiles);
for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it)
{
std::string fname(it->first);
f.serial(fname);
(*it).second.serial(f);
}
}
} catch (...) {
_List.clear();
nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str());
return;
}
}
void CHttpCache::pruneCache()
{
if (_List.size() < _MaxObjects)
return;
size_t mustDrop = _List.size() - _MaxObjects;
time_t currentTime;
time(&currentTime);
// if we over object limit, then start removing expired objects
// this does not guarantee that max limit is reached
for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) {
if (it->second.Expires <= currentTime) {
it = _List.erase(it);
--mustDrop;
if (mustDrop == 0)
break;
}
else
{
++it;
}
}
}
void CHttpCache::flushCache()
{
if (_IndexFilename.empty())
return;
pruneCache();
COFile out;
if (!out.open(_IndexFilename))
{
nlwarning("Unable to open %s for writing", _IndexFilename.c_str());
return;
}
serial(out);
out.close();
}
}

View file

@ -26,6 +26,7 @@
#include <string>
#include <limits>
#include <ctime>
#include "nel/misc/types_nl.h"
#include "nel/misc/algo.h"

View file

@ -68,6 +68,7 @@
#include "interface_v3/sbrick_manager.h"
#include "nel/gui/widget_manager.h"
#include "nel/gui/http_cache.h"
//
#include "gabarit.h"
#include "hair_set.h"
@ -1365,6 +1366,10 @@ void prelogInit()
//nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000);
}
// Initialize HTTP cache
CHttpCache::getInstance()->setCacheIndex("cache/cache.index");
CHttpCache::getInstance()->init();
// Register the reflected classes
registerInterfaceElements();

View file

@ -94,7 +94,7 @@
#include "bg_downloader_access.h"
#include "nel/gui/lua_manager.h"
#include "item_group_manager.h"
#include "nel/gui/http_cache.h"
///////////
// USING //
@ -687,6 +687,7 @@ void release()
CWidgetManager::release();
CViewRenderer::release();
CIXml::releaseLibXml();
CHttpCache::release();
#if FINAL_VERSION
// openURL ("http://www.ryzomcore.org/exit/");