mirror of
https://port.numenaute.org/aleajactaest/khanat-opennel-code.git
synced 2024-11-10 01:09:50 +00:00
Changed: Use http cache headers for image cache
--HG-- branch : develop
This commit is contained in:
parent
3d09dd2094
commit
108759952e
6 changed files with 361 additions and 16 deletions
77
code/nel/include/nel/gui/http_cache.h
Normal file
77
code/nel/include/nel/gui/http_cache.h
Normal file
|
@ -0,0 +1,77 @@
|
|||
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
|
||||
// Copyright (C) 2010 Winch Gate Property Limited
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#ifndef CL_HTTP_CACHE_H
|
||||
#define CL_HTTP_CACHE_H
|
||||
|
||||
#include "nel/misc/types_nl.h"
|
||||
|
||||
namespace NLGUI
|
||||
{
|
||||
struct CHttpCacheObject
|
||||
{
|
||||
CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "")
|
||||
: Expires(expires)
|
||||
, LastModified(lastModified)
|
||||
, Etag(etag){};
|
||||
|
||||
uint32 Expires;
|
||||
std::string LastModified;
|
||||
std::string Etag;
|
||||
|
||||
void serial(NLMISC::IStream& f);
|
||||
};
|
||||
|
||||
/**
|
||||
* Keeping track of downloaded files cache related headers
|
||||
* \author Meelis Mägi (nimetu)
|
||||
* \date 2017
|
||||
*/
|
||||
class CHttpCache
|
||||
{
|
||||
typedef std::map<std::string, CHttpCacheObject> THttpCacheMap;
|
||||
|
||||
public:
|
||||
static CHttpCache* getInstance();
|
||||
static void release();
|
||||
|
||||
public:
|
||||
void setCacheIndex(const std::string& fname);
|
||||
void init();
|
||||
|
||||
CHttpCacheObject lookup(const std::string& fname);
|
||||
void store(const std::string& fname, const CHttpCacheObject& data);
|
||||
|
||||
void flushCache();
|
||||
|
||||
void serial(NLMISC::IStream& f);
|
||||
|
||||
private:
|
||||
CHttpCache();
|
||||
~CHttpCache();
|
||||
|
||||
void pruneCache();
|
||||
|
||||
static CHttpCache* instance;
|
||||
|
||||
THttpCacheMap _List;
|
||||
|
||||
std::string _IndexFilename;
|
||||
bool _Initialized;
|
||||
size_t _MaxObjects;
|
||||
};
|
||||
}
|
||||
#endif
|
|
@ -45,6 +45,7 @@
|
|||
#include "nel/3d/texture_file.h"
|
||||
#include "nel/misc/big_file.h"
|
||||
#include "nel/gui/url_parser.h"
|
||||
#include "nel/gui/http_cache.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace NLMISC;
|
||||
|
@ -87,6 +88,15 @@ namespace NLGUI
|
|||
curl_slist_free_all(HeadersSent);
|
||||
}
|
||||
|
||||
void sendHeaders(const std::vector<std::string> headers)
|
||||
{
|
||||
for(uint i = 0; i < headers.size(); ++i)
|
||||
{
|
||||
HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str());
|
||||
}
|
||||
curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent);
|
||||
}
|
||||
|
||||
void setRecvHeader(const std::string &header)
|
||||
{
|
||||
size_t pos = header.find(": ");
|
||||
|
@ -110,12 +120,42 @@ namespace NLGUI
|
|||
return "";
|
||||
}
|
||||
|
||||
const uint32 getExpires()
|
||||
{
|
||||
time_t ret = 0;
|
||||
if (HeadersRecv.count("expires") > 0)
|
||||
ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL);
|
||||
|
||||
return ret > -1 ? ret : 0;
|
||||
}
|
||||
|
||||
const std::string getLastModified()
|
||||
{
|
||||
if (HeadersRecv.count("last-modified") > 0)
|
||||
{
|
||||
return HeadersRecv["last-modified"];
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
const std::string getEtag()
|
||||
{
|
||||
if (HeadersRecv.count("etag") > 0)
|
||||
{
|
||||
return HeadersRecv["etag"];
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
public:
|
||||
CURL *Request;
|
||||
|
||||
std::string Url;
|
||||
std::string Content;
|
||||
|
||||
private:
|
||||
// headers sent with curl request, must be released after transfer
|
||||
curl_slist * HeadersSent;
|
||||
|
||||
|
@ -274,19 +314,16 @@ namespace NLGUI
|
|||
return false;
|
||||
}
|
||||
|
||||
// TODO: replace with expire and etag headers
|
||||
if (CFile::fileExists(download.dest))
|
||||
time_t currentTime;
|
||||
time(¤tTime);
|
||||
|
||||
CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest);
|
||||
if (cache.Expires > currentTime)
|
||||
{
|
||||
time_t currentTime;
|
||||
time(¤tTime);
|
||||
uint32 mtime = CFile::getFileModificationDate(download.dest);
|
||||
if (mtime + 3600 > currentTime)
|
||||
{
|
||||
#ifdef LOG_DL
|
||||
nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime);
|
||||
nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime);
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
string tmpdest = download.dest + ".tmp";
|
||||
|
@ -322,6 +359,16 @@ namespace NLGUI
|
|||
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
||||
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
||||
|
||||
std::vector<std::string> headers;
|
||||
if (!cache.Etag.empty())
|
||||
headers.push_back("If-None-Match: " + cache.Etag);
|
||||
|
||||
if (!cache.LastModified.empty())
|
||||
headers.push_back("If-Modified-Since: " + cache.LastModified);
|
||||
|
||||
if (headers.size() > 0)
|
||||
download.data->sendHeaders(headers);
|
||||
|
||||
// catch headers
|
||||
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data);
|
||||
|
@ -601,9 +648,27 @@ namespace NLGUI
|
|||
if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString())))
|
||||
{
|
||||
NLMISC::CFile::deleteFile(tmpfile.c_str());
|
||||
|
||||
// 304 Not Modified
|
||||
if (res == CURLE_OK && r == 304)
|
||||
{
|
||||
CHttpCacheObject obj;
|
||||
obj.Expires = it->data->getExpires();
|
||||
obj.Etag = it->data->getEtag();
|
||||
obj.LastModified = it->data->getLastModified();
|
||||
|
||||
CHttpCache::getInstance()->store(it->dest, obj);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
CHttpCacheObject obj;
|
||||
obj.Expires = it->data->getExpires();
|
||||
obj.Etag = it->data->getEtag();
|
||||
obj.LastModified = it->data->getLastModified();
|
||||
|
||||
CHttpCache::getInstance()->store(it->dest, obj);
|
||||
|
||||
string finalUrl;
|
||||
if (it->type == ImgType)
|
||||
{
|
||||
|
@ -5179,11 +5244,7 @@ namespace NLGUI
|
|||
std::vector<std::string> headers;
|
||||
headers.push_back("Accept-Language: "+options.languageCode);
|
||||
headers.push_back("Accept-Charset: utf-8");
|
||||
for(uint i=0; i< headers.size(); ++i)
|
||||
{
|
||||
_CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str());
|
||||
}
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent);
|
||||
_CurlWWW->sendHeaders(headers);
|
||||
|
||||
// catch headers for redirect
|
||||
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
|
||||
|
|
200
code/nel/src/gui/http_cache.cpp
Normal file
200
code/nel/src/gui/http_cache.cpp
Normal file
|
@ -0,0 +1,200 @@
|
|||
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
|
||||
// Copyright (C) 2010 Winch Gate Property Limited
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#include "stdpch.h"
|
||||
#include "nel/gui/http_cache.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace NLMISC;
|
||||
|
||||
#ifdef DEBUG_NEW
|
||||
#define new DEBUG_NEW
|
||||
#endif
|
||||
|
||||
namespace NLGUI
|
||||
{
|
||||
CHttpCache* CHttpCache::instance = NULL;
|
||||
|
||||
CHttpCache* CHttpCache::getInstance()
|
||||
{
|
||||
if (!instance)
|
||||
{
|
||||
instance = new CHttpCache();
|
||||
}
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
void CHttpCache::release()
|
||||
{
|
||||
delete instance;
|
||||
instance = NULL;
|
||||
}
|
||||
|
||||
CHttpCache::CHttpCache()
|
||||
: _Initialized(false)
|
||||
, _MaxObjects(100)
|
||||
{ };
|
||||
|
||||
CHttpCache::~CHttpCache()
|
||||
{
|
||||
flushCache();
|
||||
}
|
||||
|
||||
void CHttpCache::setCacheIndex(const std::string& fname)
|
||||
{
|
||||
_IndexFilename = fname;
|
||||
_Initialized = false;
|
||||
}
|
||||
|
||||
CHttpCacheObject CHttpCache::lookup(const std::string& fname)
|
||||
{
|
||||
if (!_Initialized)
|
||||
init();
|
||||
|
||||
if (_List.count(fname) > 0)
|
||||
return _List[fname];
|
||||
|
||||
return CHttpCacheObject();
|
||||
}
|
||||
|
||||
void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data)
|
||||
{
|
||||
if (!_Initialized)
|
||||
init();
|
||||
|
||||
_List[fname] = data;
|
||||
}
|
||||
|
||||
void CHttpCache::init()
|
||||
{
|
||||
if (_Initialized)
|
||||
return;
|
||||
|
||||
_Initialized = true;
|
||||
|
||||
if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename))
|
||||
return;
|
||||
|
||||
CIFile in;
|
||||
if (!in.open(_IndexFilename)) {
|
||||
nlwarning("Unable to open %s for reading", _IndexFilename.c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
serial(in);
|
||||
}
|
||||
|
||||
void CHttpCacheObject::serial(NLMISC::IStream& f)
|
||||
{
|
||||
f.serialVersion(1);
|
||||
f.serial(Expires);
|
||||
f.serial(LastModified);
|
||||
f.serial(Etag);
|
||||
}
|
||||
|
||||
void CHttpCache::serial(NLMISC::IStream& f)
|
||||
{
|
||||
// saved state is ignored when version checks fail
|
||||
try {
|
||||
f.serialVersion(1);
|
||||
|
||||
// CacheIdx
|
||||
f.serialCheck(NELID("hcaC"));
|
||||
f.serialCheck(NELID("xdIe"));
|
||||
|
||||
if (f.isReading())
|
||||
{
|
||||
uint32 numFiles;
|
||||
f.serial(numFiles);
|
||||
|
||||
_List.clear();
|
||||
for (uint k = 0; k < numFiles; ++k)
|
||||
{
|
||||
std::string fname;
|
||||
f.serial(fname);
|
||||
|
||||
CHttpCacheObject obj;
|
||||
obj.serial(f);
|
||||
|
||||
_List[fname] = obj;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
uint32 numFiles = _List.size();
|
||||
f.serial(numFiles);
|
||||
|
||||
for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it)
|
||||
{
|
||||
std::string fname(it->first);
|
||||
f.serial(fname);
|
||||
|
||||
(*it).second.serial(f);
|
||||
}
|
||||
}
|
||||
} catch (...) {
|
||||
_List.clear();
|
||||
nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void CHttpCache::pruneCache()
|
||||
{
|
||||
if (_List.size() < _MaxObjects)
|
||||
return;
|
||||
|
||||
size_t mustDrop = _List.size() - _MaxObjects;
|
||||
|
||||
time_t currentTime;
|
||||
time(¤tTime);
|
||||
|
||||
// if we over object limit, then start removing expired objects
|
||||
// this does not guarantee that max limit is reached
|
||||
for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) {
|
||||
if (it->second.Expires <= currentTime) {
|
||||
it = _List.erase(it);
|
||||
|
||||
--mustDrop;
|
||||
if (mustDrop == 0)
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void CHttpCache::flushCache()
|
||||
{
|
||||
if (_IndexFilename.empty())
|
||||
return;
|
||||
|
||||
pruneCache();
|
||||
|
||||
COFile out;
|
||||
if (!out.open(_IndexFilename))
|
||||
{
|
||||
nlwarning("Unable to open %s for writing", _IndexFilename.c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
serial(out);
|
||||
out.close();
|
||||
}
|
||||
}
|
|
@ -26,6 +26,7 @@
|
|||
|
||||
#include <string>
|
||||
#include <limits>
|
||||
#include <ctime>
|
||||
|
||||
#include "nel/misc/types_nl.h"
|
||||
#include "nel/misc/algo.h"
|
||||
|
|
|
@ -68,6 +68,7 @@
|
|||
|
||||
#include "interface_v3/sbrick_manager.h"
|
||||
#include "nel/gui/widget_manager.h"
|
||||
#include "nel/gui/http_cache.h"
|
||||
//
|
||||
#include "gabarit.h"
|
||||
#include "hair_set.h"
|
||||
|
@ -1365,6 +1366,10 @@ void prelogInit()
|
|||
//nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000);
|
||||
}
|
||||
|
||||
// Initialize HTTP cache
|
||||
CHttpCache::getInstance()->setCacheIndex("cache/cache.index");
|
||||
CHttpCache::getInstance()->init();
|
||||
|
||||
// Register the reflected classes
|
||||
registerInterfaceElements();
|
||||
|
||||
|
|
|
@ -94,7 +94,7 @@
|
|||
#include "bg_downloader_access.h"
|
||||
#include "nel/gui/lua_manager.h"
|
||||
#include "item_group_manager.h"
|
||||
|
||||
#include "nel/gui/http_cache.h"
|
||||
|
||||
///////////
|
||||
// USING //
|
||||
|
@ -687,6 +687,7 @@ void release()
|
|||
CWidgetManager::release();
|
||||
CViewRenderer::release();
|
||||
CIXml::releaseLibXml();
|
||||
CHttpCache::release();
|
||||
|
||||
#if FINAL_VERSION
|
||||
// openURL ("http://www.ryzomcore.org/exit/");
|
||||
|
|
Loading…
Reference in a new issue