diff --git a/code/nel/include/nel/gui/group_html.h b/code/nel/include/nel/gui/group_html.h
index 2dc5e80bc..ead873497 100644
--- a/code/nel/include/nel/gui/group_html.h
+++ b/code/nel/include/nel/gui/group_html.h
@@ -821,13 +821,13 @@ namespace NLGUI
{
public:
CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams())
- : curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
+ : data(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
{
if (t == ImgType) imgs.push_back(CDataImageDownload(i, style));
}
public:
- CURL *curl;
+ CCurlWWWData *data;
std::string url;
std::string dest;
std::string luaScript;
diff --git a/code/nel/include/nel/gui/http_cache.h b/code/nel/include/nel/gui/http_cache.h
new file mode 100644
index 000000000..0921b2f64
--- /dev/null
+++ b/code/nel/include/nel/gui/http_cache.h
@@ -0,0 +1,77 @@
+// Ryzom - MMORPG Framework
+// Copyright (C) 2010 Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#ifndef CL_HTTP_CACHE_H
+#define CL_HTTP_CACHE_H
+
+#include "nel/misc/types_nl.h"
+
+namespace NLGUI
+{
+ struct CHttpCacheObject
+ {
+ CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "")
+ : Expires(expires)
+ , LastModified(lastModified)
+ , Etag(etag){};
+
+ uint32 Expires;
+ std::string LastModified;
+ std::string Etag;
+
+ void serial(NLMISC::IStream& f);
+ };
+
+ /**
+ * Keeping track of downloaded files cache related headers
+ * \author Meelis Mägi (nimetu)
+ * \date 2017
+ */
+ class CHttpCache
+ {
+ typedef std::map THttpCacheMap;
+
+ public:
+ static CHttpCache* getInstance();
+ static void release();
+
+ public:
+ void setCacheIndex(const std::string& fname);
+ void init();
+
+ CHttpCacheObject lookup(const std::string& fname);
+ void store(const std::string& fname, const CHttpCacheObject& data);
+
+ void flushCache();
+
+ void serial(NLMISC::IStream& f);
+
+ private:
+ CHttpCache();
+ ~CHttpCache();
+
+ void pruneCache();
+
+ static CHttpCache* instance;
+
+ THttpCacheMap _List;
+
+ std::string _IndexFilename;
+ bool _Initialized;
+ size_t _MaxObjects;
+ };
+}
+#endif
diff --git a/code/nel/src/3d/meshvp_wind_tree.cpp b/code/nel/src/3d/meshvp_wind_tree.cpp
index 1b81af721..ba27216f0 100644
--- a/code/nel/src/3d/meshvp_wind_tree.cpp
+++ b/code/nel/src/3d/meshvp_wind_tree.cpp
@@ -226,7 +226,8 @@ void CMeshVPWindTree::initVertexPrograms()
{
// setup of the VPLight fragment
uint numPls= i/4;
- bool normalize= (i&1)!=0;
+ // FIXME: normalize=true makes trees dance, workaround for issue #160
+ bool normalize= false; //(i&1)!=0;
bool specular= (i&2)!=0;
// combine
diff --git a/code/nel/src/gui/group_html.cpp b/code/nel/src/gui/group_html.cpp
index 319feae92..0c1fba8d0 100644
--- a/code/nel/src/gui/group_html.cpp
+++ b/code/nel/src/gui/group_html.cpp
@@ -45,6 +45,7 @@
#include "nel/3d/texture_file.h"
#include "nel/misc/big_file.h"
#include "nel/gui/url_parser.h"
+#include "nel/gui/http_cache.h"
using namespace std;
using namespace NLMISC;
@@ -87,6 +88,15 @@ namespace NLGUI
curl_slist_free_all(HeadersSent);
}
+ void sendHeaders(const std::vector headers)
+ {
+ for(uint i = 0; i < headers.size(); ++i)
+ {
+ HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str());
+ }
+ curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent);
+ }
+
void setRecvHeader(const std::string &header)
{
size_t pos = header.find(": ");
@@ -110,12 +120,42 @@ namespace NLGUI
return "";
}
+ const uint32 getExpires()
+ {
+ time_t ret = 0;
+ if (HeadersRecv.count("expires") > 0)
+ ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL);
+
+ return ret > -1 ? ret : 0;
+ }
+
+ const std::string getLastModified()
+ {
+ if (HeadersRecv.count("last-modified") > 0)
+ {
+ return HeadersRecv["last-modified"];
+ }
+
+ return "";
+ }
+
+ const std::string getEtag()
+ {
+ if (HeadersRecv.count("etag") > 0)
+ {
+ return HeadersRecv["etag"];
+ }
+
+ return "";
+ }
+
public:
CURL *Request;
std::string Url;
std::string Content;
+ private:
// headers sent with curl request, must be released after transfer
curl_slist * HeadersSent;
@@ -274,19 +314,16 @@ namespace NLGUI
return false;
}
- // TODO: replace with expire and etag headers
- if (CFile::fileExists(download.dest))
+ time_t currentTime;
+ time(¤tTime);
+
+ CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest);
+ if (cache.Expires > currentTime)
{
- time_t currentTime;
- time(¤tTime);
- uint32 mtime = CFile::getFileModificationDate(download.dest);
- if (mtime + 3600 > currentTime)
- {
#ifdef LOG_DL
- nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime);
+ nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime);
#endif
- return false;
- }
+ return false;
}
string tmpdest = download.dest + ".tmp";
@@ -312,7 +349,7 @@ namespace NLGUI
return false;
}
- download.curl = curl;
+ download.data = new CCurlWWWData(curl, download.url);
download.fp = fp;
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
@@ -322,6 +359,20 @@ namespace NLGUI
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+ std::vector headers;
+ if (!cache.Etag.empty())
+ headers.push_back("If-None-Match: " + cache.Etag);
+
+ if (!cache.LastModified.empty())
+ headers.push_back("If-Modified-Since: " + cache.LastModified);
+
+ if (headers.size() > 0)
+ download.data->sendHeaders(headers);
+
+ // catch headers
+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
+ curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data);
+
std::string userAgent = options.appName + "/" + options.appVersion;
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
@@ -369,7 +420,7 @@ namespace NLGUI
RunningCurls++;
#ifdef LOG_DL
- nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
+ nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size());
}
else
{
@@ -443,7 +494,7 @@ namespace NLGUI
}
RunningCurls++;
#ifdef LOG_DL
- nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
+ nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size());
}
else
{
@@ -580,26 +631,44 @@ namespace NLGUI
for (vector::iterator it=Curls.begin(); iteasy_handle == it->curl)
+ if(it->data && it->data->Request == msg->easy_handle)
{
CURLcode res = msg->data.result;
long r;
- curl_easy_getinfo(it->curl, CURLINFO_RESPONSE_CODE, &r);
+ curl_easy_getinfo(it->data->Request, CURLINFO_RESPONSE_CODE, &r);
fclose(it->fp);
+ receiveCookies(it->data->Request, _DocumentDomain, _TrustedDomain);
#ifdef LOG_DL
- nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->curl, res, r, it->url.size(), it->url.c_str());
+ nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->data->Request, res, r, it->url.size(), it->url.c_str());
#endif
- curl_multi_remove_handle(MultiCurl, it->curl);
- curl_easy_cleanup(it->curl);
+ curl_multi_remove_handle(MultiCurl, it->data->Request);
string tmpfile = it->dest + ".tmp";
if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString())))
{
NLMISC::CFile::deleteFile(tmpfile.c_str());
+
+ // 304 Not Modified
+ if (res == CURLE_OK && r == 304)
+ {
+ CHttpCacheObject obj;
+ obj.Expires = it->data->getExpires();
+ obj.Etag = it->data->getEtag();
+ obj.LastModified = it->data->getLastModified();
+
+ CHttpCache::getInstance()->store(it->dest, obj);
+ }
}
else
{
+ CHttpCacheObject obj;
+ obj.Expires = it->data->getExpires();
+ obj.Etag = it->data->getEtag();
+ obj.LastModified = it->data->getLastModified();
+
+ CHttpCache::getInstance()->store(it->dest, obj);
+
string finalUrl;
if (it->type == ImgType)
{
@@ -642,6 +711,9 @@ namespace NLGUI
}
}
+ // release CCurlWWWData
+ delete it->data;
+
Curls.erase(it);
break;
}
@@ -656,7 +728,7 @@ namespace NLGUI
{
for (vector::iterator it=Curls.begin(); itcurl == NULL) {
+ if (it->data == NULL) {
#ifdef LOG_DL
nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str());
#endif
@@ -4575,7 +4647,7 @@ namespace NLGUI
// remove download that are still queued
for (vector::iterator it=Curls.begin(); itcurl == NULL) {
+ if (it->data == NULL) {
#ifdef LOG_DL
nlwarning("Remove waiting curl download (%s)", it->url.c_str());
#endif
@@ -5172,11 +5244,7 @@ namespace NLGUI
std::vector headers;
headers.push_back("Accept-Language: "+options.languageCode);
headers.push_back("Accept-Charset: utf-8");
- for(uint i=0; i< headers.size(); ++i)
- {
- _CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str());
- }
- curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent);
+ _CurlWWW->sendHeaders(headers);
// catch headers for redirect
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
diff --git a/code/nel/src/gui/http_cache.cpp b/code/nel/src/gui/http_cache.cpp
new file mode 100644
index 000000000..f0839752c
--- /dev/null
+++ b/code/nel/src/gui/http_cache.cpp
@@ -0,0 +1,200 @@
+// Ryzom - MMORPG Framework
+// Copyright (C) 2010 Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#include "stdpch.h"
+#include "nel/gui/http_cache.h"
+
+using namespace std;
+using namespace NLMISC;
+
+#ifdef DEBUG_NEW
+#define new DEBUG_NEW
+#endif
+
+namespace NLGUI
+{
+ CHttpCache* CHttpCache::instance = NULL;
+
+ CHttpCache* CHttpCache::getInstance()
+ {
+ if (!instance)
+ {
+ instance = new CHttpCache();
+ }
+
+ return instance;
+ }
+
+ void CHttpCache::release()
+ {
+ delete instance;
+ instance = NULL;
+ }
+
+ CHttpCache::CHttpCache()
+ : _Initialized(false)
+ , _MaxObjects(100)
+ { };
+
+ CHttpCache::~CHttpCache()
+ {
+ flushCache();
+ }
+
+ void CHttpCache::setCacheIndex(const std::string& fname)
+ {
+ _IndexFilename = fname;
+ _Initialized = false;
+ }
+
+ CHttpCacheObject CHttpCache::lookup(const std::string& fname)
+ {
+ if (!_Initialized)
+ init();
+
+ if (_List.count(fname) > 0)
+ return _List[fname];
+
+ return CHttpCacheObject();
+ }
+
+ void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data)
+ {
+ if (!_Initialized)
+ init();
+
+ _List[fname] = data;
+ }
+
+ void CHttpCache::init()
+ {
+ if (_Initialized)
+ return;
+
+ _Initialized = true;
+
+ if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename))
+ return;
+
+ CIFile in;
+ if (!in.open(_IndexFilename)) {
+ nlwarning("Unable to open %s for reading", _IndexFilename.c_str());
+ return;
+ }
+
+ serial(in);
+ }
+
+ void CHttpCacheObject::serial(NLMISC::IStream& f)
+ {
+ f.serialVersion(1);
+ f.serial(Expires);
+ f.serial(LastModified);
+ f.serial(Etag);
+ }
+
+ void CHttpCache::serial(NLMISC::IStream& f)
+ {
+ // saved state is ignored when version checks fail
+ try {
+ f.serialVersion(1);
+
+ // CacheIdx
+ f.serialCheck(NELID("hcaC"));
+ f.serialCheck(NELID("xdIe"));
+
+ if (f.isReading())
+ {
+ uint32 numFiles;
+ f.serial(numFiles);
+
+ _List.clear();
+ for (uint k = 0; k < numFiles; ++k)
+ {
+ std::string fname;
+ f.serial(fname);
+
+ CHttpCacheObject obj;
+ obj.serial(f);
+
+ _List[fname] = obj;
+ }
+ }
+ else
+ {
+ uint32 numFiles = _List.size();
+ f.serial(numFiles);
+
+ for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it)
+ {
+ std::string fname(it->first);
+ f.serial(fname);
+
+ (*it).second.serial(f);
+ }
+ }
+ } catch (...) {
+ _List.clear();
+ nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str());
+ return;
+ }
+ }
+
+ void CHttpCache::pruneCache()
+ {
+ if (_List.size() < _MaxObjects)
+ return;
+
+ size_t mustDrop = _List.size() - _MaxObjects;
+
+ time_t currentTime;
+ time(¤tTime);
+
+ // if we over object limit, then start removing expired objects
+ // this does not guarantee that max limit is reached
+ for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) {
+ if (it->second.Expires <= currentTime) {
+ it = _List.erase(it);
+
+ --mustDrop;
+ if (mustDrop == 0)
+ break;
+ }
+ else
+ {
+ ++it;
+ }
+ }
+ }
+
+ void CHttpCache::flushCache()
+ {
+ if (_IndexFilename.empty())
+ return;
+
+ pruneCache();
+
+ COFile out;
+ if (!out.open(_IndexFilename))
+ {
+ nlwarning("Unable to open %s for writing", _IndexFilename.c_str());
+ return;
+ }
+
+ serial(out);
+ out.close();
+ }
+}
diff --git a/code/nel/src/gui/stdpch.h b/code/nel/src/gui/stdpch.h
index e0be5837e..a4ba0ecac 100644
--- a/code/nel/src/gui/stdpch.h
+++ b/code/nel/src/gui/stdpch.h
@@ -26,6 +26,7 @@
#include
#include
+#include
#include "nel/misc/types_nl.h"
#include "nel/misc/algo.h"
diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp
index 267042b7e..900f3c2c1 100644
--- a/code/ryzom/client/src/init.cpp
+++ b/code/ryzom/client/src/init.cpp
@@ -68,6 +68,7 @@
#include "interface_v3/sbrick_manager.h"
#include "nel/gui/widget_manager.h"
+#include "nel/gui/http_cache.h"
//
#include "gabarit.h"
#include "hair_set.h"
@@ -1365,6 +1366,10 @@ void prelogInit()
//nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000);
}
+ // Initialize HTTP cache
+ CHttpCache::getInstance()->setCacheIndex("cache/cache.index");
+ CHttpCache::getInstance()->init();
+
// Register the reflected classes
registerInterfaceElements();
diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp
index 51ec667a2..581d4c2b8 100644
--- a/code/ryzom/client/src/release.cpp
+++ b/code/ryzom/client/src/release.cpp
@@ -94,7 +94,7 @@
#include "bg_downloader_access.h"
#include "nel/gui/lua_manager.h"
#include "item_group_manager.h"
-
+#include "nel/gui/http_cache.h"
///////////
// USING //
@@ -687,6 +687,7 @@ void release()
CWidgetManager::release();
CViewRenderer::release();
CIXml::releaseLibXml();
+ CHttpCache::release();
#if FINAL_VERSION
// openURL ("http://ryzom.com/exit/");