Khaganat-NeL/khanat-opennel-code
2
0
Fork 0
Code Issues 21 Pull requests Projects Releases 1 Packages Wiki Activity

[Hit return to continue]

Browse source
This commit is contained in:
AleaJactaEst 2018-12-01 10:24:21 +01:00
commit 05a96ceea3
5 changed files with 281 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
.gitlab-ci.yml
View file

@ -52,9 +52,9 @@ Linux client build:
- echo "disableproc:$disableproc, nb core compile:$(nproc --ignore=$disableproc) optionproc:$optionproc"
# Installation des dépendances
#- echo "deb http://ftp.debian.org/debian/ jessie-backports non-free contrib main" >> /etc/apt/sources.list
- apt-get update
- apt-get -y upgrade
- apt-get install libxml2 -y g++ gcc cmake libcurl4-openssl-dev libgl1-mesa-dev libjpeg-dev libpng-dev libopenal-dev libfreetype6-dev libxxf86vm-dev libxrandr-dev libxrender-dev libvorbis-dev libluabind-dev libboost-dev libmysqlclient-dev libssl-dev liblzma-dev libxml2-dev makeself libgif-dev patch
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get install libxml2 -y g++ gcc cmake libcurl4-openssl-dev libgl1-mesa-dev libjpeg-dev libpng-dev libopenal-dev libfreetype6-dev libxxf86vm-dev libxrandr-dev libxrender-dev libvorbis-dev libluabind-dev libboost-dev libmysqlclient-dev libssl-dev liblzma-dev libxml2-dev makeself libgif-dev patch
# Installation des dépendances des dépendances statiques (à commenter si construction de Khanat en mode dynamique)
# libxml2 : python-pyicu (support d'unicode), python-dev (support de... python)
- apt-get install -y wget python-pyicu python-dev
@ -143,10 +143,9 @@ Linux server debian_amd64_strech build:
image: amd64/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server
- apt-get install -y
git
libcurl4-openssl-dev
libfreetype6-dev
@ -231,8 +230,8 @@ Linux client debian_amd64_strech build:
image: amd64/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- apt-get install -y autoconf
autogen
automake
@ -297,8 +296,8 @@ Linux client_static debian_amd64_strech build:
image: amd64/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- apt-get install -y
autoconf
autogen
@ -407,8 +406,8 @@ Linux client_static_debug debian_amd64_strech build:
image: amd64/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- apt-get install -y
autoconf
autogen
@ -516,8 +515,8 @@ Linux server_static debian_amd64_strech build:
image: amd64/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 apache2-utils autoconf autogen automake bison build-essential cmake cpputest default-libmysqlclient-dev fakeroot git libapache2-mod-php libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblua5.2-0 liblua5.2-dev liblzma-dev libogg-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libvorbis-dev libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial mysql-server php php-gd php-imagick php-mysql python3 python3-pip python3-bcrypt python3-venv rrdtool unzip wget zlib1g-dev
# Apply patch
- (for patchfile in $(cat patch/series); do echo "patch patch/$patchfile"; patch -f -Z -t -p 1 -i patch/$patchfile || exit 2; done)
@ -607,8 +606,8 @@ Linux server_static_debug debian_amd64_strech build:
image: amd64/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 apache2-utils autoconf autogen automake bison build-essential cmake cpputest default-libmysqlclient-dev fakeroot git libapache2-mod-php libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblua5.2-0 liblua5.2-dev liblzma-dev libogg-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libvorbis-dev libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial mysql-server php php-gd php-imagick php-mysql python3 python3-pip python3-bcrypt python3-venv rrdtool unzip wget zlib1g-dev
# Apply patch
- (for patchfile in $(cat patch/series); do echo "patch patch/$patchfile"; patch -f -Z -t -p 1 -i patch/$patchfile || exit 2; done)
@ -698,8 +697,8 @@ Linux client_static debian_i386_strech build:
image: i386/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- apt-get install -y autoconf autogen automake bison build-essential cmake cpputest fakeroot libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblzma-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial rrdtool unzip wget zlib1g-dev patch
- apt-get install -y liblua5.2-0 liblua5.2-dev
# Apply patch
@ -776,9 +775,9 @@ Linux client_static_debug debian_i386_strech build:
image: i386/debian:9
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- apt-get install -y autoconf autogen automake bison build-essential cmake cpputest fakeroot libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblzma-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial rrdtool unzip wget zlib1g-dev patch
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf autogen automake bison build-essential cmake cpputest fakeroot libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblzma-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial rrdtool unzip wget zlib1g-dev patch
- apt-get install -y liblua5.2-0 liblua5.2-dev
# Apply patch
- (for patchfile in $(cat patch/series); do echo "patch patch/$patchfile"; patch -f -Z -t -p 1 -i patch/$patchfile || exit 2; done)
@ -920,9 +919,9 @@ Linux client ubuntu_amd64_17_10 build:
image: amd64/ubuntu:17.10
script:
# Prepare environment
- apt-get update
- apt-get dist-upgrade -y
- apt-get install -y autoconf
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf
autogen
automake
bison
@ -1077,9 +1076,9 @@ Windows client build:
script:
# Prepare environment
- dpkg --add-architecture i386
- apt-get update
- apt-get dist-upgrade -y
- apt-get install -y
- DEBIAN_FRONTEND=noninteractive apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
- DEBIAN_FRONTEND=noninteractive apt-get install -y
p7zip-full
binfmt-support
libc6-i386
@ -1142,7 +1141,7 @@ Linux client test:
- mac-ci-build
tags:
- Docker
image: ubuntu:14.04
image: ubuntu:18.04
script:
# Test de l'installation initiale
- ./khanat-$(echo $CI_BUILD_REF | head -c 7 )-$CI_PIPELINE_ID-Linux-x86_64.run
@ -1172,7 +1171,7 @@ Linux client test:
- ryzomcore
tags:
- Docker
image: ubuntu:14.04
image: ubuntu:18.04
script:
- echo "pas encore de deployment"
dependencies:
@ -1192,7 +1191,7 @@ Linux client test:
- tags
tags:
- Docker
image: ubuntu:14.04
image: ubuntu:18.04
script:
- echo "pas encore de tag"
dependencies:

4
code/ryzom/tools/patch_gen/CMakeLists.txt
View file

@ -1,7 +1,7 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip)
SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h)
SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h)
SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h patch_gen_common.h)
SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h patch_gen_common.h)
ADD_EXECUTABLE(patch_gen ${MAIN_SRC})
TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges)

302
code/ryzom/tools/patch_gen/patch_gen_common.cpp
View file

@ -20,6 +20,11 @@
#include <cstdio>
#include <limits>
#include <string>
#include <vector>
#include <queue>
#include <mutex>
#include <thread>
#include "game_share/bnp_patch.h"
#include "nel/misc/path.h"
@ -28,12 +33,14 @@
#include "nel/misc/sstring.h"
#include "game_share/singleton_registry.h"
#include "seven_zip.h"
#include "patch_gen_main.h"
using namespace std;
using namespace NLMISC;
#define PERSISTENT_TOKEN_FAMILY RyzomTokenFamily
unsigned int jobs_simultaneously = 1;
//-----------------------------------------------------------------------------
// Handy utility functions
@ -47,11 +54,11 @@ void normalisePackageDescriptionFileName(std::string& fileName)
fileName+=".xml";
}
void GeneratePatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName)
void GeneratePatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName)
{
std::string cmd = toString("xdelta delta %s %s %s", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str());
nlinfo("Executing system command: %s", cmd.c_str());
nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str());
#ifdef NL_OS_WINDOWS
_spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "delta", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str(), NULL);
@ -60,15 +67,15 @@ void GeneratePatch(const std::string& srcFileName,const std::string& destFileNam
sint error = system (cmd.c_str());
if (error == 2)
nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error);
#endif // NL_OS_WINDOWS
}
void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string())
void ApplyPatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string())
{
std::string cmd = toString("xdelta patch %s %s %s", patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str());
nlinfo("Executing system command: %s", cmd.c_str());
nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str());
#ifdef NL_OS_WINDOWS
_spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "patch",patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str(), NULL);
@ -77,22 +84,157 @@ void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,c
sint error = system (cmd.c_str());
if (error == 2)
nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error);
#endif // NL_OS_WINDOWS
}
void GenerateLZMA(const std::string &sourceFile, const std::string &outputFile)
void GenerateLZMA(const uint32 id, const std::string &sourceFile, const std::string &outputFile)
{
{
nlinfo("Compressing %s to %s using LZMA...", sourceFile.c_str(), outputFile.c_str());
nlinfo("[id:%d] Compressing %s to %s using LZMA...", id, sourceFile.c_str(), outputFile.c_str());
}
if (!packLZMA(sourceFile, outputFile))
{
nlwarning("LZMA compress failed");
nlwarning("[id:%d, source:%s] LZMA compress failed", id, sourceFile.c_str());
}
}
//-----------------------------------------------------------------------------
// class CTaskPackageDescription
//-----------------------------------------------------------------------------
class CTaskPackageDescription
{
bool deleteRefAfterDelta;
bool usingTemporaryFile;
uint32 id;
CBNPFile * _pbnpPtr;
std::string _BnpDirectory;
std::string _RefDirectory;
std::string _PatchDirectory;
std::string _RootDirectory;
const CBNPCategorySet * _pCategories;
public:
CTaskPackageDescription(uint32 id, bool deleteRefAfterDelta, bool usingTemporaryFile, CBNPFile * _pbnpPtr,
std::string _BnpDirectory, std::string _RefDirectory, std::string _PatchDirectory,
std::string _RootDirectory, const CBNPCategorySet * _pCategories)
{
this->id = id;
this->deleteRefAfterDelta = deleteRefAfterDelta;
this->usingTemporaryFile = usingTemporaryFile;
this->_pbnpPtr = _pbnpPtr;
this->_BnpDirectory = _BnpDirectory;
this->_RefDirectory = _RefDirectory;
this->_PatchDirectory = _PatchDirectory;
this->_RootDirectory = _RootDirectory;
this->_pCategories = _pCategories;
}
void action()
{
nldebug("[id:%d] thread for %s : start", id, this->_pbnpPtr->getFileName().c_str());
std::string bnpFileName = _BnpDirectory + _pbnpPtr->getFileName();
std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
// get the last version number and the related file name
const CBNPFileVersion& curVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-1);
std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch";
// get the second last version number and the related file name
std::string prevVersionFileName;
if (_pbnpPtr->versionCount()==1)
{
prevVersionFileName= _RootDirectory + "empty_" + std::to_string(this->id);
CFile::createEmptyFile(prevVersionFileName);
usingTemporaryFile = true;
deleteRefAfterDelta= false;
}
else
{
const CBNPFileVersion& prevVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-2);
prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
}
std::string refVersionFileName= prevVersionFileName;
// create the subdirectory for this patch number
string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber());
CFile::createDirectory(versionSubDir);
// generate the lzma packed version of the bnp if needed (lzma file are slow to generate)
string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma";
if (!CFile::fileExists(lzmaFile))
{
// build the lzma compression in a temp file (avoid leaving dirty file if the
// process cannot terminate)
GenerateLZMA(id, bnpFileName, lzmaFile+".tmp");
// rename the tmp file
CFile::moveFile(lzmaFile, lzmaFile+".tmp");
}
// store the lzma file size in the descriptor
_pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile));
// if we need to generate a new patch then do it and create the new ref file
if (!NLMISC::CFile::fileExists(curVersionFileName))
{
nlinfo("- Creating patch: %s",patchFileName.c_str());
// in the case where we compress against a ref file...
if (!_pCategories->isFileIncremental(NLMISC::CFile::getFilename(bnpFileName)))
{
// setup the name of the reference file to patch against
refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref";
// delete the previous patch - because we only need the latest patch for non-incremental files
std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch";
if (NLMISC::CFile::fileExists(lastPatch.c_str()))
NLMISC::CFile::deleteFile(lastPatch.c_str());
}
// call xdelta to generate the patch
GeneratePatch(id, refVersionFileName, bnpFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(patchFileName));
uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName);
_pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).setPatchSize(nPatchSize);
// apply the incremental patch to the old ref file to create the new ref file
// and ensure that the new ref file matches the BNP
ApplyPatch(id, refVersionFileName, curVersionFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(curVersionFileName));
nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName));
}
// if we have a ref file still hanging about from the previous patch then delete it
if (NLMISC::CFile::fileExists(prevVersionFileName))
{
NLMISC::CFile::deleteFile(prevVersionFileName);
}
nldebug("[id:%d] thread for %s : end", id, this->_pbnpPtr->getFileName().c_str());
}
};
void threadWorker(std::mutex & mtx, std::queue<CTaskPackageDescription> & queueTask)
{
while(true)
{
mtx.lock();
if ( queueTask.empty())
{
mtx.unlock();
break;
}
CTaskPackageDescription task = queueTask.front();
queueTask.pop();
mtx.unlock();
task.action();
}
};
//-----------------------------------------------------------------------------
// class CPackageDescription
@ -125,6 +267,8 @@ public:
void updatePatchSizes(CBNPFileSet& packageIndex) const;
//void threadWorker(mutex & mtx, queue<CTaskPackageDescription> & queueTask) const;
// specialisation of IVersionNumberGenerator
void grabVersionNumber();
uint32 getPackageVersionNumber();
@ -334,96 +478,76 @@ void CPackageDescription::generatePatches(CBNPFileSet& packageIndex) const
{
nlinfo("Generating patches ...");
for (uint32 i = packageIndex.fileCount(); i--;)
// Generate patch with Leader/Follower pattern
uint32 maxThread=jobs_simultaneously;
uint32 i;
std::mutex mtx;
struct SPrepareTask
{
uint32 fileSize;
uint32 id;
};
struct SComparePrepareTask
{
bool operator()(const SPrepareTask& left, const SPrepareTask& right) const
{
return left.fileSize < right.fileSize;
}
};
std::priority_queue<struct SPrepareTask, std::vector<struct SPrepareTask>, SComparePrepareTask > queuePrepareTask;
std::queue<CTaskPackageDescription> queueTask;
std::vector<thread> workers;
// Initialize queue task
if ( packageIndex.fileCount() < maxThread )
maxThread = packageIndex.fileCount();
for (i = packageIndex.fileCount(); i--;)
{
struct SPrepareTask stask;
stask.id = i;
stask.fileSize = packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).getFileSize();
queuePrepareTask.push(stask);
}
// Reorder task Big file to small file
//getFileSize
std::priority_queue<struct SPrepareTask, std::vector<struct SPrepareTask>, SComparePrepareTask > queueSortedPrepareTask;
while ( ! queuePrepareTask.empty() )
{
bool deleteRefAfterDelta = true;
bool usingTemporaryFile = false;
// generate file name root
std::string bnpFileName = _BnpDirectory + packageIndex.getFile(i).getFileName();
std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
struct SPrepareTask task = queuePrepareTask.top();
queuePrepareTask.pop();
// if the file has no versions then skip on to the next file
if (packageIndex.getFile(i).versionCount()==0)
if (packageIndex.getFile(task.id).versionCount()==0)
continue;
// get the last version number and the related file name
const CBNPFileVersion& curVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1);
std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
// std::string patchFileName= patchNameRoot+NLMISC::toString("_%05d.patch",curVersion.getVersionNumber());
std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch";
// get the second last version number and the related file name
std::string prevVersionFileName;
if (packageIndex.getFile(i).versionCount()==1)
{
prevVersionFileName= _RootDirectory + "empty";
CFile::createEmptyFile(prevVersionFileName);
usingTemporaryFile = true;
deleteRefAfterDelta= false;
}
else
{
const CBNPFileVersion& prevVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-2);
prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
}
std::string refVersionFileName= prevVersionFileName;
// create the subdirectory for this patch number
string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber());
CFile::createDirectory(versionSubDir);
// generate the lzma packed version of the bnp if needed (lzma file are slow to generate)
string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma";
if (!CFile::fileExists(lzmaFile))
{
// build the lzma compression in a temp file (avoid leaving dirty file if the
// process cannot terminate)
GenerateLZMA(bnpFileName, lzmaFile+".tmp");
// rename the tmp file
CFile::moveFile(lzmaFile, lzmaFile+".tmp");
CTaskPackageDescription data(task.id,
deleteRefAfterDelta,
usingTemporaryFile,
& packageIndex.getFile(task.id),
std::ref(_BnpDirectory),
std::ref(_RefDirectory),
std::ref(_PatchDirectory),
std::ref(_RootDirectory),
& _Categories);
queueTask.push(data);
}
// store the lzma file size in the descriptor
packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile));
// launch thread workers
for(i=0;i<maxThread;++i)
workers.push_back(std::thread(threadWorker, std::ref(mtx), std::ref(queueTask)));
// if we need to generate a new patch then do it and create the new ref file
if (!NLMISC::CFile::fileExists(curVersionFileName))
{
nlinfo("- Creating patch: %s",patchFileName.c_str());
// in the case where we compress against a ref file...
if (!_Categories.isFileIncremental(NLMISC::CFile::getFilename(bnpFileName)))
{
// setup the name of the reference file to patch against
refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref";
// delete the previous patch - because we only need the latest patch for non-incremental files
std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch";
if (NLMISC::CFile::fileExists(lastPatch.c_str()))
NLMISC::CFile::deleteFile(lastPatch.c_str());
}
// call xdelta to generate the patch
GeneratePatch(refVersionFileName, bnpFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(patchFileName));
uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName);
packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).setPatchSize(nPatchSize);
// apply the incremental patch to the old ref file to create the new ref file
// and ensure that the new ref file matches the BNP
ApplyPatch(refVersionFileName, curVersionFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(curVersionFileName));
nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName));
}
// if we have a ref file still hanging about from the previous patch then delete it
if (NLMISC::CFile::fileExists(prevVersionFileName))
{
NLMISC::CFile::deleteFile(prevVersionFileName);
}
}
// wait all workers
for (std::vector<std::thread>::iterator it = workers.begin(); it != workers.end(); ++it)
it->join();
NLMISC::DebugLog->displayNL("End thread");
}
void CPackageDescription::createDirectories() const

7
code/ryzom/tools/patch_gen/patch_gen_common.h Normal file
View file

@ -0,0 +1,7 @@
#ifndef PATCH_GEN_COMMON_H
#define PATCH_GEN_COMMON_H
// We need to find other method to send global parameter/argument
extern unsigned int jobs_simultaneously;
#endif // PATCH_GEN_COMMON_H

38
code/ryzom/tools/patch_gen/patch_gen_main.cpp
View file

@ -17,6 +17,8 @@
#include "nel/misc/debug.h"
#include "nel/misc/command.h"
#include "nel/misc/sstring.h"
#include "nel/misc/cmd_args.h"
#include "patch_gen_common.h"
//#include "game_share/handy_commands.h"
//-----------------------------------------------
@ -26,27 +28,43 @@ int main(int argc,char** argv)
{
NLMISC::createDebug();
// Parse Command Line.
NLMISC::CCmdArgs args;
args.setDescription("Generate patch");
args.addArg("j", "jobs", "jobs", "Specifies the number of jobs (commands) to run simultaneously");
args.addAdditionalArg("CommandLine","command line (For a list of valid commands and their paramaters try help)",false,true);
// if there are no command line args then display a friendly message and exit
if (argc==1)
if (!args.parse(argc, argv))
{
NLMISC::InfoLog->displayNL("SYNTAX: %s <command line>",argv[0]);
NLMISC::InfoLog->displayNL("");
NLMISC::InfoLog->displayNL("For a list of valid commands and their paramaters try: %s help",argv[0]);
NLMISC::InfoLog->displayNL("");
return 0;
args.displayHelp();
return 1;
}
// Analyze option
try
{
std::string cjobs = args.haveArg("j") ? args.getArg("j").front() : "1";
jobs_simultaneously = (uint32) std::stoul(cjobs);
} catch (const std::invalid_argument) {
args.displayHelp();
NLMISC::ErrorLog->displayNL("Bad value for parameter jobs");
return 2;
}
// build the command line by concatnating input arguments (separating with spaces)
NLMISC::DebugLog->displayNL("param jobs:%u", jobs_simultaneously);
std::vector<std::string> listargs = args.getAdditionalArg("CommandLine");
NLMISC::CSString commandline;
for (int i=1;i<argc;++i)
for (std::vector<std::string>::iterator it = listargs.begin() ; it != listargs.end(); ++it)
{
NLMISC::CSString s= argv[i];
NLMISC::CSString s = *it;
// check whether the argument needs to be quote encapsulated
if (s.contains(' ') && s[0]!='\"')
s= "\""+ s+ "\"";
if (i>1)
if ( it != listargs.begin() )
commandline+=' ';
commandline+= s;
}