[Hit return to continue]

.gitlab-ci.yml

@@ -52,9 +52,9 @@ Linux client build:
     - echo "disableproc:$disableproc, nb core compile:$(nproc --ignore=$disableproc) optionproc:$optionproc"
     # Installation des dépendances
     #- echo "deb http://ftp.debian.org/debian/ jessie-backports non-free contrib main" >> /etc/apt/sources.list
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get -y upgrade
+    - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
-    - apt-get install libxml2 -y g++ gcc cmake libcurl4-openssl-dev libgl1-mesa-dev libjpeg-dev libpng-dev libopenal-dev libfreetype6-dev libxxf86vm-dev libxrandr-dev libxrender-dev libvorbis-dev libluabind-dev libboost-dev libmysqlclient-dev libssl-dev liblzma-dev libxml2-dev makeself libgif-dev patch
+    - DEBIAN_FRONTEND=noninteractive apt-get install libxml2 -y g++ gcc cmake libcurl4-openssl-dev libgl1-mesa-dev libjpeg-dev libpng-dev libopenal-dev libfreetype6-dev libxxf86vm-dev libxrandr-dev libxrender-dev libvorbis-dev libluabind-dev libboost-dev libmysqlclient-dev libssl-dev liblzma-dev libxml2-dev makeself libgif-dev patch
     # Installation des dépendances des dépendances statiques (à commenter si construction de Khanat en mode dynamique)
     # libxml2 : python-pyicu (support d'unicode), python-dev (support de... python)
     - apt-get install -y wget python-pyicu python-dev
@@ -143,10 +143,9 @@ Linux server debian_amd64_strech build:
   image: amd64/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
     - DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server
-    - apt-get install -y
                        git
                        libcurl4-openssl-dev
                        libfreetype6-dev
@@ -231,8 +230,8 @@ Linux client debian_amd64_strech build:
   image: amd64/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
     - apt-get install -y autoconf
                          autogen
                          automake
@@ -297,8 +296,8 @@ Linux client_static debian_amd64_strech build:
   image: amd64/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
     - apt-get install -y
                        autoconf
                        autogen
@@ -407,8 +406,8 @@ Linux client_static_debug debian_amd64_strech build:
   image: amd64/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
     - apt-get install -y
                        autoconf
                        autogen
@@ -516,8 +515,8 @@ Linux server_static debian_amd64_strech build:
   image: amd64/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
     - DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 apache2-utils autoconf autogen automake bison build-essential cmake cpputest default-libmysqlclient-dev fakeroot git libapache2-mod-php libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblua5.2-0 liblua5.2-dev liblzma-dev libogg-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libvorbis-dev libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial mysql-server php php-gd php-imagick php-mysql python3 python3-pip python3-bcrypt python3-venv rrdtool unzip wget zlib1g-dev
     # Apply patch
     - (for patchfile in $(cat patch/series); do echo "patch patch/$patchfile"; patch -f -Z -t -p 1 -i patch/$patchfile || exit 2; done)
@@ -607,8 +606,8 @@ Linux server_static_debug debian_amd64_strech build:
   image: amd64/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
     - DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 apache2-utils autoconf autogen automake bison build-essential cmake cpputest default-libmysqlclient-dev fakeroot git libapache2-mod-php libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblua5.2-0 liblua5.2-dev liblzma-dev libogg-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libvorbis-dev libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial mysql-server php php-gd php-imagick php-mysql python3 python3-pip python3-bcrypt python3-venv rrdtool unzip wget zlib1g-dev
     # Apply patch
     - (for patchfile in $(cat patch/series); do echo "patch patch/$patchfile"; patch -f -Z -t -p 1 -i patch/$patchfile || exit 2; done)
@@ -698,8 +697,8 @@ Linux client_static debian_i386_strech build:
   image: i386/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
     - apt-get install -y autoconf autogen automake bison build-essential cmake cpputest fakeroot libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblzma-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial rrdtool unzip wget zlib1g-dev patch
     - apt-get install -y liblua5.2-0 liblua5.2-dev
     # Apply patch
@@ -776,9 +775,9 @@ Linux client_static_debug debian_i386_strech build:
   image: i386/debian:9
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
-    - apt-get install -y autoconf autogen automake bison build-essential cmake cpputest fakeroot libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblzma-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial rrdtool unzip wget zlib1g-dev patch
+    - DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf autogen automake bison build-essential cmake cpputest fakeroot libboost-all-dev libfreetype6-dev libgif-dev libgl1-mesa-dev libjpeg62-turbo-dev liblzma-dev libopenal-dev libpng-dev libssh2-1-dev libssl-dev libtool libtool-bin libx11-dev libxml2-dev libxmu-dev libxrandr-dev libxrender-dev libxxf86vm-dev mercurial rrdtool unzip wget zlib1g-dev patch
     - apt-get install -y liblua5.2-0 liblua5.2-dev
     # Apply patch
     - (for patchfile in $(cat patch/series); do echo "patch patch/$patchfile"; patch -f -Z -t -p 1 -i patch/$patchfile || exit 2; done)
@@ -920,9 +919,9 @@ Linux client ubuntu_amd64_17_10 build:
   image: amd64/ubuntu:17.10
   script:
     # Prepare environment
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
-    - apt-get install -y autoconf
+    - DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf
                          autogen
                          automake
                          bison
@@ -1077,9 +1076,9 @@ Windows client build:
   script:
     # Prepare environment
     - dpkg --add-architecture i386
-    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get update
-    - apt-get dist-upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y
-    - apt-get install -y
+    - DEBIAN_FRONTEND=noninteractive apt-get install -y
                  p7zip-full
                  binfmt-support
                  libc6-i386
@@ -1142,7 +1141,7 @@ Linux client test:
     - mac-ci-build
   tags:
     - Docker
-  image: ubuntu:14.04
+  image: ubuntu:18.04
   script:
     # Test de l'installation initiale
     - ./khanat-$(echo $CI_BUILD_REF | head -c 7 )-$CI_PIPELINE_ID-Linux-x86_64.run
@@ -1172,7 +1171,7 @@ Linux client test:
     - ryzomcore
   tags:
     - Docker
-  image: ubuntu:14.04
+  image: ubuntu:18.04
   script:
     - echo "pas encore de deployment"
   dependencies:
@@ -1192,7 +1191,7 @@ Linux client test:
     - tags
   tags:
     - Docker
-  image: ubuntu:14.04
+  image: ubuntu:18.04
   script:
     - echo "pas encore de tag"
   dependencies:

code/ryzom/tools/patch_gen/CMakeLists.txt

@@ -1,7 +1,7 @@
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip)
 
-SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h)
+SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h patch_gen_common.h)
-SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h)
+SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h patch_gen_common.h)
 
 ADD_EXECUTABLE(patch_gen ${MAIN_SRC})
 TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges)

code/ryzom/tools/patch_gen/patch_gen_common.cpp

@@ -20,6 +20,11 @@
 
 #include <cstdio>
 #include <limits>
+#include <string>
+#include <vector>
+#include <queue>
+#include <mutex>
+#include <thread>
 
 #include "game_share/bnp_patch.h"
 #include "nel/misc/path.h"
@@ -28,12 +33,14 @@
 #include "nel/misc/sstring.h"
 #include "game_share/singleton_registry.h"
 #include "seven_zip.h"
+#include "patch_gen_main.h"
 
 using namespace std;
 using namespace NLMISC;
 
 #define PERSISTENT_TOKEN_FAMILY RyzomTokenFamily
 
+unsigned int jobs_simultaneously = 1;
 
 //-----------------------------------------------------------------------------
 // Handy utility functions
@@ -47,11 +54,11 @@ void normalisePackageDescriptionFileName(std::string& fileName)
 		fileName+=".xml";
 }
 
-void GeneratePatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName)
+void GeneratePatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName)
 {
 	std::string cmd = toString("xdelta delta %s %s %s", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str());
 
-	nlinfo("Executing system command: %s", cmd.c_str());
+	nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str());
 
 #ifdef NL_OS_WINDOWS
 	_spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "delta", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str(), NULL);
@@ -60,15 +67,15 @@ void GeneratePatch(const std::string& srcFileName,const std::string& destFileNam
 	sint error = system (cmd.c_str());
 
 	if (error == 2)
-		nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
+		nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error);
 #endif // NL_OS_WINDOWS
 }
 
-void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string())
+void ApplyPatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string())
 {
 	std::string cmd = toString("xdelta patch %s %s %s", patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str());
 
-	nlinfo("Executing system command: %s", cmd.c_str());
+	nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str());
 
 #ifdef NL_OS_WINDOWS
 	_spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "patch",patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str(), NULL);
@@ -77,22 +84,157 @@ void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,c
 	sint error = system (cmd.c_str());
 
 	if (error == 2)
-		nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
+		nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error);
 #endif // NL_OS_WINDOWS
 }
 
-void GenerateLZMA(const std::string &sourceFile, const std::string &outputFile)
+void GenerateLZMA(const uint32 id, const std::string &sourceFile, const std::string &outputFile)
 {
 	{
-		nlinfo("Compressing %s to %s using LZMA...", sourceFile.c_str(), outputFile.c_str());
+		nlinfo("[id:%d] Compressing %s to %s using LZMA...", id, sourceFile.c_str(), outputFile.c_str());
 	}
 
 	if (!packLZMA(sourceFile, outputFile))
 	{
-		nlwarning("LZMA compress failed");
+		nlwarning("[id:%d, source:%s] LZMA compress failed", id, sourceFile.c_str());
 	}
 }
 
+//-----------------------------------------------------------------------------
+// class CTaskPackageDescription
+//-----------------------------------------------------------------------------
+
+class CTaskPackageDescription
+{
+	bool deleteRefAfterDelta;
+	bool usingTemporaryFile;
+	uint32 id;
+	CBNPFile * _pbnpPtr;
+	std::string _BnpDirectory;
+	std::string _RefDirectory;
+	std::string _PatchDirectory;
+	std::string _RootDirectory;
+	const CBNPCategorySet * _pCategories;
+
+public:
+	CTaskPackageDescription(uint32 id, bool deleteRefAfterDelta, bool usingTemporaryFile, CBNPFile * _pbnpPtr,
+							std::string _BnpDirectory, std::string _RefDirectory, std::string _PatchDirectory,
+							std::string _RootDirectory, const CBNPCategorySet * _pCategories)
+	{
+		this->id = id;
+		this->deleteRefAfterDelta = deleteRefAfterDelta;
+		this->usingTemporaryFile = usingTemporaryFile;
+		this->_pbnpPtr = _pbnpPtr;
+		this->_BnpDirectory = _BnpDirectory;
+		this->_RefDirectory = _RefDirectory;
+		this->_PatchDirectory = _PatchDirectory;
+		this->_RootDirectory = _RootDirectory;
+		this->_pCategories = _pCategories;
+	}
+	void action()
+	{
+		nldebug("[id:%d] thread for %s : start", id, this->_pbnpPtr->getFileName().c_str());
+
+		std::string bnpFileName = _BnpDirectory + _pbnpPtr->getFileName();
+		std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
+		std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
+
+
+		// get the last version number and the related file name
+		const CBNPFileVersion& curVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-1);
+		std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
+		std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch";
+
+		// get the second last version number and the related file name
+		std::string prevVersionFileName;
+		if (_pbnpPtr->versionCount()==1)
+		{
+			prevVersionFileName= _RootDirectory + "empty_" + std::to_string(this->id);
+			CFile::createEmptyFile(prevVersionFileName);
+			usingTemporaryFile = true;
+			deleteRefAfterDelta= false;
+		}
+		else
+		{
+			const CBNPFileVersion& prevVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-2);
+			prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
+		}
+		std::string refVersionFileName= prevVersionFileName;
+
+		// create the subdirectory for this patch number
+		string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber());
+		CFile::createDirectory(versionSubDir);
+
+		// generate the lzma packed version of the bnp if needed (lzma file are slow to generate)
+		string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma";
+		if (!CFile::fileExists(lzmaFile))
+		{
+			// build the lzma compression in a temp file (avoid leaving dirty file if the
+			// process cannot terminate)
+			GenerateLZMA(id, bnpFileName, lzmaFile+".tmp");
+			// rename the tmp file
+			CFile::moveFile(lzmaFile, lzmaFile+".tmp");
+		}
+
+		// store the lzma file size in the descriptor
+		_pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile));
+
+		// if we need to generate a new patch then do it and create the new ref file
+		if (!NLMISC::CFile::fileExists(curVersionFileName))
+		{
+			nlinfo("- Creating patch: %s",patchFileName.c_str());
+
+			// in the case where we compress against a ref file...
+			if (!_pCategories->isFileIncremental(NLMISC::CFile::getFilename(bnpFileName)))
+			{
+				// setup the name of the reference file to patch against
+				refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref";
+
+				// delete the previous patch - because we only need the latest patch for non-incremental files
+				std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch";
+				if (NLMISC::CFile::fileExists(lastPatch.c_str()))
+					NLMISC::CFile::deleteFile(lastPatch.c_str());
+			}
+
+			// call xdelta to generate the patch
+			GeneratePatch(id, refVersionFileName, bnpFileName, patchFileName);
+			nlassert(NLMISC::CFile::fileExists(patchFileName));
+
+			uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName);
+			_pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).setPatchSize(nPatchSize);
+
+			// apply the incremental patch to the old ref file to create the new ref file
+			// and ensure that the new ref file matches the BNP
+			ApplyPatch(id, refVersionFileName, curVersionFileName, patchFileName);
+			nlassert(NLMISC::CFile::fileExists(curVersionFileName));
+			nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName));
+		}
+
+		// if we have a ref file still hanging about from the previous patch then delete it
+		if (NLMISC::CFile::fileExists(prevVersionFileName))
+		{
+			NLMISC::CFile::deleteFile(prevVersionFileName);
+		}
+		nldebug("[id:%d] thread for %s : end", id, this->_pbnpPtr->getFileName().c_str());
+	}
+};
+
+void threadWorker(std::mutex & mtx, std::queue<CTaskPackageDescription> & queueTask)
+{
+	while(true)
+	{
+		mtx.lock();
+		if ( queueTask.empty())
+		{
+			mtx.unlock();
+			break;
+		}
+		CTaskPackageDescription task = queueTask.front();
+		queueTask.pop();
+		mtx.unlock();
+		task.action();
+	}
+};
 
 //-----------------------------------------------------------------------------
 // class CPackageDescription
@@ -125,6 +267,8 @@ public:
 
 	void updatePatchSizes(CBNPFileSet& packageIndex) const;
 
+	//void threadWorker(mutex & mtx, queue<CTaskPackageDescription> & queueTask) const;
+
 	// specialisation of IVersionNumberGenerator
 	void grabVersionNumber();
 	uint32 getPackageVersionNumber();
@@ -334,96 +478,76 @@ void CPackageDescription::generatePatches(CBNPFileSet& packageIndex) const
 {
 	nlinfo("Generating patches ...");
 
-	for (uint32 i = packageIndex.fileCount(); i--;)
+	// Generate patch with Leader/Follower pattern
+	uint32 maxThread=jobs_simultaneously;
+	uint32 i;
+	std::mutex mtx;
+	struct SPrepareTask
+	{
+		uint32 fileSize;
+		uint32 id;
+	};
+	struct SComparePrepareTask
+	{
+		bool operator()(const SPrepareTask& left, const SPrepareTask& right) const
+		{
+			return left.fileSize <  right.fileSize;
+		}
+	};
+	std::priority_queue<struct SPrepareTask, std::vector<struct SPrepareTask>, SComparePrepareTask > queuePrepareTask;
+
+	std::queue<CTaskPackageDescription> queueTask;
+	std::vector<thread> workers;
+
+	// Initialize queue task
+	if ( packageIndex.fileCount() < maxThread )
+		maxThread = packageIndex.fileCount();
+
+	for (i = packageIndex.fileCount(); i--;)
+	{
+		struct SPrepareTask stask;
+		stask.id = i;
+		stask.fileSize = packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).getFileSize();
+		queuePrepareTask.push(stask);
+	}
+
+	// Reorder task Big file to small file
+	//getFileSize
+	std::priority_queue<struct SPrepareTask, std::vector<struct SPrepareTask>, SComparePrepareTask > queueSortedPrepareTask;
+
+
+	while ( ! queuePrepareTask.empty() )
 	{
 		bool deleteRefAfterDelta = true;
 		bool usingTemporaryFile = false;
-		// generate file name root
+
-		std::string bnpFileName = _BnpDirectory + packageIndex.getFile(i).getFileName();
+		struct SPrepareTask task = queuePrepareTask.top();
-		std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
+		queuePrepareTask.pop();
-		std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
 
 		// if the file has no versions then skip on to the next file
-		if (packageIndex.getFile(i).versionCount()==0)
+		if (packageIndex.getFile(task.id).versionCount()==0)
 			continue;
 
-		// get the last version number and the related file name
+		CTaskPackageDescription data(task.id,
-		const CBNPFileVersion& curVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1);
+					     deleteRefAfterDelta,
-		std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
+					     usingTemporaryFile,
-//		std::string patchFileName= patchNameRoot+NLMISC::toString("_%05d.patch",curVersion.getVersionNumber());
+					     & packageIndex.getFile(task.id),
-		std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch";
+					     std::ref(_BnpDirectory),
-
+					     std::ref(_RefDirectory),
-		// get the second last version number and the related file name
+					     std::ref(_PatchDirectory),
-		std::string prevVersionFileName;
+					     std::ref(_RootDirectory),
-		if (packageIndex.getFile(i).versionCount()==1)
+					     & _Categories);
-		{
+		queueTask.push(data);
-			prevVersionFileName= _RootDirectory + "empty";
-			CFile::createEmptyFile(prevVersionFileName);
-			usingTemporaryFile = true;
-			deleteRefAfterDelta= false;
-		}
-		else
-		{
-			const CBNPFileVersion& prevVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-2);
-			prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
-		}
-		std::string refVersionFileName= prevVersionFileName;
-
-		// create the subdirectory for this patch number
-		string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber());
-		CFile::createDirectory(versionSubDir);
-
-		// generate the lzma packed version of the bnp if needed (lzma file are slow to generate)
-		string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma";
-		if (!CFile::fileExists(lzmaFile))
-		{
-			// build the lzma compression in a temp file (avoid leaving dirty file if the
-			// process cannot terminate)
-			GenerateLZMA(bnpFileName, lzmaFile+".tmp");
-			// rename the tmp file
-			CFile::moveFile(lzmaFile, lzmaFile+".tmp");
 	}
 
-		// store the lzma file size in the descriptor
+	// launch thread workers
-		packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile));
+	for(i=0;i<maxThread;++i)
+		workers.push_back(std::thread(threadWorker, std::ref(mtx), std::ref(queueTask)));
 
-		// if we need to generate a new patch then do it and create the new ref file
+	// wait all workers
-		if (!NLMISC::CFile::fileExists(curVersionFileName))
+	for (std::vector<std::thread>::iterator it = workers.begin(); it != workers.end(); ++it)
-		{
+		it->join();
-			nlinfo("- Creating patch: %s",patchFileName.c_str());
+	NLMISC::DebugLog->displayNL("End thread");
-
-			// in the case where we compress against a ref file...
-			if (!_Categories.isFileIncremental(NLMISC::CFile::getFilename(bnpFileName)))
-			{
-				// setup the name of the reference file to patch against
-				refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref";
-
-				// delete the previous patch - because we only need the latest patch for non-incremental files
-				std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch";
-				if (NLMISC::CFile::fileExists(lastPatch.c_str()))
-					NLMISC::CFile::deleteFile(lastPatch.c_str());
-			}
-
-			// call xdelta to generate the patch
-			GeneratePatch(refVersionFileName, bnpFileName, patchFileName);
-			nlassert(NLMISC::CFile::fileExists(patchFileName));
-
-			uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName);
-			packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).setPatchSize(nPatchSize);
-
-			// apply the incremental patch to the old ref file to create the new ref file
-			// and ensure that the new ref file matches the BNP
-			ApplyPatch(refVersionFileName, curVersionFileName, patchFileName);
-			nlassert(NLMISC::CFile::fileExists(curVersionFileName));
-			nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName));
-		}
-
-		// if we have a ref file still hanging about from the previous patch then delete it
-		if (NLMISC::CFile::fileExists(prevVersionFileName))
-		{
-			NLMISC::CFile::deleteFile(prevVersionFileName);
-		}
-	}
 }
 
 void CPackageDescription::createDirectories() const

code/ryzom/tools/patch_gen/patch_gen_common.h

@@ -0,0 +1,7 @@
+#ifndef PATCH_GEN_COMMON_H
+#define PATCH_GEN_COMMON_H
+
+// We need to find other method to send global parameter/argument
+extern unsigned int jobs_simultaneously;
+
+#endif // PATCH_GEN_COMMON_H

code/ryzom/tools/patch_gen/patch_gen_main.cpp

@@ -17,6 +17,8 @@
 #include "nel/misc/debug.h"
 #include "nel/misc/command.h"
 #include "nel/misc/sstring.h"
+#include "nel/misc/cmd_args.h"
+#include "patch_gen_common.h"
 //#include "game_share/handy_commands.h"
 
 //-----------------------------------------------
@@ -26,27 +28,43 @@ int main(int argc,char** argv)
 {
 	NLMISC::createDebug();
 
+	// Parse Command Line.
+	NLMISC::CCmdArgs args;
+
+	args.setDescription("Generate patch");
+	args.addArg("j", "jobs", "jobs", "Specifies the number of jobs (commands) to run simultaneously");
+	args.addAdditionalArg("CommandLine","command line (For a list of valid commands and their paramaters try help)",false,true);
+
 	// if there are no command line args then display a friendly message and exit
-	if (argc==1)
+	if (!args.parse(argc, argv))
 	{
-		NLMISC::InfoLog->displayNL("SYNTAX: %s <command line>",argv[0]);
+		args.displayHelp();
-		NLMISC::InfoLog->displayNL("");
+		return 1;
-		NLMISC::InfoLog->displayNL("For a list of valid commands and their paramaters try: %s help",argv[0]);
+	}
-		NLMISC::InfoLog->displayNL("");
+
-		return 0;
+	// Analyze option
+	try
+	{
+		std::string cjobs = args.haveArg("j") ? args.getArg("j").front() : "1";
+		jobs_simultaneously = (uint32) std::stoul(cjobs);
+	} catch (const std::invalid_argument) {
+		args.displayHelp();
+		NLMISC::ErrorLog->displayNL("Bad value for parameter jobs");
+		return 2;
 	}
 
 	// build the command line by concatnating input arguments (separating with spaces)
+	NLMISC::DebugLog->displayNL("param jobs:%u", jobs_simultaneously);
+	std::vector<std::string> listargs = args.getAdditionalArg("CommandLine");
 	NLMISC::CSString commandline;
-	for (int i=1;i<argc;++i)
+	for (std::vector<std::string>::iterator it = listargs.begin() ; it != listargs.end(); ++it)
 	{
-		NLMISC::CSString s= argv[i];
+		NLMISC::CSString s = *it;
-
 		// check whether the argument needs to be quote encapsulated
 		if (s.contains(' ') && s[0]!='\"')
 			s= "\""+ s+ "\"";
 
-		if (i>1)
+		if ( it != listargs.begin() )
 			commandline+=' ';
 		commandline+= s;
 	}