diff --git a/code/ryzom/tools/patch_gen/CMakeLists.txt b/code/ryzom/tools/patch_gen/CMakeLists.txt index 6cde9810b..d377fb99c 100644 --- a/code/ryzom/tools/patch_gen/CMakeLists.txt +++ b/code/ryzom/tools/patch_gen/CMakeLists.txt @@ -1,7 +1,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip) -SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h) -SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h) +SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h patch_gen_common.h) +SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h patch_gen_common.h) ADD_EXECUTABLE(patch_gen ${MAIN_SRC}) TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) diff --git a/code/ryzom/tools/patch_gen/patch_gen_common.cpp b/code/ryzom/tools/patch_gen/patch_gen_common.cpp index 8ba0d4c5e..9724f08db 100644 --- a/code/ryzom/tools/patch_gen/patch_gen_common.cpp +++ b/code/ryzom/tools/patch_gen/patch_gen_common.cpp @@ -20,6 +20,11 @@ #include #include +#include +#include +#include +#include +#include #include "game_share/bnp_patch.h" #include "nel/misc/path.h" @@ -28,12 +33,14 @@ #include "nel/misc/sstring.h" #include "game_share/singleton_registry.h" #include "seven_zip.h" +#include "patch_gen_main.h" using namespace std; using namespace NLMISC; #define PERSISTENT_TOKEN_FAMILY RyzomTokenFamily +unsigned int jobs_simultaneously = 1; //----------------------------------------------------------------------------- // Handy utility functions @@ -47,11 +54,11 @@ void normalisePackageDescriptionFileName(std::string& fileName) fileName+=".xml"; } -void GeneratePatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName) +void GeneratePatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName) { std::string cmd = toString("xdelta delta %s %s %s", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str()); - nlinfo("Executing system command: %s", cmd.c_str()); + nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str()); #ifdef NL_OS_WINDOWS _spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "delta", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str(), NULL); @@ -60,15 +67,15 @@ void GeneratePatch(const std::string& srcFileName,const std::string& destFileNam sint error = system (cmd.c_str()); if (error == 2) - nlwarning("'%s' failed with error code %d", cmd.c_str(), error); + nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error); #endif // NL_OS_WINDOWS } -void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string()) +void ApplyPatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string()) { std::string cmd = toString("xdelta patch %s %s %s", patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str()); - nlinfo("Executing system command: %s", cmd.c_str()); + nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str()); #ifdef NL_OS_WINDOWS _spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "patch",patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str(), NULL); @@ -77,22 +84,157 @@ void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,c sint error = system (cmd.c_str()); if (error == 2) - nlwarning("'%s' failed with error code %d", cmd.c_str(), error); + nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error); #endif // NL_OS_WINDOWS } -void GenerateLZMA(const std::string &sourceFile, const std::string &outputFile) +void GenerateLZMA(const uint32 id, const std::string &sourceFile, const std::string &outputFile) { { - nlinfo("Compressing %s to %s using LZMA...", sourceFile.c_str(), outputFile.c_str()); + nlinfo("[id:%d] Compressing %s to %s using LZMA...", id, sourceFile.c_str(), outputFile.c_str()); } if (!packLZMA(sourceFile, outputFile)) { - nlwarning("LZMA compress failed"); + nlwarning("[id:%d, source:%s] LZMA compress failed", id, sourceFile.c_str()); } } +//----------------------------------------------------------------------------- +// class CTaskPackageDescription +//----------------------------------------------------------------------------- + +class CTaskPackageDescription +{ + bool deleteRefAfterDelta; + bool usingTemporaryFile; + uint32 id; + CBNPFile * _pbnpPtr; + std::string _BnpDirectory; + std::string _RefDirectory; + std::string _PatchDirectory; + std::string _RootDirectory; + const CBNPCategorySet * _pCategories; + +public: + CTaskPackageDescription(uint32 id, bool deleteRefAfterDelta, bool usingTemporaryFile, CBNPFile * _pbnpPtr, + std::string _BnpDirectory, std::string _RefDirectory, std::string _PatchDirectory, + std::string _RootDirectory, const CBNPCategorySet * _pCategories) + { + this->id = id; + this->deleteRefAfterDelta = deleteRefAfterDelta; + this->usingTemporaryFile = usingTemporaryFile; + this->_pbnpPtr = _pbnpPtr; + this->_BnpDirectory = _BnpDirectory; + this->_RefDirectory = _RefDirectory; + this->_PatchDirectory = _PatchDirectory; + this->_RootDirectory = _RootDirectory; + this->_pCategories = _pCategories; + } + void action() + { + nldebug("[id:%d] thread for %s : start", id, this->_pbnpPtr->getFileName().c_str()); + + std::string bnpFileName = _BnpDirectory + _pbnpPtr->getFileName(); + std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName); + std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName); + + + // get the last version number and the related file name + const CBNPFileVersion& curVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-1); + std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str()); + std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch"; + + // get the second last version number and the related file name + std::string prevVersionFileName; + if (_pbnpPtr->versionCount()==1) + { + prevVersionFileName= _RootDirectory + "empty_" + std::to_string(this->id); + CFile::createEmptyFile(prevVersionFileName); + usingTemporaryFile = true; + deleteRefAfterDelta= false; + } + else + { + const CBNPFileVersion& prevVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-2); + prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str()); + } + std::string refVersionFileName= prevVersionFileName; + + // create the subdirectory for this patch number + string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber()); + CFile::createDirectory(versionSubDir); + + // generate the lzma packed version of the bnp if needed (lzma file are slow to generate) + string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma"; + if (!CFile::fileExists(lzmaFile)) + { + // build the lzma compression in a temp file (avoid leaving dirty file if the + // process cannot terminate) + GenerateLZMA(id, bnpFileName, lzmaFile+".tmp"); + // rename the tmp file + CFile::moveFile(lzmaFile, lzmaFile+".tmp"); + } + + // store the lzma file size in the descriptor + _pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile)); + + // if we need to generate a new patch then do it and create the new ref file + if (!NLMISC::CFile::fileExists(curVersionFileName)) + { + nlinfo("- Creating patch: %s",patchFileName.c_str()); + + // in the case where we compress against a ref file... + if (!_pCategories->isFileIncremental(NLMISC::CFile::getFilename(bnpFileName))) + { + // setup the name of the reference file to patch against + refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref"; + + // delete the previous patch - because we only need the latest patch for non-incremental files + std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch"; + if (NLMISC::CFile::fileExists(lastPatch.c_str())) + NLMISC::CFile::deleteFile(lastPatch.c_str()); + } + + // call xdelta to generate the patch + GeneratePatch(id, refVersionFileName, bnpFileName, patchFileName); + nlassert(NLMISC::CFile::fileExists(patchFileName)); + + uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName); + _pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).setPatchSize(nPatchSize); + + // apply the incremental patch to the old ref file to create the new ref file + // and ensure that the new ref file matches the BNP + ApplyPatch(id, refVersionFileName, curVersionFileName, patchFileName); + nlassert(NLMISC::CFile::fileExists(curVersionFileName)); + nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName)); + } + + // if we have a ref file still hanging about from the previous patch then delete it + if (NLMISC::CFile::fileExists(prevVersionFileName)) + { + NLMISC::CFile::deleteFile(prevVersionFileName); + } + nldebug("[id:%d] thread for %s : end", id, this->_pbnpPtr->getFileName().c_str()); + } +}; + +void threadWorker(std::mutex & mtx, std::queue & queueTask) +{ + while(true) + { + mtx.lock(); + if ( queueTask.empty()) + { + mtx.unlock(); + break; + } + CTaskPackageDescription task = queueTask.front(); + queueTask.pop(); + mtx.unlock(); + task.action(); + } +}; //----------------------------------------------------------------------------- // class CPackageDescription @@ -125,6 +267,8 @@ public: void updatePatchSizes(CBNPFileSet& packageIndex) const; + //void threadWorker(mutex & mtx, queue & queueTask) const; + // specialisation of IVersionNumberGenerator void grabVersionNumber(); uint32 getPackageVersionNumber(); @@ -334,96 +478,76 @@ void CPackageDescription::generatePatches(CBNPFileSet& packageIndex) const { nlinfo("Generating patches ..."); - for (uint32 i = packageIndex.fileCount(); i--;) + // Generate patch with Leader/Follower pattern + uint32 maxThread=jobs_simultaneously; + uint32 i; + std::mutex mtx; + struct SPrepareTask + { + uint32 fileSize; + uint32 id; + }; + struct SComparePrepareTask + { + bool operator()(const SPrepareTask& left, const SPrepareTask& right) const + { + return left.fileSize < right.fileSize; + } + }; + std::priority_queue, SComparePrepareTask > queuePrepareTask; + + std::queue queueTask; + std::vector workers; + + // Initialize queue task + if ( packageIndex.fileCount() < maxThread ) + maxThread = packageIndex.fileCount(); + + for (i = packageIndex.fileCount(); i--;) + { + struct SPrepareTask stask; + stask.id = i; + stask.fileSize = packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).getFileSize(); + queuePrepareTask.push(stask); + } + + // Reorder task Big file to small file + //getFileSize + std::priority_queue, SComparePrepareTask > queueSortedPrepareTask; + + + while ( ! queuePrepareTask.empty() ) { bool deleteRefAfterDelta = true; bool usingTemporaryFile = false; - // generate file name root - std::string bnpFileName = _BnpDirectory + packageIndex.getFile(i).getFileName(); - std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName); - std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName); + + struct SPrepareTask task = queuePrepareTask.top(); + queuePrepareTask.pop(); // if the file has no versions then skip on to the next file - if (packageIndex.getFile(i).versionCount()==0) + if (packageIndex.getFile(task.id).versionCount()==0) continue; - // get the last version number and the related file name - const CBNPFileVersion& curVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1); - std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str()); -// std::string patchFileName= patchNameRoot+NLMISC::toString("_%05d.patch",curVersion.getVersionNumber()); - std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch"; - - // get the second last version number and the related file name - std::string prevVersionFileName; - if (packageIndex.getFile(i).versionCount()==1) - { - prevVersionFileName= _RootDirectory + "empty"; - CFile::createEmptyFile(prevVersionFileName); - usingTemporaryFile = true; - deleteRefAfterDelta= false; - } - else - { - const CBNPFileVersion& prevVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-2); - prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str()); - } - std::string refVersionFileName= prevVersionFileName; - - // create the subdirectory for this patch number - string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber()); - CFile::createDirectory(versionSubDir); - - // generate the lzma packed version of the bnp if needed (lzma file are slow to generate) - string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma"; - if (!CFile::fileExists(lzmaFile)) - { - // build the lzma compression in a temp file (avoid leaving dirty file if the - // process cannot terminate) - GenerateLZMA(bnpFileName, lzmaFile+".tmp"); - // rename the tmp file - CFile::moveFile(lzmaFile, lzmaFile+".tmp"); - } - - // store the lzma file size in the descriptor - packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile)); - - // if we need to generate a new patch then do it and create the new ref file - if (!NLMISC::CFile::fileExists(curVersionFileName)) - { - nlinfo("- Creating patch: %s",patchFileName.c_str()); - - // in the case where we compress against a ref file... - if (!_Categories.isFileIncremental(NLMISC::CFile::getFilename(bnpFileName))) - { - // setup the name of the reference file to patch against - refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref"; - - // delete the previous patch - because we only need the latest patch for non-incremental files - std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch"; - if (NLMISC::CFile::fileExists(lastPatch.c_str())) - NLMISC::CFile::deleteFile(lastPatch.c_str()); - } - - // call xdelta to generate the patch - GeneratePatch(refVersionFileName, bnpFileName, patchFileName); - nlassert(NLMISC::CFile::fileExists(patchFileName)); - - uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName); - packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).setPatchSize(nPatchSize); - - // apply the incremental patch to the old ref file to create the new ref file - // and ensure that the new ref file matches the BNP - ApplyPatch(refVersionFileName, curVersionFileName, patchFileName); - nlassert(NLMISC::CFile::fileExists(curVersionFileName)); - nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName)); - } - - // if we have a ref file still hanging about from the previous patch then delete it - if (NLMISC::CFile::fileExists(prevVersionFileName)) - { - NLMISC::CFile::deleteFile(prevVersionFileName); - } + CTaskPackageDescription data(task.id, + deleteRefAfterDelta, + usingTemporaryFile, + & packageIndex.getFile(task.id), + std::ref(_BnpDirectory), + std::ref(_RefDirectory), + std::ref(_PatchDirectory), + std::ref(_RootDirectory), + & _Categories); + queueTask.push(data); } + + // launch thread workers + for(i=0;i::iterator it = workers.begin(); it != workers.end(); ++it) + it->join(); + NLMISC::DebugLog->displayNL("End thread"); } void CPackageDescription::createDirectories() const diff --git a/code/ryzom/tools/patch_gen/patch_gen_common.h b/code/ryzom/tools/patch_gen/patch_gen_common.h new file mode 100644 index 000000000..5333407e3 --- /dev/null +++ b/code/ryzom/tools/patch_gen/patch_gen_common.h @@ -0,0 +1,7 @@ +#ifndef PATCH_GEN_COMMON_H +#define PATCH_GEN_COMMON_H + +// We need to find other method to send global parameter/argument +extern unsigned int jobs_simultaneously; + +#endif // PATCH_GEN_COMMON_H diff --git a/code/ryzom/tools/patch_gen/patch_gen_main.cpp b/code/ryzom/tools/patch_gen/patch_gen_main.cpp index bc2e6bf66..d45eaff6b 100644 --- a/code/ryzom/tools/patch_gen/patch_gen_main.cpp +++ b/code/ryzom/tools/patch_gen/patch_gen_main.cpp @@ -17,6 +17,8 @@ #include "nel/misc/debug.h" #include "nel/misc/command.h" #include "nel/misc/sstring.h" +#include "nel/misc/cmd_args.h" +#include "patch_gen_common.h" //#include "game_share/handy_commands.h" //----------------------------------------------- @@ -26,27 +28,43 @@ int main(int argc,char** argv) { NLMISC::createDebug(); + // Parse Command Line. + NLMISC::CCmdArgs args; + + args.setDescription("Generate patch"); + args.addArg("j", "jobs", "jobs", "Specifies the number of jobs (commands) to run simultaneously"); + args.addAdditionalArg("CommandLine","command line (For a list of valid commands and their paramaters try help)",false,true); + // if there are no command line args then display a friendly message and exit - if (argc==1) + if (!args.parse(argc, argv)) { - NLMISC::InfoLog->displayNL("SYNTAX: %s ",argv[0]); - NLMISC::InfoLog->displayNL(""); - NLMISC::InfoLog->displayNL("For a list of valid commands and their paramaters try: %s help",argv[0]); - NLMISC::InfoLog->displayNL(""); - return 0; + args.displayHelp(); + return 1; + } + + // Analyze option + try + { + std::string cjobs = args.haveArg("j") ? args.getArg("j").front() : "1"; + jobs_simultaneously = (uint32) std::stoul(cjobs); + } catch (const std::invalid_argument) { + args.displayHelp(); + NLMISC::ErrorLog->displayNL("Bad value for parameter jobs"); + return 2; } // build the command line by concatnating input arguments (separating with spaces) + NLMISC::DebugLog->displayNL("param jobs:%u", jobs_simultaneously); + std::vector listargs = args.getAdditionalArg("CommandLine"); NLMISC::CSString commandline; - for (int i=1;i::iterator it = listargs.begin() ; it != listargs.end(); ++it) { - NLMISC::CSString s= argv[i]; - + NLMISC::CSString s = *it; // check whether the argument needs to be quote encapsulated if (s.contains(' ') && s[0]!='\"') s= "\""+ s+ "\""; - if (i>1) + if ( it != listargs.begin() ) commandline+=' '; commandline+= s; }