update databasexml conversion

README.md

@@ -1,3 +1,43 @@
 # clientbot
 
 Emulate Client (Python Script)
+
+# spykhanat.py
+
+Convert pcap (capture network) on yaml file to see communication between server and client.
+
+## Usage
+
+### Launch network capture
+
+sudo tcpdump -i [networkd card] -w [Pcap output]
+
+ex.: sudo tcpdump -i eth0 -w capture-2020-11-28-17-37-57.pcap
+
+### Extract information
+python3 spykhanat.py -m [localization msg.xml] --yaml [Yaml Output file]  -w [localisation database.xml] -p [Pcap input] --filter-host-service='[Ip address: Port server khaganat]'
+
+Ex.: python3 spykhanat.py -m ~/khanat/khanat-opennel-code/code/ryzom/common/data_common/msg.xml --yaml capture-2020-11-28-17-37-57.yml  -w ~/khanat/khanat-opennel-code/code/ryzom/common/data_common/database.xml -p capture-2020-11-28-17-37-57.pcap --filter-host-service='127.0.0.1:47851'
+
+### Analyze result
+
+you can see the result in yaml output
+
+Field:
+ * packet : raw data
+ * block_Client : data sent by client
+ * block_Server : data sent by server
+ * state : message docoded or partially decoded)
+ * impulse : impulse message
+ * impulseserver : message impulse server decoded
+ * Message : Message analyzed (one line by block)
+ 
+ 
+Detail message format (ex.: <0:31> (Sint32) CurrentSendNumber => 42 : 00000000000000000000000000101010)
+ <Position data> (Type) [Function] => Value : [Value in binary] [(optional) value real]
+ * position data : Begin:End
+ * Format data (Signed/Unsigned Integer, String, Number of bit)
+ * Function (type of value, function in khaganat)
+ * Value : value in integer
+ * Value in binary
+ * Value convert for khaganat (sometimes is keyword)

tools/CGenericMultiPartTemp.py

@@ -66,7 +66,7 @@ class CGenericMultiPartTemp():
                     ret = decodeImpulse.execute(bms, world)
                 except:
                     logging.getLogger(LOGGER).error("CGenericMultiPartTemp : Error to decode - Number:%d len:%d/%d msg:%s"  % (Number, len(self.block), self.NbBlock,  bms.showAllData()))
-                    #return ret
+                 #   #return ret
                     raise ValueError
             logging.getLogger(LOGGER).debug("CGenericMultiPartTemp : data : %s" % bms.showAllData())
             self.MsgDecoded = bms

tools/DecodeDatabase.py

@@ -122,6 +122,10 @@ class LeafDatabase():
         if xmldata.get('count'):
             self.count = int(xmldata.get('count'))
 
+    def loadXmlWithoutCount(self,  xmldata, extra=""):
+        self.name = xmldata.get('name') + extra
+        self.type = xmldata.get('type')
+
     def countLeaves(self):
         if self.count:
             logging.getLogger(LOGGER).debug("countLeaves leaf %s (nb:%s)" % (self.name, str(self.count)))
@@ -131,7 +135,8 @@ class LeafDatabase():
             return 1
 
     def show(self, level=1):
-        logging.getLogger(LOGGER).debug("%s %s Leaf %s : %s : %s" % (" " * level,  str(level), self.name, str(self.count), str(self.type)))
+        #logging.getLogger(LOGGER).debug("%s %s Leaf %s : %s : %s" % ("*" * level,  str(level), self.name, str(self.count), str(self.type)))
+        logging.getLogger(LOGGER).debug("%s %s Leaf %s [%s]" % ("*" * level,  str(level), self.name, str(self.type)))
 
     def execute(self,  msgin,  name=""):
         if name:
@@ -204,23 +209,103 @@ class BranchDatabase():
         self.atom = False
         self.count = None
         
-    def loadXml(self,  xmldata,  filter=None):
+    def loadXmlCount(self,  xmldata, extra=""):
-#        print("xmldata:", xmldata)
+        if  xmldata.get('atom'):
-#        print("keys:", xmldata.keys())
+            self.atom = True
-#        print("filter:", filter)
+        self.name = xmldata.get('name') + extra
-        #print("bank:",  xmldata['bank'])
+        for ele in xmldata:
+            if ele.tag == 'branch':
+                newbranch = BranchDatabase()
+                newbranch.loadXml(ele,  extra)
+                self.branch.append(newbranch)
+            elif ele.tag == 'leaf':
+                newleaf = LeafDatabase()
+                newleaf.loadXml(ele)
+                self.leaf.append(newleaf)
+        
+    def loadXml2(self,  xmldata, extra="",  filter=None):
         if filter:
             if 'bank' in xmldata:
                 if filter != xmldata['bank']:
                     return
             else:
                 return
-        if  xmldata.get('atom'):
-            self.atom = True
         if xmldata.get('count'):
             self.count = int(xmldata.get('count'))
             self.name = xmldata.get('name')
-#        print(self.name)
+            newbranch = BranchDatabase()
+            for i in range(0,  self.count):
+                newbranch.loadXmlCount(xmldata,  str(i))
+            self.branch.append(newbranch)
+        else:
+            self.loadXmlCount(xmldata,  extra)
+
+    def loadXmlWithoutCount(self,  xmldata, extra=""):
+        if  xmldata.get('atom'):
+            self.atom = True
+        self.name = xmldata.get('name') + extra
+        for ele in xmldata:
+            if ele.tag == 'branch':
+                newbranch = BranchDatabase()
+                newbranch.loadXml(ele)
+                self.branch.append(newbranch)
+            elif ele.tag == 'leaf':
+                if ele.get('count'):
+                    count = int(ele.get('count'))
+                    for i in range(0,  count):
+                        newleaf = LeafDatabase()
+                        newleaf.loadXmlWithoutCount(ele,  str(i))
+                        self.leaf.append(newleaf)
+                else:
+                    newleaf = LeafDatabase()
+                    newleaf.loadXmlWithoutCount(ele)
+                    self.leaf.append(newleaf)
+        
+    def loadXml(self,  xmldata, extra="",  filter=None):
+        if filter:
+            if 'bank' in xmldata:
+                if filter != xmldata['bank']:
+                    return
+            else:
+                return
+        self.name = xmldata.get('name') + extra
+        if  xmldata.get('atom'):
+            self.atom = True
+        for ele in xmldata:
+            if ele.get('count'):
+                count = int(ele.get('count'))
+                if ele.tag == 'branch':
+                    for i in range(0,  count):
+                        newbranch = BranchDatabase()
+                        newbranch.loadXml(ele,  str(i))
+                        self.branch.append(newbranch)
+                elif ele.tag == 'leaf':
+                    for i in range(0,  count):
+                        newleaf = LeafDatabase()
+                        newleaf.loadXmlWithoutCount(ele,  str(i))
+                        self.leaf.append(newleaf)
+            else:
+                if ele.tag == 'branch':
+                    newbranch = BranchDatabase()
+                    newbranch.loadXml(ele,  "")
+                    self.branch.append(newbranch)
+                elif ele.tag == 'leaf':
+                    newleaf = LeafDatabase()
+                    newleaf.loadXmlWithoutCount(ele,  "")
+                    self.leaf.append(newleaf)
+                
+    def loadXmlOld(self,  xmldata, extra="",  filter=None):
+        if filter:
+            if 'bank' in xmldata:
+                if filter != xmldata['bank']:
+                    return
+            else:
+                return
+        if xmldata.get('count'):
+            self.count = int(xmldata.get('count'))
+        if  xmldata.get('atom'):
+            self.atom = True
+        self.name = xmldata.get('name')
         for ele in xmldata:
             if ele.tag == 'branch':
                 newbranch = BranchDatabase()
@@ -261,9 +346,9 @@ class BranchDatabase():
             count += ele.countLeaves()
         for ele in self.leaf:
             count += ele.countLeaves()
-        if self.count and self.atom:
+#        if self.count and self.atom:
 #            logging.getLogger(LOGGER).debug("countLeaves branch <- %s (nb:%s)" % (self.name, str(self.count)))
-            count *= self.count
+#            count *= self.count
         return count 
         
     def execute_atom_found(self, level,  pos,  msgin,  name=""):
@@ -294,7 +379,8 @@ class BranchDatabase():
         return level 
 
     def show(self, level=0,  pos=0,  filterlevel=None):
-        logging.getLogger(LOGGER).debug( "%s %s pos:%s Branch:%s : %s : %s - %s" % (" " * level, str(level), str(pos), str(self.name), str(self.count), str(self.atom), str(self.getIdBits())))
+        # logging.getLogger(LOGGER).debug( "%s %s pos:%s Branch:%s : %s : %s - %s" % ("*" * level, str(level), str(pos), str(self.name), str(self.count), str(self.atom), str(self.getIdBits())))
+        logging.getLogger(LOGGER).debug( "%s %s pos:%s Branch:%s atom:%s getIdBits:%s" % ("*" * level, str(level), str(pos), str(self.name), str(self.atom), str(self.getIdBits())))
         if filterlevel is not None:
             if filterlevel <= level:
                 return
@@ -341,9 +427,11 @@ class BranchDatabase():
         #nbBit = getPowerOf2.getPowerOf2_Bis(nbchild)
         nbBit = getPowerOf2.getPowerOf2_ter(nbchild)
         if nbBit > msgin.needRead() :
-            return
+            logging.getLogger(LOGGER).debug("nbBit:" + str(nbBit) + " nbchild:" + str(nbchild) + " needRead:" + str(msgin.needRead() ))
+            return False
         logging.getLogger(LOGGER).debug("needRead:" + str(msgin.needRead()) + " nbBit:" + str(nbBit))
-        id = msgin.readSerial(nbBit, name=parent, typeName='Number:'+str(nbBit), emulate=True)
+        id = msgin.readSerial(nbBit, name=parent, typeName='Uint'+str(nbBit), emulate=True)
+        logging.getLogger(LOGGER).debug("read : needRead:" + str(msgin.needRead()) + " nbBit:" + str(nbBit) + " id:" + str(id))
         i = 0
         ii = 0
         for ele in self.branch:
@@ -363,10 +451,10 @@ class BranchDatabase():
                 else:
                     idnameshort = parent
                 idname = idnameshort + ele.name
-                _= msgin.readSerial(nbBit, name=idname, typeName='Number:'+str(nbBit), emulate=False, commentValue=ele.name+comment)
+                _= msgin.readSerial(nbBit, name=idname, typeName='Uint'+str(nbBit), emulate=False, commentValue=ele.name+comment)
                 logging.getLogger(LOGGER).debug("name:" + ele.name + ", count:" + str(ele.count) + ", atom:" + str(ele.atom))
                 ele.execute(msgin,  idnameshort)
-                return
+                return True
             ii = i
         for ele in self.leaf:
             logging.getLogger(LOGGER).debug(str(i) + " id:" + str(id) + " name:" + str(ele.name))
@@ -385,31 +473,49 @@ class BranchDatabase():
                     idnameshort = parent
                 idname = idnameshort + ele.name + comment
                 logging.getLogger(LOGGER).debug(str(i) + " id:" + str(id) + " name:" + str(ele.name))
-                _ = msgin.readSerial(nbBit, name= idname, typeName='Number:'+str(nbBit), emulate=False, commentValue=ele.name+comment)
+                _ = msgin.readSerial(nbBit, name= idname, typeName='Uint'+str(nbBit), emulate=False, commentValue=ele.name+comment)
                 logging.getLogger(LOGGER).debug("name:" + ele.name + ", count:" + str(ele.count))
                 ele.execute(msgin, name=idnameshort)
-                return
+                return True
             ii = i
         
+        id = msgin.readSerial(nbBit, name=parent, typeName='Uint'+str(nbBit))
+        logging.getLogger(LOGGER).debug("OUT : needRead:" + str(msgin.needRead()) + " nbBit:" + str(nbBit) + " id:" + str(id)+ " i:" + str(ii))
+        print(msgin.showAllData())
+        raise "Oh"
+        return False
+    
     def execute(self, msgin,  parent='DatabaseXML/'):
         if self.atom:
+            logging.getLogger(LOGGER).debug("execute:" + parent)
             self.execute_atom(msgin, parent)
         else:
-            self.execute_normal(msgin,  parent)
+            logging.getLogger(LOGGER).debug("execute:" + parent)
+            ok = self.execute_normal(msgin,  parent)
 
-    def execute_root(self,  msgin):
+    def execute2(self, msgin,  parent='DatabaseXML/'):
+        ok = True
+        while ok:
             if self.atom:
-            nbchild = self.countLeaves()
+                logging.getLogger(LOGGER).debug("execute:" + parent)
-            #nbBit = getPowerOf2.getPowerOf2_Bis(nbchild) 
+                self.execute_atom(msgin, parent)
-            nbBit = getPowerOf2.getPowerOf2(nbchild) 
-            raise "A faire"
             else:
-            nbchild = self.getIdBits()
+                logging.getLogger(LOGGER).debug("execute:" + parent)
-            #nbBit = getPowerOf2.getPowerOf2_Bis(nbchild)
+                ok = self.execute_normal(msgin,  parent)
-            nbBit = getPowerOf2.getPowerOf2(nbchild)
+        
-            while msgin.needRead() > nbBit:
+#    def execute_root(self,  msgin):
-                logging.getLogger(LOGGER).debug("needRead:" + str(msgin.needRead()) + ", nbBit:" + str(nbBit))
+#        if self.atom:
-                self.execute(msgin)
+#            nbchild = self.countLeaves()
+#            #nbBit = getPowerOf2.getPowerOf2_Bis(nbchild) 
+#            nbBit = getPowerOf2.getPowerOf2(nbchild) 
+#            raise "A faire"
+#        else:
+#            nbchild = self.getIdBits()
+#            #nbBit = getPowerOf2.getPowerOf2_Bis(nbchild)
+#            nbBit = getPowerOf2.getPowerOf2(nbchild)
+#            while msgin.needRead() > nbBit:
+#                logging.getLogger(LOGGER).debug("needRead:" + str(msgin.needRead()) + ", nbBit:" + str(nbBit))
+#                self.execute(msgin)
 
 
 class DecodeDatabase():
@@ -420,9 +526,13 @@ class DecodeDatabase():
     def loadDatabase(self, databaseXml):
         self.databasePlr = BranchDatabase()
         self.databasePlr.loadRootXml(databaseXml, 'PLR')
+        self.databasePlr.show()
+        #raise "ok"
     
     def execute(self, msgin, world):
+        logging.getLogger(LOGGER).debug("Start execute")
         self.databasePlr.execute(msgin)
+        logging.getLogger(LOGGER).debug("End execute")
 
     def loadDatabase2(self, databaseXml):
         logging.getLogger(LOGGER).debug("loadDatabase")

tools/Impulse.py

@@ -670,7 +670,7 @@ class impulseDatabaseInitPlayer(ImpulseBase):
             if msgin.needRead() > 5:
                 logging.getLogger(LOGGER).debug(msgin.showAllData())
         logging.getLogger(LOGGER).debug(msgin.showAllData())
-        raise "TODO"
+        #raise "TODO"
 
 
 class ImpulseConnectionDeleteChar(ImpulseBase):