update spykhanat, adding other output CSV file

README.md

@@ -15,9 +15,9 @@ sudo tcpdump -i [networkd card] -w [Pcap output]
 ex.: sudo tcpdump -i eth0 -w capture-2020-11-28-17-37-57.pcap
 
 ### Extract information
-python3 spykhanat.py -m [localization msg.xml] --yaml [Yaml Output file]  -w [localisation database.xml] -p [Pcap input] --filter-host-service='[Ip address: Port server khaganat]'
+python3 spykhanat.py -m [localization msg.xml] --yaml [Yaml Output file]  -w [localisation database.xml] -p [Pcap input] --filter-host-service='[Ip address: Port server khaganat]' --csv='[file output CSV {comma separator} - extract only normal message]'
 
-Ex.: python3 spykhanat.py -m ~/khanat/khanat-opennel-code/code/ryzom/common/data_common/msg.xml --yaml capture-2020-11-28-17-37-57.yml  -w ~/khanat/khanat-opennel-code/code/ryzom/common/data_common/database.xml -p capture-2020-11-28-17-37-57.pcap --filter-host-service='127.0.0.1:47851'
+Ex.: python3 spykhanat.py -m ~/khanat/khanat-opennel-code/code/ryzom/common/data_common/msg.xml --yaml capture-2020-11-28-17-37-57.yml  -w ~/khanat/khanat-opennel-code/code/ryzom/common/data_common/database.xml -p capture-2020-11-28-17-37-57.pcap --filter-host-service='127.0.0.1:47851' --csv capture-2020-11-28-17-37-57.csv
 
 ### Analyze result
 

spykhanat.py

@@ -77,7 +77,7 @@ def write_yaml_str_or_array(outyaml,  nbspace,  value):
     outyaml.write(" " * nbspace + str(type(value)) + "\n")
 
 class SpyPcap():
-    def __init__(self, khanat_host_service, pcap_file, msg_xml, database_xml, filter_host_service, show_raw_packet, show_message_decoded, outyaml=None):
+    def __init__(self, khanat_host_service, pcap_file, msg_xml, database_xml, filter_host_service, show_raw_packet, show_message_decoded, outyaml=None,  outcsv=False):
         if khanat_host_service:
             self.khanat_host_service = re.compile(khanat_host_service)
         else:
@@ -113,6 +113,7 @@ class SpyPcap():
         self.decodeImpulseSimple.loadDatabase(self.decodeDatabase)
         # outyaml
         self.outyaml = outyaml
+        self.outcsv = outcsv
 
     def readRaw(self):
         file = open( self.pcap_file , 'rb')
@@ -435,7 +436,7 @@ class SpyPcap():
             self.client_state[clientid]['AckBitMask'] = AckBitMask
             logging.getLogger(LOGGER).info("[Client -> Server] Normal Mode {CurrentReceivedNumber:%d, src:%s, dst:%s, LastReceivedAck:%d}" % (CurrentReceivedNumber, clientid, dst, LastReceivedAck))
             # self.decode_server(msgin, _CurrentReceivedNumber, _CurrentReceivedNumber-1)
-            actions, _ = self.decode_client_send_normal_message(msgin, clientid, dst, sequenceid, "%s_%d" % (target, 0), Parent)
+            actions, impulses = self.decode_client_send_normal_message(msgin, clientid, dst, sequenceid, "%s_%d" % (target, 0), Parent)
         else:
             message = msgin.readUint8('message')
             try:
@@ -708,7 +709,8 @@ class SpyPcap():
         sequencenum = 1
         if self.outyaml:
             self.outyaml.write("# Generated : %s\n\n" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
-
+        if self.outcsv:
+            self.outcsv.write("Date,Packet Id,Source,Destination,Key,Value\n")
         for pkt in pcapfile.packets:
             eth_frame = ethernet.Ethernet(pkt.raw())
             if eth_frame.type == 2048:
@@ -783,21 +785,67 @@ class SpyPcap():
                             havedata = True
                         else:
                             havedata = False
+                        if not havedata:
+                            for action in actions_servers:
+                                if action.get_notice():
+                                    havedata = True
+                                    break
+                        if not havedata:
+                            for action in actions_clients:
+                                if action.get_notice():
+                                    havedata = True
+                                    break
                         if not havedata:
                             for impulse_data in impulses_servers:
                                 if impulse_data.get_notice():
                                     havedata = True
                                     break
+                        if not havedata:
+                            for impulse_data in impulses_clients:
+                                if impulse_data.get_notice():
+                                    havedata = True
+                                    break
                         if havedata:
-                            print(datetime.fromtimestamp(pkt.timestamp).strftime("%Y/%m/%d %H:%M:%S"),  _provenance,  "(", list_host[src], "=>", list_host[dst], ") [",  Reference,  "] ")
+                            print(datetime.fromtimestamp(pkt.timestamp).strftime("%Y/%m/%d %H:%M:%S"), "[",  Reference,  "]", list_host[src], "->", list_host[dst])
                             if importantinfo:
                                 for key in importantinfo:
                                     print(" " * 3, key, ":", importantinfo[key])
-                            if impulses_servers:
+                            for action in actions_servers:
+                                data = action.get_notice()
+                                for key in data:
+                                    print(" " * 3, key, ":", data[key])
+                            for action in actions_clients:
+                                data = action.get_notice()
+                                for key in data:
+                                    print(" " * 3, key, ":", data[key])
                             for impulse_data in impulses_servers:
                                 data = impulse_data.get_notice()
                                 for key in data:
                                     print(" " * 3, key, ":", data[key])
+                            for impulse_data in impulses_clients:
+                                data = impulse_data.get_notice()
+                                for key in data:
+                                    print(" " * 3, key, ":", data[key])
+                            print("")
+                            if self.outcsv:
+                                for key in importantinfo:
+                                    self.outcsv.write("%s,%s,%s,%s,%s,%s\n" % (datetime.fromtimestamp(pkt.timestamp).strftime("%Y/%m/%d %H:%M:%S"), Reference,  list_host[src], list_host[dst], key,  importantinfo[key]))
+                                for action in actions_servers:
+                                    data = action.get_notice()
+                                    for key in data:
+                                        self.outcsv.write("%s,%s,%s,%s,%s,%s\n" % (datetime.fromtimestamp(pkt.timestamp).strftime("%Y/%m/%d %H:%M:%S"), Reference,  list_host[src], list_host[dst], key,  data[key]))
+                                for action in actions_clients:
+                                    data = action.get_notice()
+                                    for key in data:
+                                        self.outcsv.write("%s,%s,%s,%s,%s,%s\n" % (datetime.fromtimestamp(pkt.timestamp).strftime("%Y/%m/%d %H:%M:%S"), Reference,  list_host[src], list_host[dst], key,  data[key]))
+                                for impulse_data in impulses_servers:
+                                    data = impulse_data.get_notice()
+                                    for key in data:
+                                        self.outcsv.write("%s,%s,%s,%s,%s,%s\n" % (datetime.fromtimestamp(pkt.timestamp).strftime("%Y/%m/%d %H:%M:%S"), Reference,  list_host[src], list_host[dst], key,  data[key]))
+                                for impulse_data in impulses_clients:
+                                    data = impulse_data.get_notice()
+                                    for key in data:
+                                        self.outcsv.write("%s,%s,%s,%s,%s,%s\n" % (datetime.fromtimestamp(pkt.timestamp).strftime("%Y/%m/%d %H:%M:%S"), Reference,  list_host[src], list_host[dst], key,  data[key]))
 
                         if self.outyaml:
                             self.outyaml.write("\n%s:\n    sequence: %d\n    time: %s\n    source: %s\n    destination: %s\n    function: %s\n    adress_source: %s\n    adress_destination: %s\n    state: %s\n    message:\n" % (
@@ -961,8 +1009,10 @@ class SpyPcap():
             logging.getLogger(LOGGER).debug("%s [server tick:%d, client tick:%d]" %(client, self.client_state[client]['CurrentSendNumber'], self.client_state[client]['CurrentReceivedNumber']))
         if fullconverted:
             logging.getLogger(LOGGER).info("Full converted")
+            print("\nEnd : Full converted")
         else:
             logging.getLogger(LOGGER).info("Partially converted")
+            print("\nEnd : Partially converted")
         logging.getLogger(LOGGER).info("Conversion => End")
 
 
@@ -996,6 +1046,7 @@ def main():
     parser.add_argument("-w", "--database-xml", help="file database.xml (from server khanat)", required=True)
     parser.add_argument("-r", "--raw", help="show message raw", action='store_true')
     parser.add_argument("--yaml", help="generate YAML file (decode all message)", type=argparse.FileType('w'),  default=None)
+    parser.add_argument("--csv", help="generate CSV file (essential message)", type=argparse.FileType('w'),  default=None)
     parser.add_argument("--show-raw-packet", help="show packet (raw data)", action='store_true')
     parser.add_argument("--show-message-decoded", help="show packet (raw data)", action='store_true')
 
@@ -1020,7 +1071,8 @@ def main():
                   filter_host_service=args.filter_host_service,
                   show_raw_packet=args.show_raw_packet,
                   show_message_decoded=args.show_message_decoded,
-                  outyaml=args.yaml)
+                  outyaml=args.yaml, 
+                  outcsv=args.csv)
     if args.raw:
         spy.readRaw()
     else:

tools/CAction.py

@@ -67,6 +67,25 @@ class CAction:
         self.world = world
         self.Reference = []
         self.Name = ""
+        self.notice = {}
+        self.headernotice = None
+
+    def set_header_notice(self, header):
+        self.headernotice = header
+
+    def append_notice(self, data):
+        for key in data:
+            #print("Add",  key)
+            self.notice.setdefault(key, data [key])
+
+    def add_notice(self, id, value):
+        if not self.headernotice:
+            return
+        ref = { self.headernotice + '/' +id:  value}
+        self.append_notice(ref)
+
+    def get_notice(self):
+        return self.notice
 
     def set_name(self, name):
         self.Name = name
@@ -154,6 +173,7 @@ class CActionPosition(CAction):
         self.Position16 = [0, 0, 0]
         self.IsRelative = False
         self.Interior = False
+        self.set_header_notice("Action/Position")
 
     def __str__(self):
         return "CActionPosition" + super().__str__() + " x:" + str(self.Position16[0]) + " y:" + str(self.Position16[1]) + " z:" + str(self.Position16[2]) + " IsRelative:" + str(self.IsRelative) + " Interior:" + str(self.Interior)
@@ -173,6 +193,11 @@ class CActionPosition(CAction):
         self.Position16[2] = message.readUint16('pz')
         self.IsRelative = (self.Position16[2] & 0x1) != 0
         self.Interior = (self.Position16[2] & 0x2) != 0
+        self.add_notice('px', self.Position16[0] )
+        self.add_notice('py', self.Position16[1] )
+        self.add_notice('pz', self.Position16[2] )
+        self.add_notice('IsRelative', self.Position16[2] & 0x1 )
+        self.add_notice('Interior', self.Position16[2] & 0x2 )
 
 #   message.serialAndLog1( Position16[0] );
 #   message.serialAndLog1( Position16[1] );
@@ -576,6 +601,9 @@ class CActionFake():
     def get_name(self):
         return self.Name
 
+    def get_notice(self):
+        return {}
+
     def get_parameter(self):
         ret = {"Type": "CActionFake.%s" % self.Type}
         if not self._Message.checkOnlyZeroAtEnd():

tools/Impulse.py

@@ -469,7 +469,7 @@ class ImpulsePosition(ImpulseBase):
         # khanat-opennel-code/code/ryzom/server/src/gpm_service/client_messages.cpp void cbClientPosition( CMessage& msgin, const string &serviceName, NLNET::TServiceId serviceId )
         logging.getLogger(LOGGER).debug("read")
         self.name = name.replace(':',  '_')
-        self.readBool(msgin, '')
+        #self.readBool(msgin, '')
         self.readSint32(msgin, 'X')
         self.readSint32(msgin, 'Y')
         self.readSint32(msgin, 'Z')