djiparsetxt
parseRecord.cpp
Go to the documentation of this file.
1 /**********
2 This program is free software: you can redistribute it and/or modify
3 it under the terms of the GNU General Public License as published by
4 the Free Software Foundation, either version 3 of the License, or
5 (at your option) any later version.
6 
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 GNU General Public License for more details.
11 
12 You should have received a copy of the GNU General Public License
13 along with this program. If not, see <http://www.gnu.org/licenses/>.
14 **********/
15 /*
16  A C++ program to parse DJI's ".txt" log files (recorded by the "DJI Go 4" app).
17  Version 2021-02-18
18 
19  Copyright (c) 2021 Live Networks, Inc. All rights reserved.
20  For the latest version of this program (and more information), visit http://djilogs.live555.com
21 
22  Parsing records within DJI ".txt" files.
23  Implementation.
24 */
26 #include "scrambleBytes.hh"
27 
28 #include <stdio.h>
29 #include <string.h>
30 
31 #define RECORD_TYPE_OSD 0x01
32 #define RECORD_TYPE_HOME 0x02
33 #define RECORD_TYPE_GIMBAL 0x03
34 #define RECORD_TYPE_RC 0x04
35 #define RECORD_TYPE_CUSTOM 0x05
36 #define RECORD_TYPE_DEFORM 0x06
37 #define RECORD_TYPE_CENTER_BATTERY 0x07
38 #define RECORD_TYPE_SMART_BATTERY 0x08
39 #define RECORD_TYPE_APP_TIP 0x09
40 #define RECORD_TYPE_APP_WARN 0x0A
41 #define RECORD_TYPE_RC_GPS 0x0B
42 #define RECORD_TYPE_RC_DEBUG 0x0C
43 #define RECORD_TYPE_RECOVER 0x0D
44 #define RECORD_TYPE_APP_GPS 0x0E
45 #define RECORD_TYPE_FIRMWARE 0x0F
46 #define RECORD_TYPE_OFDM_DEBUG 0x10
47 #define RECORD_TYPE_VISION_GROUP 0x11
48 #define RECORD_TYPE_VISION_WARN 0x12
49 #define RECORD_TYPE_MC_PARAM 0x13
50 #define RECORD_TYPE_APP_OPERATION 0x14
51 // What is record type 0x16? #####
52 #define RECORD_TYPE_APP_SER_WARN 0x18
53 // What is record type 0x19? #####
54 // What is record type 0x1a? #####
55 // What is record type 0x1e? #####
56 #define RECORD_TYPE_COMPONENT 0x28
57 #define RECORD_TYPE_JPEG 0x39
58 #define RECORD_TYPE_OTHER 0xFE
59 
60 #define JPEG_SOI_BYTE 0xD8
61 
62 int RecordAndDetailsParser::parseRecord(u_int8_t const*& ptr, u_int8_t const* limit, int isScrambled) {
63  // Attempt to parse a record; Returns 1 iff it succeeds.
64  try {
65  // The first two bytes are the 'record type' and the 'record length':
66  u_int8_t recordType = getByte(ptr, limit);
67  unsigned recordLength = getByte(ptr, limit);
68 
69  // Record statistics about the record type and length:
70  ++fNumRecords;
71  RecordTypeStat& stat = fRecordTypeStats[recordType]; // alias
72  ++stat.count;
74  if (recordLength < stat.minLength) stat.minLength = recordLength;
75  if (recordLength > stat.maxLength) stat.maxLength = recordLength;
76 #ifdef DEBUG_RECORD_PARSING
77  char const* recordTypeName = fRecordTypeName[recordType];
78  if (recordTypeName == NULL) recordTypeName = "???";
79  fprintf(stderr, "[%d]\trecordType %d[%s], recordLength %d\n", fRecordTypeStats[RECORD_TYPE_OSD].count, recordType, recordTypeName, recordLength);
80 #endif
81 
82  if (recordType == RECORD_TYPE_JPEG) {
83  u_int16_t next2Bytes = get2BytesBE(ptr, limit); ptr -= 2; // back up over those two bytes
84  if (next2Bytes == 0x0000) {
85  // This record contains one or more JPEG images, and needs to be handled especially.
86  // (In particular, the 'recordLength' seems to be irrelevant in this case)
87  return parseRecord_JPEG(ptr, limit);
88  }
89  } else if (recordType == 0xFF && recordLength == JPEG_SOI_BYTE) {
90  // Some old log formats start JPEG images this way. Back up 4 bytes; handle them the same way:
91  ptr -=4;
92  return parseRecord_JPEG(ptr, limit);
93  }
94 
95  // Check the record length, and whether there's a 0xFF byte at the end:
96  if (ptr + recordLength + 1 >= limit) throw END_OF_DATA;
97  if (ptr[recordLength] != 0xFF) {
98  // In newer versions of the format, some records have an extra byte before the 0xFF. Check:
99  if (ptr + recordLength + 2 < limit && ptr[recordLength+1] == 0xFF) {
100  fprintf(stderr, "Extra byte 0x%02x seen in record (type 0x%x)\n", ptr[recordLength], recordType);
101  ++recordLength;
102  } else {
103  fprintf(stderr, "'End of record' byte (0xFF) not seen; advancing to the next 0xFF\n");
104  do {
105  ++recordLength;
106  } while (ptr + recordLength + 1 < limit && ptr[recordLength+1] != 0xFF);
107  if (ptr + recordLength + 1 >= limit) throw END_OF_DATA;
108  }
109  }
110  u_int8_t const* recordStart = ptr;
111  u_int8_t const* recordLimit = ptr + recordLength; // position of the 0xFF 'End of record' byte
112  ptr += recordLength + 1; // advance to the next record, if any
113 
114  u_int8_t unscrambledRecord[recordLength-1]; // used only if "isScrambled"
115  if (isScrambled) {
116  // We need to unscramble the record data before we can parse it.
117 
118  // The next byte (along with the 'record type') is used as a key to unscramble the data:
119  u_int8_t keyByte = getByte(recordStart, limit);
120  --recordLength;
121 
122  // Get an array of 8 bytes - used to unscramble the data:
123  u_int8_t scrambleBytes[8];
124  getScrambleBytes(recordType, keyByte, scrambleBytes);
125 
126  // Unscramble each byte in the record by XORing it with the 'scrambleBytes':
127  for (unsigned i = 0; i < recordLength; ++i) {
128  unscrambledRecord[i] = recordStart[i] ^ scrambleBytes[i%8];
129  }
130 
131  recordStart = unscrambledRecord;
132  recordLimit = unscrambledRecord + recordLength;
133  }
134 
135  switch (recordType) {
136  case RECORD_TYPE_OSD: {
137  // Because an 'OSD' record effectively starts a new row of data, output a row of data
138  // before we parse it (except for the very first 'OSD' record, where we output
139  // the column labels instead):
141  parseRecord_OSD(recordStart, recordLimit);
142  break;
143  }
144  case RECORD_TYPE_HOME: {
145  parseRecord_HOME(recordStart, recordLimit);
146  break;
147  }
148  case RECORD_TYPE_GIMBAL: {
149  parseRecord_GIMBAL(recordStart, recordLimit);
150  break;
151  }
152  case RECORD_TYPE_RC: {
153  parseRecord_RC(recordStart, recordLimit);
154  break;
155  }
156  case RECORD_TYPE_CUSTOM: {
157  parseRecord_CUSTOM(recordStart, recordLimit);
158  break;
159  }
160  case RECORD_TYPE_DEFORM: {
161  parseRecord_DEFORM(recordStart, recordLimit);
162  break;
163  }
165  parseRecord_CENTER_BATTERY(recordStart, recordLimit);
166  break;
167  }
169  parseRecord_SMART_BATTERY(recordStart, recordLimit);
170  break;
171  }
172  case RECORD_TYPE_APP_TIP: {
173  parseRecord_APP_TIP(recordStart, recordLimit);
174  break;
175  }
176  case RECORD_TYPE_APP_WARN: {
177  parseRecord_APP_WARN(recordStart, recordLimit);
178  break;
179  }
180  case RECORD_TYPE_RC_GPS: {
181  parseRecordUnknownFormat("RC_GPS", recordStart, recordLimit);
182  break;
183  }
184  case RECORD_TYPE_RC_DEBUG: {
185  parseRecordUnknownFormat("RC_DEBUG", recordStart, recordLimit);
186  break;
187  }
188  case RECORD_TYPE_RECOVER: {
189  parseRecord_RECOVER(recordStart, recordLimit);
190  break;
191  }
192  case RECORD_TYPE_APP_GPS: {
193  parseRecord_APP_GPS(recordStart, recordLimit);
194  break;
195  }
196  case RECORD_TYPE_FIRMWARE: {
197  parseRecord_FIRMWARE(recordStart, recordLimit);
198  break;
199  }
200  case RECORD_TYPE_OFDM_DEBUG: {
201  parseRecordUnknownFormat("OFDM_DEBUG", recordStart, recordLimit);
202  break;
203  }
205  parseRecordUnknownFormat("VISION_GROUP", recordStart, recordLimit);
206  break;
207  }
209  parseRecordUnknownFormat("VISION_WARN", recordStart, recordLimit);
210  break;
211  }
212  case RECORD_TYPE_MC_PARAM: {
213  parseRecordUnknownFormat("MC_PARAM", recordStart, recordLimit);
214  break;
215  }
217  parseRecordUnknownFormat("APP_OPERATION", recordStart, recordLimit);
218  break;
219  }
221  parseRecord_APP_SER_WARN(recordStart, recordLimit);
222  break;
223  }
224  case RECORD_TYPE_COMPONENT: {
225  parseRecord_COMPONENT(recordStart, recordLimit);
226  break;
227  }
228  default: {
229 #ifdef DEBUG_RECORD_PARSING
230  char const* recordTypeName = fRecordTypeName[recordType];
231  if (recordTypeName == NULL) {
232  fprintf(stderr, "Unknown record type 0x%02x\n", recordType);
233  } else {
234  fprintf(stderr, "Unhandled record type 0x%02x [%s]\n", recordType, recordTypeName);
235  }
236 #else
237  fprintf(stderr, "Unhandled record type 0x%02x\n", recordType);
238 #endif
239  }
240  }
241  } catch (int /*e*/) {
242  fprintf(stderr, "Unexpected error in parsing\n");
243  return 0;
244  }
245 
246  return 1;
247 }
248 
250 #ifdef DEBUG_RECORD_PARSING
251  fprintf(stderr, "%d records parsed; max num records for one type: %d\n", fNumRecords, fMaxNumRecordsForOneType);
252  unsigned maxRecordTypeFieldLen = 0;
253  for (unsigned i = 0; i < 256; ++i) {
254  char const* recordTypeName = fRecordTypeName[i];
255  unsigned recordTypeNameLen = recordTypeName == NULL ? 3 : strlen(recordTypeName);
256  unsigned iLog10 = i<10 ? 0 : i<100 ? 1 : 2;
257  unsigned recordTypeFieldLen = iLog10 + 2 + recordTypeNameLen + 2;
258  if (recordTypeFieldLen > maxRecordTypeFieldLen) maxRecordTypeFieldLen = recordTypeFieldLen;
259  }
260  unsigned maxNumTabs = maxRecordTypeFieldLen/8 + 1;
261 
262  for (unsigned i = 0; i < 255; ++i) {
263  if (fRecordTypeStats[i].count > 0) {
264  char const* recordTypeName = fRecordTypeName[i];
265  if (recordTypeName == NULL) recordTypeName = "???";
266  unsigned iLog10 = i<10 ? 0 : i<100 ? 1 : 2;
267  unsigned recordTypeFieldLen = iLog10 + 2 + strlen(recordTypeName) + 2;
268  unsigned numTabs = maxNumTabs - recordTypeFieldLen/8; // >0
269 
270  fprintf(stderr, "%d[%s]:", i, recordTypeName);
271  for (unsigned j = 0; j < numTabs; ++j) fprintf(stderr, "\t");
272  fprintf(stderr, "%d\t", fRecordTypeStats[i].count);
273  if (fRecordTypeStats[i].minLength == fRecordTypeStats[i].maxLength) {
274  fprintf(stderr, "length:\t\t%d\n", fRecordTypeStats[i].minLength);
275  } else {
276  fprintf(stderr, "lengths:\t%d-%d\n", fRecordTypeStats[i].minLength, fRecordTypeStats[i].maxLength);
277  }
278  }
279  }
280 #endif
281 }
void parseRecord_HOME(u_int8_t const *&ptr, u_int8_t const *limit)
u_int8_t getByte(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_OSD
Definition: parseRecord.cpp:31
#define RECORD_TYPE_SMART_BATTERY
Definition: parseRecord.cpp:38
#define RECORD_TYPE_VISION_WARN
Definition: parseRecord.cpp:48
RecordTypeStat fRecordTypeStats[256]
void parseRecord_RECOVER(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_APP_GPS(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecordUnknownFormat(char const *recordTypeName, u_int8_t const *&ptr, u_int8_t const *limit)
virtual void summarizeRecordParsing()
#define RECORD_TYPE_APP_TIP
Definition: parseRecord.cpp:39
void getScrambleBytes(u_int8_t recordType, u_int8_t keyByte, u_int8_t *resultScrambleBytes)
#define RECORD_TYPE_COMPONENT
Definition: parseRecord.cpp:56
#define RECORD_TYPE_RC
Definition: parseRecord.cpp:34
void parseRecord_APP_WARN(u_int8_t const *&ptr, u_int8_t const *limit)
virtual int parseRecord(u_int8_t const *&ptr, u_int8_t const *limit, int isScrambled)
Definition: parseRecord.cpp:62
#define RECORD_TYPE_RC_GPS
Definition: parseRecord.cpp:41
#define END_OF_DATA
Definition: DJITxtParser.hh:47
#define RECORD_TYPE_DEFORM
Definition: parseRecord.cpp:36
#define RECORD_TYPE_GIMBAL
Definition: parseRecord.cpp:33
void parseRecord_COMPONENT(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_HOME
Definition: parseRecord.cpp:32
void parseRecord_CENTER_BATTERY(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_RC_DEBUG
Definition: parseRecord.cpp:42
#define RECORD_TYPE_APP_OPERATION
Definition: parseRecord.cpp:50
void parseRecord_CUSTOM(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_DEFORM(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_APP_SER_WARN
Definition: parseRecord.cpp:52
#define RECORD_TYPE_FIRMWARE
Definition: parseRecord.cpp:45
void parseRecord_APP_TIP(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_OFDM_DEBUG
Definition: parseRecord.cpp:46
#define RECORD_TYPE_APP_GPS
Definition: parseRecord.cpp:44
#define RECORD_TYPE_APP_WARN
Definition: parseRecord.cpp:40
#define JPEG_SOI_BYTE
Definition: parseRecord.cpp:60
int parseRecord_JPEG(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_RC(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_CENTER_BATTERY
Definition: parseRecord.cpp:37
void parseRecord_APP_SER_WARN(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_CUSTOM
Definition: parseRecord.cpp:35
void parseRecord_SMART_BATTERY(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_GIMBAL(u_int8_t const *&ptr, u_int8_t const *limit)
virtual void outputOneRow(int outputColumnLabels)
Definition: rowOutput.cpp:39
#define RECORD_TYPE_VISION_GROUP
Definition: parseRecord.cpp:47
void parseRecord_FIRMWARE(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_MC_PARAM
Definition: parseRecord.cpp:49
#define RECORD_TYPE_RECOVER
Definition: parseRecord.cpp:43
#define RECORD_TYPE_JPEG
Definition: parseRecord.cpp:57
u_int16_t get2BytesBE(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_OSD(u_int8_t const *&ptr, u_int8_t const *limit)