djiparsetxt
parseRecord.cpp
Go to the documentation of this file.
1 /**********
2 This program is free software: you can redistribute it and/or modify
3 it under the terms of the GNU General Public License as published by
4 the Free Software Foundation, either version 3 of the License, or
5 (at your option) any later version.
6 
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 GNU General Public License for more details.
11 
12 You should have received a copy of the GNU General Public License
13 along with this program. If not, see <http://www.gnu.org/licenses/>.
14 **********/
15 /*
16  A C++ program to parse DJI's ".txt" log files (recorded by the "DJI Go 4" app).
17  Version 2019-02-08
18 
19  Copyright (c) 2019 Live Networks, Inc. All rights reserved.
20  For the latest version of this program (and more information), visit http://djilogs.live555.com
21 
22  Parsing records within DJI ".txt" files.
23  Implementation.
24 */
25 
27 
28 #include <stdio.h>
29 #include <string.h>
30 
31 #define RECORD_TYPE_OSD 0x01
32 #define RECORD_TYPE_HOME 0x02
33 #define RECORD_TYPE_GIMBAL 0x03
34 #define RECORD_TYPE_RC 0x04
35 #define RECORD_TYPE_CUSTOM 0x05
36 #define RECORD_TYPE_DEFORM 0x06
37 #define RECORD_TYPE_CENTER_BATTERY 0x07
38 #define RECORD_TYPE_SMART_BATTERY 0x08
39 #define RECORD_TYPE_APP_TIP 0x09
40 #define RECORD_TYPE_APP_WARN 0x0A
41 #define RECORD_TYPE_RC_GPS 0x0B
42 #define RECORD_TYPE_RC_DEBUG 0x0C
43 #define RECORD_TYPE_RECOVER 0x0D
44 #define RECORD_TYPE_APP_GPS 0x0E
45 #define RECORD_TYPE_FIRMWARE 0x0F
46 #define RECORD_TYPE_OFDM_DEBUG 0x10
47 #define RECORD_TYPE_VISION_GROUP 0x11
48 #define RECORD_TYPE_VISION_WARN 0x12
49 #define RECORD_TYPE_MC_PARAM 0x13
50 #define RECORD_TYPE_APP_OPERATION 0x14
51 // What is record type 0x16? #####
52 #define RECORD_TYPE_APP_SER_WARN 0x18
53 // What is record type 0x19? #####
54 // What is record type 0x1a? #####
55 // What is record type 0x1e? #####
56 // What is record type 0x28? #####
57 #define RECORD_TYPE_JPEG 0x39
58 #define RECORD_TYPE_OTHER 0xFE
59 
60 #define JPEG_SOI_BYTE 0xD8
61 
62 int RecordAndDetailsParser::parseRecord(u_int8_t const*& ptr, u_int8_t const* limit, int isScrambled) {
63  // Attempt to parse a record; Returns 1 iff it succeeds.
64  try {
65  // The first two bytes are the 'record type' and the 'record length':
66  u_int8_t recordType = getByte(ptr, limit);
67  u_int8_t recordLength = getByte(ptr, limit);
68 
69  // Record statistics about the record type and length:
70  ++fNumRecords;
71  RecordTypeStat& stat = fRecordTypeStats[recordType]; // alias
72  ++stat.count;
74  if (recordLength < stat.minLength) stat.minLength = recordLength;
75  if (recordLength > stat.maxLength) stat.maxLength = recordLength;
76 #ifdef DEBUG_RECORD_PARSING
77  char const* recordTypeName = fRecordTypeName[recordType];
78  if (recordTypeName == NULL) recordTypeName = "???";
79  fprintf(stderr, "[%d]\trecordType %d[%s], recordLength %d\n", fRecordTypeStats[RECORD_TYPE_OSD].count, recordType, recordTypeName, recordLength);
80 #endif
81 
82  if (recordType == RECORD_TYPE_JPEG) {
83  // This record contains one or more JPEG images, and needs to be handled especially.
84  // (In particular, the 'recordLength' seems to be irrelevant in this case)
85  return parseRecord_JPEG(ptr, limit);
86  } else if (recordType == 0xFF && recordLength == JPEG_SOI_BYTE) {
87  // Some old log formats start JPEG images this way. Back up 4 bytes; handle them the same way:
88  ptr -=4;
89  return parseRecord_JPEG(ptr, limit);
90  }
91 
92  // Check the record length, and whether there's a 0xFF byte at the end:
93  if (ptr + recordLength + 1 > limit) throw END_OF_DATA;
94  if (ptr[recordLength] != 0xFF) {
95  fprintf(stderr, "'End of record' byte not seen\n");
96  return 0;
97  }
98  u_int8_t const* recordStart = ptr;
99  u_int8_t const* recordLimit = ptr + recordLength; // position of the 0xFF 'End of record' byte
100  ptr += recordLength + 1; // advance to the next record, if any
101 
102  u_int8_t unscrambledRecord[recordLength-1]; // used only if "isScrambled"
103  if (isScrambled) {
104  // We need to unscramble the record data before we can parse it.
105 
106  // The next byte (along with the 'record type') is an index into the 'scramble table':
107  u_int8_t keyIndexLowByte = getByte(recordStart, limit);
108  u_int16_t scrambleTableIndex = ((recordType-1)<<8)|keyIndexLowByte;
109  --recordLength;
110 
111  if (scrambleTableIndex >= 0x1000) {
112  // Our current 'scramble table' is not large enough to handle this record type. #####
113  fprintf(stderr, "WARNING: for record type 0x%02x", recordType);
114 #ifdef DEBUG_RECORD_PARSING
115  fprintf(stderr, "[%s]", recordTypeName);
116 #endif
117  fprintf(stderr, ", scrambleTableIndex 0x%x is too large (>0x1000) for our current 'scramble table'; we can't unscramble this data!\n", scrambleTableIndex);
118  } else {
119  // Normal case: We know how to unscramble this record's data:
120  extern u_int8_t const scrambleTable[0x1000][8];
121  u_int8_t const* scrambleBytes = scrambleTable[scrambleTableIndex]; // an array of 8 bytes
122 
123  // Unscramble each byte in the record by XORing it with the 'scrambleBytes':
124  for (unsigned i = 0; i < recordLength; ++i) {
125  unscrambledRecord[i] = recordStart[i] ^ scrambleBytes[i%8];
126  }
127  recordStart = unscrambledRecord;
128  recordLimit = unscrambledRecord + recordLength;
129  }
130  }
131 
132  switch (recordType) {
133  case RECORD_TYPE_OSD: {
134  // Because an 'OSD' record effectively starts a new row of data, output a row of data
135  // before we parse it (except for the very first 'OSD' record, where we output
136  // the column labels instead):
138  parseRecord_OSD(recordStart, recordLimit);
139  break;
140  }
141  case RECORD_TYPE_HOME: {
142  parseRecord_HOME(recordStart, recordLimit);
143  break;
144  }
145  case RECORD_TYPE_GIMBAL: {
146  parseRecord_GIMBAL(recordStart, recordLimit);
147  break;
148  }
149  case RECORD_TYPE_RC: {
150  parseRecord_RC(recordStart, recordLimit);
151  break;
152  }
153  case RECORD_TYPE_CUSTOM: {
154  parseRecord_CUSTOM(recordStart, recordLimit);
155  break;
156  }
157  case RECORD_TYPE_DEFORM: {
158  parseRecord_DEFORM(recordStart, recordLimit);
159  break;
160  }
162  parseRecord_CENTER_BATTERY(recordStart, recordLimit);
163  break;
164  }
166  parseRecord_SMART_BATTERY(recordStart, recordLimit);
167  break;
168  }
169  case RECORD_TYPE_APP_TIP: {
170  parseRecord_APP_TIP(recordStart, recordLimit);
171  break;
172  }
173  case RECORD_TYPE_APP_WARN: {
174  parseRecord_APP_WARN(recordStart, recordLimit);
175  break;
176  }
177  case RECORD_TYPE_RC_GPS: {
178  parseRecordUnknownFormat("RC_GPS", recordStart, recordLimit);
179  break;
180  }
181  case RECORD_TYPE_RC_DEBUG: {
182  parseRecordUnknownFormat("RC_DEBUG", recordStart, recordLimit);
183  break;
184  }
185  case RECORD_TYPE_RECOVER: {
186  parseRecord_RECOVER(recordStart, recordLimit);
187  break;
188  }
189  case RECORD_TYPE_APP_GPS: {
190  parseRecord_APP_GPS(recordStart, recordLimit);
191  break;
192  }
193  case RECORD_TYPE_FIRMWARE: {
194  parseRecord_FIRMWARE(recordStart, recordLimit);
195  break;
196  }
197  case RECORD_TYPE_OFDM_DEBUG: {
198  parseRecordUnknownFormat("OFDM_DEBUG", recordStart, recordLimit);
199  break;
200  }
202  parseRecordUnknownFormat("VISION_GROUP", recordStart, recordLimit);
203  break;
204  }
206  parseRecordUnknownFormat("VISION_WARN", recordStart, recordLimit);
207  break;
208  }
209  case RECORD_TYPE_MC_PARAM: {
210  parseRecordUnknownFormat("MC_PARAM", recordStart, recordLimit);
211  break;
212  }
214  parseRecordUnknownFormat("APP_OPERATION", recordStart, recordLimit);
215  break;
216  }
218  parseRecordUnknownFormat("APP_SER_WARN", recordStart, recordLimit);
219  break;
220  }
221  default: {
222 #ifdef DEBUG_RECORD_PARSING
223  char const* recordTypeName = fRecordTypeName[recordType];
224  if (recordTypeName == NULL) {
225  fprintf(stderr, "Unknown record type 0x%02x\n", recordType);
226  } else {
227  fprintf(stderr, "Unhandled record type 0x%02x [%s]\n", recordType, recordTypeName);
228  }
229 #else
230  fprintf(stderr, "Unhandled record type 0x%02x\n", recordType);
231 #endif
232  }
233  }
234  } catch (int /*e*/) {
235  fprintf(stderr, "Unexpected error in parsing\n");
236  return 0;
237  }
238 
239  return 1;
240 }
241 
243 #ifdef DEBUG_RECORD_PARSING
244  fprintf(stderr, "%d records parsed; max num records for one type: %d\n", fNumRecords, fMaxNumRecordsForOneType);
245  unsigned maxRecordTypeFieldLen = 0;
246  for (unsigned i = 0; i < 256; ++i) {
247  char const* recordTypeName = fRecordTypeName[i];
248  unsigned recordTypeNameLen = recordTypeName == NULL ? 3 : strlen(recordTypeName);
249  unsigned iLog10 = i<10 ? 0 : i<100 ? 1 : 2;
250  unsigned recordTypeFieldLen = iLog10 + 2 + recordTypeNameLen + 2;
251  if (recordTypeFieldLen > maxRecordTypeFieldLen) maxRecordTypeFieldLen = recordTypeFieldLen;
252  }
253  unsigned maxNumTabs = maxRecordTypeFieldLen/8 + 1;
254 
255  for (unsigned i = 0; i < 255; ++i) {
256  if (fRecordTypeStats[i].count > 0) {
257  char const* recordTypeName = fRecordTypeName[i];
258  if (recordTypeName == NULL) recordTypeName = "???";
259  unsigned iLog10 = i<10 ? 0 : i<100 ? 1 : 2;
260  unsigned recordTypeFieldLen = iLog10 + 2 + strlen(recordTypeName) + 2;
261  unsigned numTabs = maxNumTabs - recordTypeFieldLen/8; // >0
262 
263  fprintf(stderr, "%d[%s]:", i, recordTypeName);
264  for (unsigned j = 0; j < numTabs; ++j) fprintf(stderr, "\t");
265  fprintf(stderr, "%d\t", fRecordTypeStats[i].count);
266  if (fRecordTypeStats[i].minLength == fRecordTypeStats[i].maxLength) {
267  fprintf(stderr, "length:\t\t%d\n", fRecordTypeStats[i].minLength);
268  } else {
269  fprintf(stderr, "lengths:\t%d-%d\n", fRecordTypeStats[i].minLength, fRecordTypeStats[i].maxLength);
270  }
271  }
272  }
273 #endif
274 }
void parseRecord_HOME(u_int8_t const *&ptr, u_int8_t const *limit)
u_int8_t getByte(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_OSD
Definition: parseRecord.cpp:31
#define RECORD_TYPE_SMART_BATTERY
Definition: parseRecord.cpp:38
#define RECORD_TYPE_VISION_WARN
Definition: parseRecord.cpp:48
RecordTypeStat fRecordTypeStats[256]
void parseRecord_RECOVER(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_APP_GPS(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecordUnknownFormat(char const *recordTypeName, u_int8_t const *&ptr, u_int8_t const *limit)
virtual void summarizeRecordParsing()
#define RECORD_TYPE_APP_TIP
Definition: parseRecord.cpp:39
#define RECORD_TYPE_RC
Definition: parseRecord.cpp:34
void parseRecord_APP_WARN(u_int8_t const *&ptr, u_int8_t const *limit)
virtual int parseRecord(u_int8_t const *&ptr, u_int8_t const *limit, int isScrambled)
Definition: parseRecord.cpp:62
#define RECORD_TYPE_RC_GPS
Definition: parseRecord.cpp:41
#define END_OF_DATA
Definition: DJITxtParser.hh:47
#define RECORD_TYPE_DEFORM
Definition: parseRecord.cpp:36
#define RECORD_TYPE_GIMBAL
Definition: parseRecord.cpp:33
#define RECORD_TYPE_HOME
Definition: parseRecord.cpp:32
void parseRecord_CENTER_BATTERY(u_int8_t const *&ptr, u_int8_t const *limit)
u_int8_t const scrambleTable[0x1000][8]
#define RECORD_TYPE_RC_DEBUG
Definition: parseRecord.cpp:42
#define RECORD_TYPE_APP_OPERATION
Definition: parseRecord.cpp:50
void parseRecord_CUSTOM(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_DEFORM(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_APP_SER_WARN
Definition: parseRecord.cpp:52
#define RECORD_TYPE_FIRMWARE
Definition: parseRecord.cpp:45
void parseRecord_APP_TIP(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_OFDM_DEBUG
Definition: parseRecord.cpp:46
#define RECORD_TYPE_APP_GPS
Definition: parseRecord.cpp:44
#define RECORD_TYPE_APP_WARN
Definition: parseRecord.cpp:40
#define JPEG_SOI_BYTE
Definition: parseRecord.cpp:60
int parseRecord_JPEG(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_RC(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_CENTER_BATTERY
Definition: parseRecord.cpp:37
#define RECORD_TYPE_CUSTOM
Definition: parseRecord.cpp:35
void parseRecord_SMART_BATTERY(u_int8_t const *&ptr, u_int8_t const *limit)
void parseRecord_GIMBAL(u_int8_t const *&ptr, u_int8_t const *limit)
virtual void outputOneRow(int outputColumnLabels)
Definition: rowOutput.cpp:39
#define RECORD_TYPE_VISION_GROUP
Definition: parseRecord.cpp:47
void parseRecord_FIRMWARE(u_int8_t const *&ptr, u_int8_t const *limit)
#define RECORD_TYPE_MC_PARAM
Definition: parseRecord.cpp:49
#define RECORD_TYPE_RECOVER
Definition: parseRecord.cpp:43
#define RECORD_TYPE_JPEG
Definition: parseRecord.cpp:57
void parseRecord_OSD(u_int8_t const *&ptr, u_int8_t const *limit)