parseRecord.cpp

Go to the documentation of this file.

/**********
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
**********/
/*
    A C++ program to parse DJI's ".txt" log files (recorded by the "DJI Go 4" app).
    Version 2021-05-20

    Copyright (c) 2021 Live Networks, Inc.  All rights reserved.
    For the latest version of this program (and more information), visit http://djilogs.live555.com

    Parsing records within DJI ".txt" files.
    Implementation.
*/
#include "RecordAndDetailsParser.hh"
#include "scrambleBytes.hh"

#include <stdio.h>
#include <string.h>

#define RECORD_TYPE_OSD 0x01
#define RECORD_TYPE_HOME 0x02
#define RECORD_TYPE_GIMBAL 0x03
#define RECORD_TYPE_RC 0x04
#define RECORD_TYPE_CUSTOM 0x05
#define RECORD_TYPE_DEFORM 0x06
#define RECORD_TYPE_CENTER_BATTERY 0x07
#define RECORD_TYPE_SMART_BATTERY 0x08
#define RECORD_TYPE_APP_TIP 0x09
#define RECORD_TYPE_APP_WARN 0x0A
#define RECORD_TYPE_RC_GPS 0x0B
#define RECORD_TYPE_RC_DEBUG 0x0C
#define RECORD_TYPE_RECOVER 0x0D
#define RECORD_TYPE_APP_GPS 0x0E
#define RECORD_TYPE_FIRMWARE 0x0F
#define RECORD_TYPE_OFDM_DEBUG 0x10
#define RECORD_TYPE_VISION_GROUP 0x11
#define RECORD_TYPE_VISION_WARN 0x12
#define RECORD_TYPE_MC_PARAM 0x13
#define RECORD_TYPE_APP_OPERATION 0x14
// What is record type 0x16? #####
#define RECORD_TYPE_APP_SER_WARN 0x18
// What is record type 0x19? #####
// What is record type 0x1a? #####
// What is record type 0x1e? #####
#define RECORD_TYPE_COMPONENT 0x28
#define RECORD_TYPE_JPEG 0x39
#define RECORD_TYPE_OTHER 0xFE

#define JPEG_SOI_BYTE 0xD8

int RecordAndDetailsParser::parseRecord(u_int8_t const*& ptr, u_int8_t const* limit, int isScrambled) {
  // Attempt to parse a record; Returns 1 iff it succeeds.
  try {
    // The first two bytes are the 'record type' and the 'record length':
    u_int8_t recordType = getByte(ptr, limit);
    unsigned recordLength = getByte(ptr, limit);

    // Record statistics about the record type and length:
    ++fNumRecords;
    RecordTypeStat& stat = fRecordTypeStats[recordType]; // alias
    ++stat.count;
    if (stat.count > fMaxNumRecordsForOneType) fMaxNumRecordsForOneType = stat.count;
    if (recordLength < stat.minLength) stat.minLength = recordLength;
    if (recordLength > stat.maxLength) stat.maxLength = recordLength;
#ifdef DEBUG_RECORD_PARSING
    char const* recordTypeName = fRecordTypeName[recordType];
    if (recordTypeName == NULL) recordTypeName = "???";
    fprintf(stderr, "[%d]\trecordType %d[%s], recordLength %d\n", fRecordTypeStats[RECORD_TYPE_OSD].count, recordType, recordTypeName, recordLength);
#endif

    if (recordType == RECORD_TYPE_JPEG) {
      u_int16_t next2Bytes = get2BytesBE(ptr, limit); ptr -= 2; // back up over those two bytes
      if (next2Bytes == 0x0000) {
        // This record contains one or more JPEG images, and needs to be handled especially.
        // (In particular, the 'recordLength' seems to be irrelevant in this case)
        return parseRecord_JPEG(ptr, limit);
      }
    } else if (recordType == 0xFF && recordLength == JPEG_SOI_BYTE) {
      // Some old log formats start JPEG images this way.  Back up 4 bytes; handle them the same way:
      ptr -=4;
      return parseRecord_JPEG(ptr, limit);
    }

    // Check the record length, and whether there's a 0xFF byte at the end:
    if (ptr + recordLength + 1 >= limit) throw END_OF_DATA;
    if (ptr[recordLength] != 0xFF) {
      // In newer versions of the format, some records have an extra byte before the 0xFF.  Check:
      if (ptr + recordLength + 2 < limit && ptr[recordLength+1] == 0xFF) {
        fprintf(stderr, "Extra byte 0x%02x seen in record (type 0x%x)\n", ptr[recordLength], recordType);
        ++recordLength;
      } else {
        fprintf(stderr, "'End of record' byte (0xFF) not seen; advancing to the next 0xFF\n");
        do {
          ++recordLength;
        } while (ptr + recordLength + 1 < limit && ptr[recordLength+1] != 0xFF);
        if (ptr + recordLength + 1 >= limit) throw END_OF_DATA;
      }
    }
    u_int8_t const* recordStart = ptr;
    u_int8_t const* recordLimit = ptr + recordLength; // position of the 0xFF 'End of record' byte
    ptr += recordLength + 1; // advance to the next record, if any

    u_int8_t unscrambledRecord[recordLength-1]; // used only if "isScrambled"
    if (isScrambled) {
      // We need to unscramble the record data before we can parse it.

      // The next byte (along with the 'record type') is used as a key to unscramble the data:
      u_int8_t keyByte = getByte(recordStart, limit);
      --recordLength;

      // Get an array of 8 bytes - used to unscramble the data:
      u_int8_t scrambleBytes[8];
      getScrambleBytes(recordType, keyByte, scrambleBytes);

      // Unscramble each byte in the record by XORing it with the 'scrambleBytes':
      for (unsigned i = 0; i < recordLength; ++i) {
        unscrambledRecord[i] = recordStart[i] ^ scrambleBytes[i%8];
      }

      recordStart = unscrambledRecord;
      recordLimit = unscrambledRecord + recordLength;
    }

    switch (recordType) {
      case RECORD_TYPE_OSD: {
        // Because an 'OSD' record effectively starts a new row of data, output a row of data
        // before we parse it (except for the very first 'OSD' record, where we output
        // the column labels instead):
        outputOneRow(fRecordTypeStats[RECORD_TYPE_OSD].count == 1);
        parseRecord_OSD(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_HOME: {
        parseRecord_HOME(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_GIMBAL: {
        parseRecord_GIMBAL(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_RC: {
        parseRecord_RC(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_CUSTOM: {
        parseRecord_CUSTOM(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_DEFORM: {
        parseRecord_DEFORM(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_CENTER_BATTERY: {
        parseRecord_CENTER_BATTERY(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_SMART_BATTERY: {
        parseRecord_SMART_BATTERY(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_APP_TIP: {
        parseRecord_APP_TIP(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_APP_WARN: {
        parseRecord_APP_WARN(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_RC_GPS: {
        parseRecordUnknownFormat("RC_GPS", recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_RC_DEBUG: {
        parseRecordUnknownFormat("RC_DEBUG", recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_RECOVER: {
        parseRecord_RECOVER(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_APP_GPS: {
        parseRecord_APP_GPS(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_FIRMWARE: {
        parseRecord_FIRMWARE(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_OFDM_DEBUG: {
        parseRecordUnknownFormat("OFDM_DEBUG", recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_VISION_GROUP: {
        parseRecordUnknownFormat("VISION_GROUP", recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_VISION_WARN: {
        parseRecordUnknownFormat("VISION_WARN", recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_MC_PARAM: {
        parseRecordUnknownFormat("MC_PARAM", recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_APP_OPERATION: {
        parseRecordUnknownFormat("APP_OPERATION", recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_APP_SER_WARN: {
        parseRecord_APP_SER_WARN(recordStart, recordLimit);
        break;
      }
      case RECORD_TYPE_COMPONENT: {
        parseRecord_COMPONENT(recordStart, recordLimit);
        break;
      }
      default: {
#ifdef DEBUG_RECORD_PARSING
        char const* recordTypeName = fRecordTypeName[recordType];
        if (recordTypeName == NULL) {
          fprintf(stderr, "Unknown record type 0x%02x\n", recordType);
        } else {
          fprintf(stderr, "Unhandled record type 0x%02x [%s]\n", recordType, recordTypeName);
        }
#else
        fprintf(stderr, "Unhandled record type 0x%02x\n", recordType);
#endif
      }
    }
  } catch (int /*e*/) {
    fprintf(stderr, "Unexpected error in parsing\n");
    return 0;
  }

  return 1;
}

void RecordAndDetailsParser::summarizeRecordParsing() {
#ifdef DEBUG_RECORD_PARSING
  fprintf(stderr, "%d records parsed; max num records for one type: %d\n", fNumRecords, fMaxNumRecordsForOneType);
  unsigned maxRecordTypeFieldLen = 0;
  for (unsigned i = 0; i < 256; ++i) {
    char const* recordTypeName = fRecordTypeName[i];
    unsigned recordTypeNameLen = recordTypeName == NULL ? 3 : strlen(recordTypeName);
    unsigned iLog10 = i<10 ? 0 : i<100 ? 1 : 2;
    unsigned recordTypeFieldLen = iLog10 + 2 + recordTypeNameLen + 2;
    if (recordTypeFieldLen > maxRecordTypeFieldLen) maxRecordTypeFieldLen = recordTypeFieldLen;
  }
  unsigned maxNumTabs = maxRecordTypeFieldLen/8 + 1;

  for (unsigned i = 0; i < 255; ++i) {
    if (fRecordTypeStats[i].count > 0) {
      char const* recordTypeName = fRecordTypeName[i];
      if (recordTypeName == NULL) recordTypeName = "???";
      unsigned iLog10 = i<10 ? 0 : i<100 ? 1 : 2;
      unsigned recordTypeFieldLen = iLog10 + 2 + strlen(recordTypeName) + 2;
      unsigned numTabs = maxNumTabs - recordTypeFieldLen/8; // >0

      fprintf(stderr, "%d[%s]:", i, recordTypeName);
      for (unsigned j = 0; j < numTabs; ++j) fprintf(stderr, "\t");
      fprintf(stderr, "%d\t", fRecordTypeStats[i].count);
      if (fRecordTypeStats[i].minLength == fRecordTypeStats[i].maxLength) {
        fprintf(stderr, "length:\t\t%d\n", fRecordTypeStats[i].minLength);
      } else {
        fprintf(stderr, "lengths:\t%d-%d\n", fRecordTypeStats[i].minLength, fRecordTypeStats[i].maxLength);
      }
    }
  }
#endif
}