fix #747, 'hf mfu eload' - now detects and converts between plain/old/new mfu binary format

CHANGELOG.md

@@ -5,6 +5,9 @@ This project uses the changelog in accordance with [keepchangelog](http://keepac
 ## [unreleased][unreleased]
 
 ## [ice coffee.4.][2020-05-21]
+ - Fix `hf mfu eload` - now detects and converts between plain/old/new mfu binary format (@iceman1001)
+ - Change log files moved to subfolders (@doegex)
+ - Change lib lua unbundled. (@doegex)
  - Updated documentation (@doegox, @iceman1001)
  - Change `pm3test.sh` - more regression tests to  (@doegox, @iceman1001)
  - Change `hf 15 dump` - now supports basic json format (@iceman1001)

client/src/cmdhfmf.c

@@ -3688,10 +3688,15 @@ int CmdHF14AMfELoad(const char *Cmd) {
         return usage_hf14_eload();
 
     uint8_t *data = calloc(4096, sizeof(uint8_t));
+    if (data == NULL) {
+        PrintAndLogEx(WARNING, "Fail, cannot allocate memory");
+        return PM3_EMALLOC;
+    }
+         
     size_t datalen = 0;
     //int res = loadFile(filename, ".bin", data, maxdatalen, &datalen);
     int res = loadFileEML(filename, data, &datalen);
-    if (res) {
+    if (res != PM3_SUCCESS) {
         free(data);
         return PM3_EFILE;
     }
@@ -3703,15 +3708,31 @@ int CmdHF14AMfELoad(const char *Cmd) {
         return PM3_ESOFT;
     }
 
-    // convert old mfu format to new
+    // convert plain or old mfu format to new format
     if (blockWidth == 4) {
-        res = convertOldMfuDump(&data, &datalen);
+
-        if (res) {
+        res = convert_mfu_dump_format(&data, &datalen, true);    
+        if (res != PM3_SUCCESS) {
             PrintAndLogEx(FAILED, "Failed convert on load to new Ultralight/NTAG format");
             free(data);
             return res;
         }
         
+        mfu_dump_t *mfu_dump = (mfu_dump_t *)data;
+
+        PrintAndLogEx(INFO, _CYAN_("MFU dump file information"));
+        PrintAndLogEx(INFO, "     version  %s", sprint_hex(mfu_dump->version, sizeof(mfu_dump->version)));
+        PrintAndLogEx(INFO, "        tb 0  %s", sprint_hex(mfu_dump->tbo, sizeof(mfu_dump->tbo)));
+        PrintAndLogEx(INFO, "        tb 1  %s", sprint_hex(mfu_dump->tbo1, sizeof(mfu_dump->tbo1)));
+        for(uint8_t m = 0; m < 3; m++) {
+            PrintAndLogEx(INFO, "   counter %d  %s - tearing 0x%02x", m + 1, sprint_hex(mfu_dump->counter_tearing[m], 3),  mfu_dump->counter_tearing[m][3]);
+        }
+        PrintAndLogEx(INFO, "   signature  %s", sprint_hex(mfu_dump->signature, sizeof(mfu_dump->signature)));
+        PrintAndLogEx(INFO, "        data  %s... (only first 8 bytes showing)", sprint_hex(mfu_dump->data, 8));        
+        PrintAndLogEx(INFO, "    max data page %d,  data len %d bytes", mfu_dump->pages, (mfu_dump->pages + 1) * 4);
+        PrintAndLogEx(INFO, " file header size %d", MFU_DUMP_PREFIX_LENGTH);
+        PrintAndLogEx(INFO, "----------------------------------------------");
+
         // update expected blocks to match converted data.
         if (numBlocks != datalen / 4) {
             numBlocks = datalen / 4;

client/src/cmdhfmfu.c

@@ -2039,6 +2039,7 @@ static int CmdHF14AMfURestore(const char *Cmd) {
     bool write_special = false;
     bool write_extra = false;
     bool read_key = false;
+    bool verbose = false;
     size_t filelen = 0;
     FILE *f;
 
@@ -2087,6 +2088,9 @@ static int CmdHF14AMfURestore(const char *Cmd) {
                 cmdp++;
                 read_key = true;
                 break;
+            case 'v':
+                cmdp++;
+                verbose = true;
             default:
                 PrintAndLogEx(WARNING, "Unknown parameter: " _RED_("'%c'"), param_getchar(Cmd, cmdp));
                 errors = true;
@@ -2128,10 +2132,9 @@ static int CmdHF14AMfURestore(const char *Cmd) {
         return 1;
     }
 
-    // convert old format to new format, if need
+    int res = convert_mfu_dump_format(&dump, &bytes_read, verbose);    
-    int res = convertOldMfuDump(&dump, &bytes_read);
     if (res != PM3_SUCCESS) {
-        PrintAndLogEx(WARNING, "Failed convert on load to new Ultralight/NTAG format");
+        PrintAndLogEx(FAILED, "Failed convert on load to new Ultralight/NTAG format");
         free(dump);
         return res;
     }

client/src/cmdhfmfu.h

@@ -18,19 +18,16 @@ typedef struct {
     uint8_t signature[32];
     //uint8_t counter[3];
     uint8_t data[1024];
-} old_mfu_dump_t;
+} PACKED old_mfu_dump_t;
-
 
 uint32_t GetHF14AMfU_Type(void);
 int ul_print_type(uint32_t tagtype, uint8_t spaces);
-
 void printMFUdump(mfu_dump_t *card);
 void printMFUdumpEx(mfu_dump_t *card, uint16_t pages, uint8_t startpage);
 
 int CmdHFMFUltra(const char *Cmd);
 
 uint16_t ul_ev1_packgen_VCNEW(uint8_t *uid, uint32_t pwd);
-
 uint32_t ul_ev1_otpgenA(uint8_t *uid);
 
 typedef enum TAGTYPE_UL {

client/src/fileutils.c

@@ -313,6 +313,9 @@ out:
 }
 
 int saveFileJSON(const char *preferredName, JSONFileType ftype, uint8_t *data, size_t datalen) {
+    return saveFileJSONex(preferredName, ftype, data, datalen, true);
+}
+int saveFileJSONex(const char *preferredName, JSONFileType ftype, uint8_t *data, size_t datalen, bool verbose) {
 
     if (data == NULL) return PM3_EINVARG;
 
@@ -559,7 +562,9 @@ int saveFileJSON(const char *preferredName, JSONFileType ftype, uint8_t *data, s
         retval = 200;
         goto out;
     }
+    if (verbose)
         PrintAndLogEx(SUCCESS, "saved to json file " _YELLOW_("%s"), fileName);
+
     json_decref(root);
 
 out:
@@ -678,7 +683,6 @@ int createMfcKeyDump(const char *preferredName, uint8_t sectorsCnt, sector_t *e_
     return PM3_SUCCESS;
 }
 
-
 int loadFile(const char *preferredName, const char *suffix, void *data, size_t maxdatalen, size_t *datalen) {
 
     if (data == NULL) return 1;
@@ -820,6 +824,7 @@ int loadFileEML(const char *preferredName, void *data, size_t *datalen) {
         if (fgets(line, sizeof(line), f) == NULL) {
             if (feof(f))
                 break;
+
             fclose(f);
             PrintAndLogEx(FAILED, "File reading error.");
             retval = PM3_EFILE;
@@ -829,10 +834,14 @@ int loadFileEML(const char *preferredName, void *data, size_t *datalen) {
         if (line[0] == '#')
             continue;
 
+        strcleanrn(line, sizeof(line));
+        
         int res = param_gethex_to_eol(line, 0, buf, sizeof(buf), &hexlen);
-        if (res == 0 || res == 1) {
+        if (res == 0) {
             memcpy(udata + counter, buf, hexlen);
             counter += hexlen;
+        } else {
+            retval = PM3_ESOFT;
         }
     }
     fclose(f);
@@ -847,6 +856,9 @@ out:
 }
 
 int loadFileJSON(const char *preferredName, void *data, size_t maxdatalen, size_t *datalen) {
+    return loadFileJSONex(preferredName, data, maxdatalen, datalen, true);
+}
+int loadFileJSONex(const char *preferredName, void *data, size_t maxdatalen, size_t *datalen, bool verbose) {
 
     if (data == NULL) return PM3_EINVARG;
     char *fileName = filenamemcopy(preferredName, ".json");
@@ -986,7 +998,9 @@ int loadFileJSON(const char *preferredName, void *data, size_t maxdatalen, size_
         }
         *datalen = sptr;
     }
+    if (verbose)
         PrintAndLogEx(SUCCESS, "loaded from JSON file " _YELLOW_("%s"), fileName);
+
     if (!strcmp(ctype, "settings")) {
         preferences_load_callback(root);
     }
@@ -1185,38 +1199,137 @@ out:
     return retval;
 }
 
-int convertOldMfuDump(uint8_t **dump, size_t *dumplen) {
+mfu_df_e detect_mfu_dump_format(uint8_t **dump, size_t *dumplen, bool verbose) {
-    if (!dump || !dumplen || *dumplen < OLD_MFU_DUMP_PREFIX_LENGTH)
+
-        return 1;
+    mfu_df_e retval = MFU_DF_UNKNOWN;
-    // try to check new file format
+    uint8_t bcc0, bcc1;
-    mfu_dump_t *mfu_dump = (mfu_dump_t *) *dump;
+    uint8_t ct = 0x88;
-    if ((*dumplen - MFU_DUMP_PREFIX_LENGTH) / 4 - 1 == mfu_dump->pages)
+ 
-        return 0;
+    // detect new
+    mfu_dump_t *new = (mfu_dump_t *)*dump;
+    bcc0 = ct ^ new->data[0] ^ new->data[1] ^ new->data[2];
+    bcc1 = new->data[4] ^ new->data[5] ^ new->data[6] ^ new->data[7];   
+    if (bcc0 == new->data[3] && bcc1 == new->data[8]) {
+        retval = MFU_DF_NEWBIN;
+    }
+
+    // detect old
+    if (retval == MFU_DF_UNKNOWN) {
+        old_mfu_dump_t *old = (old_mfu_dump_t *)*dump;
+        bcc0 = ct ^ old->data[0] ^ old->data[1] ^ old->data[2];
+        bcc1 = old->data[4] ^ old->data[5] ^ old->data[6] ^ old->data[7];
+        if (bcc0 == old->data[3] && bcc1 == old->data[8]) {
+            retval = MFU_DF_OLDBIN;
+        }
+    }
+
+    // detect plain
+    if (retval == MFU_DF_UNKNOWN) {
+        uint8_t *plain = *dump;
+        bcc0 = ct ^ plain[0] ^ plain[1] ^ plain[2];
+        bcc1 = plain[4] ^ plain[5] ^ plain[6] ^ plain[7];
+        if ((bcc0 == plain[3]) && (bcc1 == plain[8])) {
+            retval = MFU_DF_PLAINBIN;
+        }
+    }
+
+    if (verbose) {
+        switch(retval) {
+            case MFU_DF_NEWBIN: 
+                PrintAndLogEx(INFO, "detected " _GREEN_("new") " mfu dump format");
+                break;
+            case MFU_DF_OLDBIN:
+                PrintAndLogEx(INFO, "detected " _GREEN_("old") " mfu dump format");
+                break;
+            case MFU_DF_PLAINBIN:
+                PrintAndLogEx(INFO, "detected " _GREEN_("plain") " mfu dump format");
+                break;
+            case MFU_DF_UNKNOWN:
+                PrintAndLogEx(WARNING, "failed to detected mfu dump format");
+                break;
+        }
+    }
+    return retval;
+}
+
+static int convert_plain_mfu_dump(uint8_t **dump, size_t *dumplen, bool verbose) {
+    
+    mfu_dump_t *mfu = (mfu_dump_t *) calloc( sizeof(mfu_dump_t), sizeof(uint8_t));
+    if (mfu == NULL) {
+        return PM3_EMALLOC;
+    }
+
+    memcpy(mfu->data, *dump, *dumplen);
+
+    mfu->pages = *dumplen / 4 - 1;
+    
+    if (verbose) {
+        PrintAndLogEx(SUCCESS, "plain mfu dump format was converted to " _GREEN_("%d") " blocks", mfu->pages + 1);
+    }
+    
+    *dump = (uint8_t *)mfu;
+    *dumplen += MFU_DUMP_PREFIX_LENGTH ;
+    return PM3_SUCCESS;
+}
+
+static int convert_old_mfu_dump(uint8_t **dump, size_t *dumplen, bool verbose) {
+
     // convert old format
-    old_mfu_dump_t *old_mfu_dump = (old_mfu_dump_t *) *dump;
+    old_mfu_dump_t *old_mfu_dump = (old_mfu_dump_t *)*dump;
 
     size_t old_data_len = *dumplen - OLD_MFU_DUMP_PREFIX_LENGTH;
     size_t new_dump_len = old_data_len + MFU_DUMP_PREFIX_LENGTH;
   
-    mfu_dump = (mfu_dump_t *) calloc(new_dump_len, sizeof(uint8_t));
+    mfu_dump_t *mfu_dump = (mfu_dump_t *) calloc( sizeof(mfu_dump_t), sizeof(uint8_t));
+    if (mfu_dump == NULL) {
+        return PM3_EMALLOC;
+    }
+
+    memcpy(mfu_dump->version, old_mfu_dump->version, sizeof(mfu_dump->version));
+    memcpy(mfu_dump->tbo, old_mfu_dump->tbo, sizeof(mfu_dump->tbo));
+    memcpy(mfu_dump->signature, old_mfu_dump->signature, sizeof(mfu_dump->signature));
     
-    memcpy(mfu_dump->version, old_mfu_dump->version, 8);
-    memcpy(mfu_dump->tbo, old_mfu_dump->tbo, 2);
     mfu_dump->tbo1[0] = old_mfu_dump->tbo1[0];
-    memcpy(mfu_dump->signature, old_mfu_dump->signature, 32);
-    for (int i = 0; i < 3; i++)
-        mfu_dump->counter_tearing[i][3] = old_mfu_dump->tearing[i];
 
-    memcpy(mfu_dump->data, old_mfu_dump->data, old_data_len);
+    for (int i = 0; i < 3; i++) {
+        mfu_dump->counter_tearing[i][3] = old_mfu_dump->tearing[i];
+    }
+
+    memcpy(mfu_dump->data, old_mfu_dump->data, sizeof(mfu_dump->data));
+
     mfu_dump->pages = old_data_len / 4 - 1;
-    // free old buffer, return new buffer
+
-    *dumplen = new_dump_len;
+    if (verbose) {  
+        PrintAndLogEx(SUCCESS, "old mfu dump format was converted to " _GREEN_("%d") " blocks", mfu_dump->pages + 1);
+    }
+
     free(*dump);
-    *dump = (uint8_t *) mfu_dump;
+    *dump = (uint8_t *)mfu_dump;
-    PrintAndLogEx(SUCCESS, "old mfu dump format, was converted on load to " _GREEN_("%d") " pages", mfu_dump->pages + 1);
+    *dumplen = new_dump_len;
     return PM3_SUCCESS;
 }
 
+int convert_mfu_dump_format(uint8_t **dump, size_t *dumplen, bool verbose) {
+
+    if (!dump || !dumplen || *dumplen < OLD_MFU_DUMP_PREFIX_LENGTH) {
+        return PM3_EINVARG;
+    }
+    
+    mfu_df_e res = detect_mfu_dump_format(dump, dumplen, verbose);
+
+    switch(res) {
+        case MFU_DF_NEWBIN: 
+            return PM3_SUCCESS;
+        case MFU_DF_OLDBIN:
+            return convert_old_mfu_dump(dump, dumplen, verbose);
+        case MFU_DF_PLAINBIN:
+            return convert_plain_mfu_dump(dump, dumplen, verbose);
+        case MFU_DF_UNKNOWN:
+        default:
+            return PM3_ESOFT;
+    }
+}
+
 static int filelist(const char *path, const char *ext, bool last, bool tentative) {
     struct dirent **namelist;
     int n;

client/src/fileutils.h

@@ -116,6 +116,7 @@ int saveFileEML(const char *preferredName, uint8_t *data, size_t datalen, size_t
  * @return 0 for ok, 1 for failz
  */
 int saveFileJSON(const char *preferredName, JSONFileType ftype, uint8_t *data, size_t datalen);
+int saveFileJSONex(const char *preferredName, JSONFileType ftype, uint8_t *data, size_t datalen, bool verbose);
 
 /** STUB
  * @brief Utility function to save WAVE data to a file. This method takes a preferred name, but if that
@@ -198,6 +199,8 @@ int loadFileEML(const char *preferredName, void *data, size_t *datalen);
  * @return 0 for ok, 1 for failz
 */
 int loadFileJSON(const char *preferredName, void *data, size_t maxdatalen, size_t *datalen);
+int loadFileJSONex(const char *preferredName, void *data, size_t maxdatalen, size_t *datalen, bool verbose);
+
 
 /**
  * @brief  Utility function to load data from a DICTIONARY textfile. This method takes a preferred name.
@@ -242,14 +245,23 @@ int loadFileDICTIONARYEx(const char *preferredName, void *data, size_t maxdatale
 */
 int loadFileDICTIONARY_safe(const char *preferredName, void **pdata, uint8_t keylen, uint32_t *keycnt);
 
+
+typedef enum {
+    MFU_DF_UNKNOWN,
+    MFU_DF_PLAINBIN,
+    MFU_DF_OLDBIN,
+    MFU_DF_NEWBIN
+} mfu_df_e;
 /**
- * @brief  Utility function to check and convert old mfu dump format to new
+ * @brief  Utility function to check and convert plain mfu dump format to new mfu binary format.
- *
+ * plain dumps doesn't have any extra data, like version, signature etc.
  * @param dump pointer to loaded dump to check and convert format
  * @param dumplen the number of bytes loaded dump and converted
- * @return 0 for ok, 1 for fails
+ * @param verbose - extra debug output
+ * @return PM3_SUCCESS for ok, PM3_ESOFT for fails
 */
-int convertOldMfuDump(uint8_t **dump, size_t *dumplen);
+int convert_mfu_dump_format(uint8_t **dump, size_t *dumplen, bool verbose);
+mfu_df_e detect_mfu_dump_format(uint8_t **dump, size_t *dumplen, bool verbose); 
 
 int searchAndList(const char *pm3dir, const char *ext);
 int searchFile(char **foundpath, const char *pm3dir, const char *searchname, const char *suffix, bool silent);