From d8c18e6db85c9bc46fffa02f8c5eee5fcb288589 Mon Sep 17 00:00:00 2001 From: mwalker33 <51802811+mwalker33@users.noreply.github.com> Date: Wed, 31 Aug 2022 18:51:39 +1000 Subject: [PATCH 1/3] Prep spiffs for bigger data files - added spiffs check after flase wipe to force it to update its status - added spiffs write and append to write in 8192 byte chunks to allow spiffs space to be freed in time. - fixed spiffs dump to correctly handle issues if it could not allocate bugbuff space. --- armsrc/appmain.c | 30 ++++++++++++++++++------------ armsrc/flashmem.c | 4 ++++ armsrc/spiffs.c | 30 ++++++++++++++++++++++++++++-- client/src/comms.c | 2 ++ 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/armsrc/appmain.c b/armsrc/appmain.c index 330bf2c48..0f474933e 100644 --- a/armsrc/appmain.c +++ b/armsrc/appmain.c @@ -2166,20 +2166,26 @@ static void PacketReceived(PacketCommandNG *packet) { uint32_t size = packet->oldarg[1]; uint8_t *buff = BigBuf_malloc(size); - rdv40_spiffs_read_as_filetype((char *)filename, (uint8_t *)buff, size, RDV40_SPIFFS_SAFETY_SAFE); + if (buff == NULL) { + if (g_dbglevel >= DBG_DEBUG) Dbprintf ("Could not allocate buffer"); + // Trigger a finish downloading signal with an PM3_EMALLOC + reply_ng(CMD_SPIFFS_DOWNLOAD, PM3_EMALLOC, NULL, 0); + } else { + rdv40_spiffs_read_as_filetype((char *)filename, (uint8_t *)buff, size, RDV40_SPIFFS_SAFETY_SAFE); + // arg0 = filename + // arg1 = size + // arg2 = RFU - // arg0 = filename - // arg1 = size - // arg2 = RFU - - for (size_t i = 0; i < size; i += PM3_CMD_DATA_SIZE) { - size_t len = MIN((size - i), PM3_CMD_DATA_SIZE); - int result = reply_old(CMD_SPIFFS_DOWNLOADED, i, len, 0, buff + i, len); - if (result != PM3_SUCCESS) - Dbprintf("transfer to client failed :: | bytes between %d - %d (%d) | result: %d", i, i + len, len, result); + for (size_t i = 0; i < size; i += PM3_CMD_DATA_SIZE) { + size_t len = MIN((size - i), PM3_CMD_DATA_SIZE); + int result = reply_old(CMD_SPIFFS_DOWNLOADED, i, len, 0, buff + i, len); + if (result != PM3_SUCCESS) + Dbprintf("transfer to client failed :: | bytes between %d - %d (%d) | result: %d", i, i + len, len, result); + } + // Trigger a finish downloading signal with an ACK frame + reply_ng(CMD_SPIFFS_DOWNLOAD, PM3_SUCCESS, NULL, 0); + BigBuf_free (); } - // Trigger a finish downloading signal with an ACK frame - reply_ng(CMD_SPIFFS_DOWNLOAD, PM3_SUCCESS, NULL, 0); LED_B_OFF(); break; } diff --git a/armsrc/flashmem.c b/armsrc/flashmem.c index fb1f74140..fd5a95c97 100644 --- a/armsrc/flashmem.c +++ b/armsrc/flashmem.c @@ -22,6 +22,7 @@ #include "ticks.h" #include "dbprint.h" #include "string.h" +#include "spiffs.h" /* here: use NCPS2 @ PA10: */ #define SPI_CSR_NUM 2 @@ -449,6 +450,9 @@ bool Flash_WipeMemoryPage(uint8_t page) { Flash_CheckBusy(BUSY_TIMEOUT); FlashStop(); + + // let spiffs check and update its info post flash erase + rdv40_spiffs_check (); return true; } // Wipes flash memory completely, fills with 0xFF diff --git a/armsrc/spiffs.c b/armsrc/spiffs.c index 027563729..aa7275cce 100644 --- a/armsrc/spiffs.c +++ b/armsrc/spiffs.c @@ -433,15 +433,41 @@ int rdv40_spiffs_lazy_mount_rollback(int changed) { // statement or some function taking function parameters // TODO : forbid writing to a filename which already exists as lnk ! // TODO : forbid writing to a filename.lnk which already exists without lnk ! +// Note: Writing in 8192 byte chucks helps to ensure "free space" has been erased by GC (Garbage collection) int rdv40_spiffs_write(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) { RDV40_SPIFFS_SAFE_FUNCTION( - write_to_spiffs(filename, src, size); + uint16_t idx; + if (size <= 8192) { + // write small file + write_to_spiffs(filename, src, size); + size = 0; + } else { // + // write first 8192 bytes + // need to write the first chuck of data, then append + write_to_spiffs(filename, src, 8192); + } + // append remaing 8192 byte chuncks + for (idx = 1; idx < (size / 8192); idx++) { + append_to_spiffs(filename, &src[8192 * idx], 8192); + } + // append remaing bytes + if (((int64_t)size - (8192 * idx)) > 0) { + append_to_spiffs(filename, &src[8192 * idx], size - (8192 * idx)); + } ) } int rdv40_spiffs_append(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) { RDV40_SPIFFS_SAFE_FUNCTION( - append_to_spiffs(filename, src, size); + uint16_t idx; + // Append any 8192 byte chunks + for (idx = 0; idx < (size/8192); idx++) { + append_to_spiffs(filename, &src[8192 * idx], 8192); + } + // Append remain bytes + if (((int64_t)size - (8192 * idx)) > 0) { + append_to_spiffs(filename, &src[8192 * idx], size - (8192 * idx)); + } ) } diff --git a/client/src/comms.c b/client/src/comms.c index 2b521d2d7..a8ed8be4c 100644 --- a/client/src/comms.c +++ b/client/src/comms.c @@ -856,6 +856,8 @@ static bool dl_it(uint8_t *dest, uint32_t bytes, PacketResponseNG *response, siz if (response->cmd == CMD_ACK) return true; + if (response->cmd == CMD_SPIFFS_DOWNLOAD && response->status == PM3_EMALLOC) + return false; // Spiffs // fpgamem-plot download is converted to NG, if (response->cmd == CMD_SPIFFS_DOWNLOAD || response->cmd == CMD_FPGAMEM_DOWNLOAD) return true; From 96876c01113455bcfd014fb3af9c653ae1a713df Mon Sep 17 00:00:00 2001 From: mwalker33 <51802811+mwalker33@users.noreply.github.com> Date: Wed, 31 Aug 2022 19:10:15 +1000 Subject: [PATCH 2/3] Update spiffs.c patch type size comparison --- armsrc/spiffs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/armsrc/spiffs.c b/armsrc/spiffs.c index aa7275cce..665fb2cf8 100644 --- a/armsrc/spiffs.c +++ b/armsrc/spiffs.c @@ -436,7 +436,7 @@ int rdv40_spiffs_lazy_mount_rollback(int changed) { // Note: Writing in 8192 byte chucks helps to ensure "free space" has been erased by GC (Garbage collection) int rdv40_spiffs_write(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) { RDV40_SPIFFS_SAFE_FUNCTION( - uint16_t idx; + uint32_t idx; if (size <= 8192) { // write small file write_to_spiffs(filename, src, size); @@ -459,7 +459,7 @@ int rdv40_spiffs_write(const char *filename, uint8_t *src, uint32_t size, RDV40S int rdv40_spiffs_append(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) { RDV40_SPIFFS_SAFE_FUNCTION( - uint16_t idx; + uint32_t idx; // Append any 8192 byte chunks for (idx = 0; idx < (size/8192); idx++) { append_to_spiffs(filename, &src[8192 * idx], 8192); From 9406ef9fd3d5997b3c038d3cd0451b626bd49594 Mon Sep 17 00:00:00 2001 From: mwalker33 <51802811+mwalker33@users.noreply.github.com> Date: Thu, 1 Sep 2022 08:15:30 +1000 Subject: [PATCH 3/3] spiffs prep added define for chuck size update changelog --- CHANGELOG.md | 2 ++ armsrc/spiffs.c | 28 ++++++++++++++-------------- armsrc/spiffs.h | 2 ++ 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93c557838..bbb1b0049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ All notable changes to this project will be documented in this file. This project uses the changelog in accordance with [keepchangelog](http://keepachangelog.com/). Please use this to write notable changes, which is not the same as git commit log... ## [unreleased][unreleased] + - Changed spiffs write/apped to send in 8192 chucks to ensure its eraised (@mwalker) + - Fixed spiffs dump to ensure to fails correctly if no big_buff was allocated (@mwalker) - Change Client Makefile to respect global flags (@blshkv) - Change Makefile, honors global CC values (@blshkv) - Fixed bad memory handling in MifareSim device side (@iceman1001) diff --git a/armsrc/spiffs.c b/armsrc/spiffs.c index 665fb2cf8..ef7ca3174 100644 --- a/armsrc/spiffs.c +++ b/armsrc/spiffs.c @@ -433,26 +433,26 @@ int rdv40_spiffs_lazy_mount_rollback(int changed) { // statement or some function taking function parameters // TODO : forbid writing to a filename which already exists as lnk ! // TODO : forbid writing to a filename.lnk which already exists without lnk ! -// Note: Writing in 8192 byte chucks helps to ensure "free space" has been erased by GC (Garbage collection) +// Note: Writing in SPIFFS_WRITE_CHUNK_SIZE (8192) byte chucks helps to ensure "free space" has been erased by GC (Garbage collection) int rdv40_spiffs_write(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) { RDV40_SPIFFS_SAFE_FUNCTION( uint32_t idx; - if (size <= 8192) { + if (size <= SPIFFS_WRITE_CHUNK_SIZE) { // write small file write_to_spiffs(filename, src, size); size = 0; } else { // - // write first 8192 bytes + // write first SPIFFS_WRITE_CHUNK_SIZE bytes // need to write the first chuck of data, then append - write_to_spiffs(filename, src, 8192); + write_to_spiffs(filename, src, SPIFFS_WRITE_CHUNK_SIZE); } - // append remaing 8192 byte chuncks - for (idx = 1; idx < (size / 8192); idx++) { - append_to_spiffs(filename, &src[8192 * idx], 8192); + // append remaing SPIFFS_WRITE_CHUNK_SIZE byte chuncks + for (idx = 1; idx < (size / SPIFFS_WRITE_CHUNK_SIZE); idx++) { + append_to_spiffs(filename, &src[SPIFFS_WRITE_CHUNK_SIZE * idx], SPIFFS_WRITE_CHUNK_SIZE); } // append remaing bytes - if (((int64_t)size - (8192 * idx)) > 0) { - append_to_spiffs(filename, &src[8192 * idx], size - (8192 * idx)); + if (((int64_t)size - (SPIFFS_WRITE_CHUNK_SIZE * idx)) > 0) { + append_to_spiffs(filename, &src[SPIFFS_WRITE_CHUNK_SIZE * idx], size - (SPIFFS_WRITE_CHUNK_SIZE * idx)); } ) } @@ -460,13 +460,13 @@ int rdv40_spiffs_write(const char *filename, uint8_t *src, uint32_t size, RDV40S int rdv40_spiffs_append(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) { RDV40_SPIFFS_SAFE_FUNCTION( uint32_t idx; - // Append any 8192 byte chunks - for (idx = 0; idx < (size/8192); idx++) { - append_to_spiffs(filename, &src[8192 * idx], 8192); + // Append any SPIFFS_WRITE_CHUNK_SIZE byte chunks + for (idx = 0; idx < (size/SPIFFS_WRITE_CHUNK_SIZE); idx++) { + append_to_spiffs(filename, &src[SPIFFS_WRITE_CHUNK_SIZE * idx], SPIFFS_WRITE_CHUNK_SIZE); } // Append remain bytes - if (((int64_t)size - (8192 * idx)) > 0) { - append_to_spiffs(filename, &src[8192 * idx], size - (8192 * idx)); + if (((int64_t)size - (SPIFFS_WRITE_CHUNK_SIZE * idx)) > 0) { + append_to_spiffs(filename, &src[SPIFFS_WRITE_CHUNK_SIZE * idx], size - (SPIFFS_WRITE_CHUNK_SIZE * idx)); } ) } diff --git a/armsrc/spiffs.h b/armsrc/spiffs.h index c540a023f..6af4fbac8 100644 --- a/armsrc/spiffs.h +++ b/armsrc/spiffs.h @@ -129,6 +129,8 @@ void rdv40_spiffs_safe_wipe(void); #define SPIFFS_ERR_TEST -10100 +// Amount of data to write/append to a file in one go. +#define SPIFFS_WRITE_CHUNK_SIZE 8192 // spiffs file descriptor index type. must be signed typedef s16_t spiffs_file;