From d8c18e6db85c9bc46fffa02f8c5eee5fcb288589 Mon Sep 17 00:00:00 2001
From: mwalker33 <51802811+mwalker33@users.noreply.github.com>
Date: Wed, 31 Aug 2022 18:51:39 +1000
Subject: [PATCH] Prep spiffs for bigger data files

- added spiffs check after flase wipe to force it to update its status
- added spiffs write and append to write in 8192 byte chunks to allow spiffs space to be freed in time.
- fixed spiffs dump to correctly handle issues if it could not allocate bugbuff space.
---
 armsrc/appmain.c   | 30 ++++++++++++++++++------------
 armsrc/flashmem.c  |  4 ++++
 armsrc/spiffs.c    | 30 ++++++++++++++++++++++++++++--
 client/src/comms.c |  2 ++
 4 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/armsrc/appmain.c b/armsrc/appmain.c
index 330bf2c48..0f474933e 100644
--- a/armsrc/appmain.c
+++ b/armsrc/appmain.c
@@ -2166,20 +2166,26 @@ static void PacketReceived(PacketCommandNG *packet) {
             uint32_t size = packet->oldarg[1];
 
             uint8_t *buff = BigBuf_malloc(size);
-            rdv40_spiffs_read_as_filetype((char *)filename, (uint8_t *)buff, size, RDV40_SPIFFS_SAFETY_SAFE);
+            if (buff == NULL) {
+                if (g_dbglevel >= DBG_DEBUG) Dbprintf ("Could not allocate buffer");
+                // Trigger a finish downloading signal with an PM3_EMALLOC
+                reply_ng(CMD_SPIFFS_DOWNLOAD, PM3_EMALLOC, NULL, 0);
+            } else {
+                rdv40_spiffs_read_as_filetype((char *)filename, (uint8_t *)buff, size, RDV40_SPIFFS_SAFETY_SAFE);
+                // arg0 = filename
+                // arg1 = size
+                // arg2 = RFU
 
-            // arg0 = filename
-            // arg1 = size
-            // arg2 = RFU
-
-            for (size_t i = 0; i < size; i += PM3_CMD_DATA_SIZE) {
-                size_t len = MIN((size - i), PM3_CMD_DATA_SIZE);
-                int result = reply_old(CMD_SPIFFS_DOWNLOADED, i, len, 0, buff + i, len);
-                if (result != PM3_SUCCESS)
-                    Dbprintf("transfer to client failed ::  | bytes between %d - %d (%d) | result: %d", i, i + len, len, result);
+                for (size_t i = 0; i < size; i += PM3_CMD_DATA_SIZE) {
+                    size_t len = MIN((size - i), PM3_CMD_DATA_SIZE);
+                    int result = reply_old(CMD_SPIFFS_DOWNLOADED, i, len, 0, buff + i, len);
+                    if (result != PM3_SUCCESS)
+                        Dbprintf("transfer to client failed ::  | bytes between %d - %d (%d) | result: %d", i, i + len, len, result);
+                }
+                // Trigger a finish downloading signal with an ACK frame
+                reply_ng(CMD_SPIFFS_DOWNLOAD, PM3_SUCCESS, NULL, 0);
+                BigBuf_free ();
             }
-            // Trigger a finish downloading signal with an ACK frame
-            reply_ng(CMD_SPIFFS_DOWNLOAD, PM3_SUCCESS, NULL, 0);
             LED_B_OFF();
             break;
         }
diff --git a/armsrc/flashmem.c b/armsrc/flashmem.c
index fb1f74140..fd5a95c97 100644
--- a/armsrc/flashmem.c
+++ b/armsrc/flashmem.c
@@ -22,6 +22,7 @@
 #include "ticks.h"
 #include "dbprint.h"
 #include "string.h"
+#include "spiffs.h"
 
 /* here: use NCPS2 @ PA10: */
 #define SPI_CSR_NUM      2
@@ -449,6 +450,9 @@ bool Flash_WipeMemoryPage(uint8_t page) {
     Flash_CheckBusy(BUSY_TIMEOUT);
 
     FlashStop();
+
+    // let spiffs check and update its info post flash erase
+    rdv40_spiffs_check ();
     return true;
 }
 // Wipes flash memory completely, fills with 0xFF
diff --git a/armsrc/spiffs.c b/armsrc/spiffs.c
index 027563729..aa7275cce 100644
--- a/armsrc/spiffs.c
+++ b/armsrc/spiffs.c
@@ -433,15 +433,41 @@ int rdv40_spiffs_lazy_mount_rollback(int changed) {
 // statement or some function taking function parameters
 // TODO : forbid writing to a filename which already exists as lnk !
 // TODO : forbid writing to a filename.lnk which already exists without lnk !
+// Note: Writing in 8192 byte chucks helps to ensure "free space" has been erased by GC (Garbage collection)
 int rdv40_spiffs_write(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) {
     RDV40_SPIFFS_SAFE_FUNCTION(
-        write_to_spiffs(filename, src, size);
+        uint16_t idx;
+        if (size <= 8192) {
+            // write small file
+            write_to_spiffs(filename, src, size);
+            size = 0;
+        } else { //
+            // write first 8192 bytes
+            // need to write the first chuck of data, then append
+            write_to_spiffs(filename, src, 8192);
+        }
+        // append remaing 8192 byte chuncks
+        for (idx = 1; idx < (size / 8192);  idx++) {
+            append_to_spiffs(filename, &src[8192 * idx], 8192);
+        }
+        // append remaing bytes
+        if (((int64_t)size - (8192 * idx)) > 0) {
+            append_to_spiffs(filename, &src[8192 * idx], size - (8192 * idx));
+        }
     )
 }
 
 int rdv40_spiffs_append(const char *filename, uint8_t *src, uint32_t size, RDV40SpiFFSSafetyLevel level) {
     RDV40_SPIFFS_SAFE_FUNCTION(
-        append_to_spiffs(filename, src, size);
+        uint16_t idx;
+        // Append any 8192 byte chunks
+        for (idx = 0; idx < (size/8192);  idx++) {
+            append_to_spiffs(filename, &src[8192 * idx], 8192);
+        }
+        // Append remain bytes
+        if (((int64_t)size - (8192 * idx)) > 0) {
+            append_to_spiffs(filename, &src[8192 * idx], size - (8192 * idx));
+        }
     )
 }
 
diff --git a/client/src/comms.c b/client/src/comms.c
index 2b521d2d7..a8ed8be4c 100644
--- a/client/src/comms.c
+++ b/client/src/comms.c
@@ -856,6 +856,8 @@ static bool dl_it(uint8_t *dest, uint32_t bytes, PacketResponseNG *response, siz
 
             if (response->cmd == CMD_ACK)
                 return true;
+            if (response->cmd == CMD_SPIFFS_DOWNLOAD && response->status == PM3_EMALLOC)
+                return false;
             // Spiffs // fpgamem-plot download is converted to NG,
             if (response->cmd == CMD_SPIFFS_DOWNLOAD || response->cmd == CMD_FPGAMEM_DOWNLOAD)
                 return true;