LFS patch updates following review II and testing

app/coap/endpoints.c

@@ -162,7 +162,8 @@ end:
     return coap_make_response(scratch, outpkt, NULL, 0, id_hi, id_lo, &inpkt->tok, COAP_RSPCODE_NOT_FOUND, COAP_CONTENTTYPE_NONE);
 }
 
-extern lua_Load gLoad;
+extern int lua_put_line(const char *s, size_t l);
+
 static const coap_endpoint_path_t path_command = {2, {"v1", "c"}};
 static int handle_post_command(const coap_endpoint_t *ep, coap_rw_buffer_t *scratch, const coap_packet_t *inpkt, coap_packet_t *outpkt, uint8_t id_hi, uint8_t id_lo)
 {
@@ -170,16 +171,9 @@ static int handle_post_command(const coap_endpoint_t *ep, coap_rw_buffer_t *scra
         return coap_make_response(scratch, outpkt, NULL, 0, id_hi, id_lo, &inpkt->tok, COAP_RSPCODE_BAD_REQUEST, COAP_CONTENTTYPE_TEXT_PLAIN);
     if (inpkt->payload.len > 0)
     {
-        lua_Load *load = &gLoad;
+        char line[LUA_MAXINPUT];
-        if(load->line_position == 0){
+        if (!coap_buffer_to_string(line, LUA_MAXINPUT, &inpkt->payload) &&
-            coap_buffer_to_string(load->line, load->len,&inpkt->payload);
+            lua_put_line(line, c_strlen(line))) {
-            load->line_position = c_strlen(load->line)+1;
-            // load->line[load->line_position-1] = '\n';
-            // load->line[load->line_position] = 0;
-            // load->line_position++;
-            load->done = 1;
-            NODE_DBG("Get command:\n");
-            NODE_DBG(load->line); // buggy here
             NODE_DBG("\nResult(if any):\n");
             system_os_post (LUA_TASK_PRIO, LUA_PROCESS_LINE_SIG, 0);
         }

app/esp-gdbstub/Makefile

@@ -23,7 +23,7 @@ endif
 #   makefile at its root level - these are then overridden
 #   for a subtree within the makefile rooted therein
 #
-#DEFINES += 
+#DEFINES += -DGDBSTUB_REDIRECT_CONSOLE_OUTPUT 
 
 #############################################################
 # Recursion Magic - Don't touch this!!

app/include/user_config.h

@@ -1,8 +1,8 @@
 #ifndef __USER_CONFIG_H__
 #define __USER_CONFIG_H__
 
-// The firmware supports a range of Flash sizes, though 4Mbyte seems to be 
+// The firmware supports a range of Flash sizes, though 4 Mbyte seems to be
-// currently the most common.  Current builds include a discovery function
+// the most common currently.  NodeMCU builds include a discovery function
 // which is enabled by FLASH_AUTOSIZE, but you can override this by commenting
 // this out and enabling the explicitly size, e.g. FLASH_4M.  Valid sizes are
 // FLASH_512K, FLASH_1M, FLASH_2M, FLASH_4M, FLASH_8M, FLASH_16M.
@@ -11,15 +11,15 @@
 //#define FLASH_4M
 
 
-// The firmware now selects a baudrate of 115,200 by default, but the driver also 
+// The firmware now selects a baudrate of 115,200 by default, but the driver
-// includes automatic baud rate detection at start-up by default.  If you want to
+// also includes automatic baud rate detection at start-up.  If you want to change
-// change the default rate then vaild rates are  300, 600, 1200, 2400, 4800, 9600, 
+// the default rate then vaild rates are  300, 600, 1200, 2400, 4800, 9600, 19200,
-// 19200, 31250, 38400, 57600, 74880, 115200, 230400, 256000, 460800 [, 921600, 
+// 31250, 38400, 57600, 74880, 115200, 230400, 256000, 460800 [, 921600, 1843200,
-// 1843200, 368640].  Note that the last 3 rates are not recommended as these 
+// 368640].  Note that the last 3 rates are not recommended as these might be
-// might be unreliable.
+// unreliable, but 460800 seems to work well for most USB-serial devices.
 
 #define BIT_RATE_DEFAULT BIT_RATE_115200
-#define BIT_RATE_AUTOBAUD
+//#define BIT_RATE_AUTOBAUD
 
 
 // Three separate build variants are now supported. The main difference is in the
@@ -27,14 +27,16 @@
 // all numeric calculations are done in integer, with divide being an integer
 // operations, and decimal fraction constants are illegal.  Otherwise all
 // numeric operations use floating point, though they are exact for integer
-// expressions < 2^53.  The main advantage of INTEGRAL builds is that the basic
+// expressions < 2^53.
-// internal storage unit, the TValue, is 8 bytes long, rather than the default 
+
-// on floating point builds of 16 bytes.  We have now also introduced an 
+// The main advantage of INTEGRAL builds is that the basic internal storage unit,
-// experimental option LUA_PACK_TVALUES which reduces the floating point TValues
+// the TValue, is 8 bytes long.  We have now reduced the size of FP TValues to
-// to 12 bytes without any performance impact.
+// 12 bytes rather than the previous 16 as this gives a material RAM saving with
+// no performance loss.  However, you can define LUA_DWORD_ALIGNED_TVALUES and
+// this will force 16 byte TValues on FP builds.
 
 //#define LUA_NUMBER_INTEGRAL
-//#define LUA_PACK_TVALUES
+//#define LUA_DWORD_ALIGNED_TVALUES
 
 
 // The Lua Flash Store (LFS) allows you to store Lua code in Flash memory and
@@ -48,19 +50,19 @@
 
 // By default Lua executes the file init.lua at start up.  The following
 // define allows you to replace this with an alternative startup.  Warning:
-// you must protect this execution otherwise you will enter a panic loop.
+// you must protect this execution otherwise you will enter a panic loop;
+// the simplest way is to wrap the action in a function invoked by a pcall.
 // The example provided executes the LFS module "_init" at startup or fails
 // through to the interactive prompt.
-// ********* WARNING THIS OPTION ISN'T CURRENTLY WORKING
+
-//#define LUA_INIT_STRING "local fi=node.flashindex; return pcall(fi and fi'_init')"
+//#define LUA_INIT_STRING "pcall(function() node.flashindex'_init'() end)"
-// ********* WARNING THIS OPTION ISN'T CURRENTLY WORKING
 
 
 // NodeMCU supports two file systems: SPIFFS and FATFS, the first is available
 // on all ESP8266 modules.  The latter requires extra H/W so is less common.
 // If you use SPIFFS then there are a number of options which impact the
 // RAM overhead and performance of the file system.
-//
+
 // If you use the spiffsimg tool to create your own FS images on your dev PC
 // then we recommend that you fix the location and size of the FS, allowing
 // some headroom for rebuilding flash images and LFS.  As an alternative to
@@ -70,23 +72,23 @@
 // gives the fastest start-up and imaging times.
 
 #define BUILD_SPIFFS
-//#define BUILD_FATFS
-
 //#define SPIFFS_FIXED_LOCATION        0x100000
-//#define SPIFFS_MAX_FILESYSTEM_SIZE    0x10000	
+//#define SPIFFS_MAX_FILESYSTEM_SIZE    0x20000	
 //#define SPIFFS_SIZE_1M_BOUNDARY
 #define SPIFFS_CACHE 1          // Enable if you use you SPIFFS in R/W mode
 #define SPIFFS_MAX_OPEN_FILES 4 // maximum number of open files for SPIFFS
 #define FS_OBJ_NAME_LEN 31      // maximum length of a filename
 
+//#define BUILD_FATFS
+
 
 // The HTTPS stack requires client SSL to be enabled.  The SSL buffer size is
 // used only for espconn-layer secure connections, and is ignored otherwise.
 // Some HTTPS  applications require a larger buffer size to work.  See
 // https://github.com/nodemcu/nodemcu-firmware/issues/1457 for details.
-// The SHA2 and MD2 libraries are also used by the crypto functions.  The 
+// The SHA2 and MD2 libraries are also optionally used by the crypto functions.
-// MD2 function are implemented in the ROM BIOS, and the SHA2 by NodeMCU
+// The SHA1 and MD5 function are implemented in the ROM BIOS. The MD2 and SHA2
-// code, so only enable SHA2 if you need this functionality.
+// are by firmware code, and can be enabled if you need this functionality.
 
 //#define CLIENT_SSL_ENABLE
 //#define MD2_ENABLE
@@ -97,8 +99,8 @@
 // GPIO_INTERRUPT_ENABLE needs to be defined if your application uses the
 // gpio.trig() or related GPIO interrupt service routine code.  Likewise the
 // GPIO interrupt hook is requited for a few modules such as rotary.  If you
-// don't require this functionality, then we recommend commenting out these 
+// don't require this functionality, then commenting out these options out
-// options which removes any associated runtime overhead. 
+// will remove any associated runtime overhead.
 
 #define GPIO_INTERRUPT_ENABLE
 #define GPIO_INTERRUPT_HOOK_ENABLE
@@ -116,13 +118,21 @@
 // management, using internal timer callbacks.  Whilst many Lua developers
 // prefer to implement equivalent features in Lua, others will prefer the
 // Wifi module to do this for them.  Uncomment the following to enable
-// this functionality. The event sub-options are ignore if the SMART 
+// this functionality.  See the relevant WiFi module documentation for 
-// functionality is not enabled.
+// further details, as the scope of these changes is not obvious.
 
+//  Enable the wifi.startsmart() and wifi.stopsmart()  
 //#define WIFI_SMART_ENABLE
+
+//  Enable wifi.sta.config() event callbacks
 #define WIFI_SDK_EVENT_MONITOR_ENABLE
+
+//  Enable creation on the wifi.eventmon.reason table
 #define WIFI_EVENT_MONITOR_DISCONNECT_REASON_LIST_ENABLE
 
+//  Enable use of the WiFi.monitor sub-module
+//#define LUA_USE_MODULES_WIFI_MONITOR
+
 
 // Whilst the DNS client details can be configured through the WiFi API,
 // the defaults can be exposed temporarily during start-up.  The following
@@ -196,19 +206,24 @@ extern void luaL_dbgbreak(void);
 #endif
 #endif
 
+#if !defined(LUA_NUMBER_INTEGRAL) && defined (LUA_DWORD_ALIGNED_TVALUES)
+  #define LUA_PACK_TVALUES
+#else
+  #undef LUA_PACK_TVALUES
+#endif
+
 #ifdef DEVELOP_VERSION
 #define NODE_DEBUG
 #define COAP_DEBUG
 #endif /* DEVELOP_VERSION */
 
-#define NODE_ERROR
-
 #ifdef NODE_DEBUG
 #define NODE_DBG dbg_printf
 #else
 #define NODE_DBG
 #endif	/* NODE_DEBUG */
 
+#define NODE_ERROR
 #ifdef NODE_ERROR
 #define NODE_ERR dbg_printf
 #else
@@ -218,9 +233,14 @@ extern void luaL_dbgbreak(void);
 // #define GPIO_SAFE_NO_INTR_ENABLE
 #define ICACHE_STORE_TYPEDEF_ATTR __attribute__((aligned(4),packed))
 #define ICACHE_STORE_ATTR __attribute__((aligned(4)))
-#define ICACHE_RAM_STRING(x) ICACHE_RAM_STRING2(x)
+#define ICACHE_STRING(x) ICACHE_STRING2(x)
-#define ICACHE_RAM_STRING2(x) #x
+#define ICACHE_STRING2(x) #x
-#define ICACHE_RAM_ATTR     __attribute__((section(".iram0.text." __FILE__ "." ICACHE_RAM_STRING(__LINE__))))
+#define ICACHE_RAM_ATTR \
+  __attribute__((section(".iram0.text." __FILE__ "." ICACHE_STRING(__LINE__))))
+#define ICACHE_FLASH_RESERVED_ATTR \
+  __attribute__((section(".irom.reserved." __FILE__ "." ICACHE_STRING(__LINE__)),\
+                 used,unused,aligned(INTERNAL_FLASH_SECTOR_SIZE)))
+
 #ifdef  GPIO_SAFE_NO_INTR_ENABLE
 #define NO_INTR_CODE ICACHE_RAM_ATTR __attribute__ ((noinline))
 #else
@@ -228,4 +248,3 @@ extern void luaL_dbgbreak(void);
 #endif
 
 #endif	/* __USER_CONFIG_H__ */
-

app/libc/c_stdlib.c

@@ -15,18 +15,6 @@
 #include "c_stdlib.h"
 #include "c_types.h"
 #include "c_string.h"
-
-extern const char lua_init_value[];
-
-const char *c_getenv(const char *__string)
-{
-    if (c_strcmp(__string, "LUA_INIT") == 0)
-    {
-        return lua_init_value;
-    }
-    return NULL;
-}
-
 #include <_ansi.h>
 //#include <reent.h>
 //#include "mprec.h"

app/libc/c_stdlib.h

@@ -46,8 +46,7 @@
 
 // void c_exit(int);
 
-// c_getenv() get env "LUA_INIT" string for lua initialization.
+//const char *c_getenv(const char *__string);
-const char *c_getenv(const char *__string);
 
 // void *c_malloc(size_t __size);
 // void *c_zalloc(size_t __size);

app/lua/lauxlib.c

@@ -48,7 +48,7 @@
 //#define DEBUG_ALLOCATOR
 #ifdef DEBUG_ALLOCATOR 
 #ifdef LUA_CROSS_COMPILER
-static void break_hook(void)
+static void break_hook(void) {}
 #define ASSERT(s) if (!(s)) {break_hook();}
 #else
 #define ASSERT(s) if (!(s)) {asm ("break 0,0" ::);}
@@ -186,7 +186,7 @@ void *debug_realloc (void *b, size_t oldsize, size_t size) {
 /* }====================================================================== */
 #else
 #define this_realloc(p,os,s) c_realloc(p,s)
-#endif
+#endif /* DEBUG_ALLOCATOR */
 
 /*
 ** {======================================================

app/lua/lflash.c

@@ -41,7 +41,7 @@ static uint32_t curOffset;
 #define FLASH_PAGE_SIZE INTERNAL_FLASH_SECTOR_SIZE
 #define FLASH_PAGES  (FLASH_SIZE/FLASH_PAGE_SIZE)
 
-#define BREAK_ON_STARTUP_PIN  1  // GPIO 5 or setting to 0 will disable pin startup 
+char flash_region_base[FLASH_SIZE] ICACHE_FLASH_RESERVED_ATTR;
 
 #ifdef NODE_DEBUG
 extern void dbg_printf(const char *fmt, ...) __attribute__ ((format (printf, 1, 2)));
@@ -94,6 +94,7 @@ static char *flashBlock(const void* b, size_t size)  {
   return cur;
 }
 
+
 static void flashErase(uint32_t start, uint32_t end){
   int i;
   if (start == -1) start = FLASH_PAGES - 1;
@@ -104,38 +105,41 @@ static void flashErase(uint32_t start, uint32_t end){
 }
 
 
-/* =====================================================================================
- * Hook in user_main.c to allocate flash memory for the lua flash store
- */
-extern void luaL_dbgbreak(void);   //<<<<<<<<<<<<< Temp
-void luaN_user_init(void) {
-  curOffset    = 0;
-  flashSector = platform_flash_reserve_section( FLASH_SIZE, &flashAddrPhys );
-  flashAddr   = cast(char *,platform_flash_phys2mapped(flashAddrPhys));
-  NODE_DBG("Flash initialised: %x %08x\n", flashSector, flashAddr); 
-//  luaL_dbgbreak();      //<<<<<<<<<<<<< Temp
-}
-
-
 /*
  * Hook in lstate.c:f_luaopen() to set up ROstrt and ROpvmain if needed
  */  
 LUAI_FUNC void luaN_init (lua_State *L) {
+//  luaL_dbgbreak();
+  curOffset      = 0;
+  flashAddr     = flash_region_base;
+  flashAddrPhys = platform_flash_mapped2phys((uint32_t)flashAddr);
+  flashSector   = platform_flash_get_sector_of_address(flashAddrPhys);
   FlashHeader *fh = cast(FlashHeader *, flashAddr);
+
   /*
    * For the LFS to be valid, its signature has to be correct for this build variant,
    * thr ROhash and main proto fields must be defined and the main proto address 
    * be within the LFS address bounds. (This last check is primarily to detect the
    * direct imaging of an absolute LFS with the wrong base address. 
    */
-  if ((fh->flash_sig & (~FLASH_SIG_ABSOLUTE)) == FLASH_SIG && 
+
-      fh->pROhash   != ALL_SET &&
+  if ((fh->flash_sig & (~FLASH_SIG_ABSOLUTE)) != FLASH_SIG ) {
-      ((fh->mainProto - cast(FlashAddr, fh)) < fh->flash_size)) {
+    NODE_ERR("Flash sig not correct: %p vs %p\n",
+       fh->flash_sig & (~FLASH_SIG_ABSOLUTE), FLASH_SIG); 
+    return;
+  }
+
+  if (fh->pROhash == ALL_SET ||
+      ((fh->mainProto - cast(FlashAddr, fh)) >= fh->flash_size)) {
+    NODE_ERR("Flash size check failed: %p vs 0xFFFFFFFF; %p >= %p\n",
+       fh->mainProto - cast(FlashAddr, fh), fh->flash_size);
+    return;
+  }
+ 
   G(L)->ROstrt.hash = cast(GCObject **, fh->pROhash);
   G(L)->ROstrt.nuse = fh->nROuse ;
   G(L)->ROstrt.size = fh->nROsize;
   G(L)->ROpvmain    = cast(Proto *,fh->mainProto);
-  }
 }
 
 #define BYTE_OFFSET(t,f) cast(size_t, &(cast(t *, NULL)->f))

app/lua/lflash.h

@@ -10,14 +10,14 @@
 #include "lstate.h"
 #include "lzio.h"
 
-#ifdef LUA_NUNBER_INTEGRAL
+#ifdef LUA_NUMBER_INTEGRAL
 # define FLASH_SIG_B1 0x02
 #else
 # define FLASH_SIG_B1 0x00
 #endif
 
 #ifdef LUA_PACK_TVALUES
-#ifdef LUA_NUNBER_INTEGRAL
+#ifdef LUA_NUMBER_INTEGRAL
 #error "LUA_PACK_TVALUES is only valid for Floating point builds" 
 #endif
 # define FLASH_SIG_B2 0x04
@@ -40,7 +40,6 @@ typedef struct {
   lu_int32  fill2;          /* reserved */
 } FlashHeader;
 
-void luaN_user_init(void);
 LUAI_FUNC void luaN_init (lua_State *L);
 LUAI_FUNC int  luaN_flashSetup (lua_State *L);
 LUAI_FUNC int  luaN_reload_reboot (lua_State *L);

app/lua/lgc.c

@@ -65,7 +65,8 @@
 static void removeentry (Node *n) {
   lua_assert(ttisnil(gval(n)));
   if (ttype(gkey(n)) != LUA_TDEADKEY && iscollectable(gkey(n)))
-    lua_assert(!isLFSobject(&((gkey(n))->value.gc->gch)));
+//  The gkey is always in RAM so it can be marked as DEAD even though it
+//  refers to an LFS object.
     setttype(gkey(n), LUA_TDEADKEY);  /* dead key; remove it */
 }
 

app/lua/loadlib.c

@@ -613,7 +613,7 @@ static int ll_seeall (lua_State *L) {
 
 static void setpath (lua_State *L, const char *fieldname, const char *envname,
                                    const char *def) {
-  const char *path = c_getenv(envname);
+  const char *path = NULL;  /* getenv(envname) not used in NodeMCU */;
   if (path == NULL)  /* no environment variable? */
     lua_pushstring(L, def);  /* use default */
   else {

app/lua/lua.c

@@ -29,11 +29,9 @@
 
 lua_State *globalL = NULL;
 
-lua_Load gLoad;
+static lua_Load gLoad;
-
 static const char *progname = LUA_PROGNAME;
 
-
 static void l_message (const char *pname, const char *msg) {
 #if defined(LUA_USE_STDIO)
   if (pname) c_fprintf(c_stderr, "%s: ", pname);
@@ -237,19 +235,16 @@ static int runargs (lua_State *L, char **argv, int n) {
 }
 
 
-#ifdef LUA_INIT_STRING
+#ifndef LUA_INIT_STRING
-const char lua_init_value[] = LUA_INIT_STRING;
+#define LUA_INIT_STRING "@init.lua"
-#else
-const char lua_init_value[] = "@init.lua";
 #endif
 
 static int handle_luainit (lua_State *L) {
-  const char *init = c_getenv(LUA_INIT);
+  const char *init = LUA_INIT_STRING;
-  if (init == NULL) return 0;  /* status OK */
+  if (init[0] == '@')
-  else if (init[0] == '@')
     return dofsfile(L, init+1);
   else
-    return dostring(L, init, "=" LUA_INIT);
+    return dostring(L, init, LUA_INIT);
 }
 
 
@@ -284,7 +279,6 @@ static int pmain (lua_State *L) {
 
 static void dojob(lua_Load *load);
 static bool readline(lua_Load *load);
-char line_buffer[LUA_MAXINPUT];
 
 #ifdef LUA_RPC
 int main (int argc, char **argv) {
@@ -315,24 +309,39 @@ int lua_main (int argc, char **argv) {
   gLoad.L = L;
   gLoad.firstline = 1;
   gLoad.done = 0;
-  gLoad.line = line_buffer;
+  gLoad.line = c_malloc(LUA_MAXINPUT);
   gLoad.len = LUA_MAXINPUT;
   gLoad.line_position = 0;
   gLoad.prmt = get_prompt(L, 1);
 
   dojob(&gLoad);
 
-  NODE_DBG("Heap size::%d.\n",system_get_free_heap_size());
+  NODE_DBG("Heap size:%d.\n",system_get_free_heap_size());
   legc_set_mode( L, EGC_ALWAYS, 4096 );
   // legc_set_mode( L, EGC_ON_MEM_LIMIT, 4096 );
   // lua_close(L);
   return (status || s.status) ? EXIT_FAILURE : EXIT_SUCCESS;
 }
 
+int lua_put_line(const char *s, size_t l) {
+  if (s == NULL || ++l < LUA_MAXINPUT || gLoad.line_position > 0)
+    return 0;
+  c_memcpy(gLoad.line, s, l);
+  gLoad.line[l] = '\0';
+  gLoad.line_position = l;
+  gLoad.done = 1;
+  NODE_DBG("Get command: %s\n", gLoad.line);
+  return 1;
+}
+
 void lua_handle_input (bool force)
 {
-  while (gLoad.L && (force || readline (&gLoad)))
+  while (gLoad.L && (force || readline (&gLoad))) { 
+    NODE_DBG("Handle Input: first=%u, pos=%u, len=%u, actual=%u, line=%s\n", gLoad.firstline,
+              gLoad.line_position, gLoad.len, c_strlen(gLoad.line), gLoad.line);
     dojob (&gLoad);
+    force = false;
+  }
 }
 
 void donejob(lua_Load *load){
@@ -462,11 +471,12 @@ static bool readline(lua_Load *load){
         {
           /* Get a empty line, then go to get a new line */
           c_puts(load->prmt);
+          continue;
         } else {
           load->done = 1;
           need_dojob = true;
+          break;
         }
-        continue;
       }
 
       /* other control character or not an acsii character */

app/lua/luac_cross/lflashimg.c

@@ -155,7 +155,7 @@ static void *fromFashAddr(FlashAddr a) {
  * Add a TS found in the Proto Load to the table at the ToS
  */
 static void addTS(lua_State *L, TString *ts) {
-  lua_assert(ttisstring(&(ts->tsv)));
+  lua_assert(ts->tsv.tt==LUA_TSTRING);
   lua_pushnil(L);
   setsvalue(L, L->top-1, ts);
   lua_pushinteger(L, 1);

app/modules/node.c

@@ -169,27 +169,17 @@ static int node_heap( lua_State* L )
   return 1;
 }
 
-extern lua_Load gLoad;
+extern int  lua_put_line(const char *s, size_t l);
 extern bool user_process_input(bool force);
+
 // Lua: input("string")
-static int node_input( lua_State* L )
+static int node_input( lua_State* L ) {
-{
   size_t l = 0;
   const char *s = luaL_checklstring(L, 1, &l);
-  if (s != NULL && l > 0 && l < LUA_MAXINPUT - 1)
+  if (lua_put_line(s, l)) {
-  {
+    NODE_DBG("Result (if any):\n");
-    lua_Load *load = &gLoad;
-    if (load->line_position == 0) {
-      c_memcpy(load->line, s, l);
-      load->line[l + 1] = '\0';
-      load->line_position = c_strlen(load->line) + 1;
-      load->done = 1;
-      NODE_DBG("Get command:\n");
-      NODE_DBG(load->line); // buggy here
-      NODE_DBG("\nResult(if any):\n");
     user_process_input(true);
   }
-  }
   return 0;
 }
 

app/platform/common.c

@@ -24,7 +24,7 @@ extern char _flash_used_end[];
 
 // Helper function: find the flash sector in which an address resides
 // Return the sector number, as well as the start and end address of the sector
-static uint32_t flashh_find_sector( uint32_t address, uint32_t *pstart, uint32_t *pend )
+static uint32_t flash_find_sector( uint32_t address, uint32_t *pstart, uint32_t *pend )
 {
 #ifdef INTERNAL_FLASH_SECTOR_SIZE
   // All the sectors in the flash have the same size, so just align the address
@@ -53,7 +53,7 @@ static uint32_t flashh_find_sector( uint32_t address, uint32_t *pstart, uint32_t
 
 uint32_t platform_flash_get_sector_of_address( uint32_t addr )
 {
-  return flashh_find_sector( addr, NULL, NULL );
+  return flash_find_sector( addr, NULL, NULL );
 }
 
 uint32_t platform_flash_get_num_sectors(void)
@@ -67,54 +67,22 @@ uint32_t platform_flash_get_num_sectors(void)
 #endif // #ifdef INTERNAL_FLASH_SECTOR_SIZE
 }
 
-static uint32_t allocated = 0;
-static uint32_t phys_flash_used_end = 0;  //Phyiscal address of last byte in last flash used sector 
-
-uint32_t platform_flash_reserve_section( uint32_t regsize, uint32_t *start )
-{
-  // Return Flash sector no (and optional flash mapped address of first allocated byte)
-
-  if(phys_flash_used_end == 0) 
-    flashh_find_sector(platform_flash_mapped2phys( (uint32_t)_flash_used_end - 1), NULL, &phys_flash_used_end );
-
-  /* find sector and last byte address of previous allocation */
-  uint32_t end;
-  uint32_t sect = flashh_find_sector( phys_flash_used_end + allocated, NULL, &end );
-  if(start)
-    *start = end + 1;
-
-  /* allocated regions are always sector aligned */ 
-  flashh_find_sector( phys_flash_used_end + allocated + regsize, NULL, &end );
-  allocated = end - phys_flash_used_end;
-
-  NODE_DBG("Flash base: %08x %08x %08x\n", regsize, allocated, phys_flash_used_end);
-  return sect + 1;
-}
-
 uint32_t platform_flash_get_first_free_block_address( uint32_t *psect )
 {
   // Round the total used flash size to the closest flash block address
   uint32_t start, end, sect;
   NODE_DBG("_flash_used_end:%08x\n", (uint32_t)_flash_used_end);
-#if 0
   if(_flash_used_end>0){ // find the used sector
-    sect = flashh_find_sector( platform_flash_mapped2phys ( (uint32_t)_flash_used_end - 1), NULL, &end );
+    sect = flash_find_sector( platform_flash_mapped2phys ( (uint32_t)_flash_used_end - 1), NULL, &end );
-    sect++;
+    if( psect )
-    start = end + 1;
+      *psect = sect + 1;
-  }else{
+    return end + 1;
-    sect = flashh_find_sector( 0, &start, NULL ); // find the first free sector
+  } else {
-  }
+    sect = flash_find_sector( 0, &start, NULL ); // find the first free sector
-  if(_flash_used_end>0){ // find the used sector
-    uint32_t sta1, sec1;
-    sec1 = platform_flash_reserve_section( 0, &sta1 ); 
-    NODE_DBG("Flash base: %p %p %p %p\n", sect, start, sec1, sta1);
-  }
-#endif
-  sect = _flash_used_end ? platform_flash_reserve_section( 0, &start ) :
-                           flashh_find_sector( 0, &start, NULL );
     if( psect )
       *psect = sect;
     return start;
+  }
 }
 
 uint32_t platform_flash_write( const void *from, uint32_t toaddr, uint32_t size )

app/platform/flash_api.c

@@ -116,10 +116,11 @@ bool flash_rom_set_size_type(uint8_t size)
     uint8_t data[SPI_FLASH_SEC_SIZE] ICACHE_STORE_ATTR;
     if (SPI_FLASH_RESULT_OK == spi_flash_read(0, (uint32 *)data, SPI_FLASH_SEC_SIZE))
     {
-        ((SPIFlashInfo *)(&data[0]))->size = size;
+        NODE_DBG("\nflash_rom_set_size_type(%u), was %u\n", size, ((SPIFlashInfo *)data)->size );
+        ((SPIFlashInfo *)data)->size = size;
         if (SPI_FLASH_RESULT_OK == spi_flash_erase_sector(0 * SPI_FLASH_SEC_SIZE))
         {
-            NODE_DBG("\nERASE SUCCESS\n");
+            NODE_DBG("\nSECTOR 0 ERASE SUCCESS\n");
         }
         if (SPI_FLASH_RESULT_OK == spi_flash_write(0, (uint32 *)data, SPI_FLASH_SEC_SIZE))
         {
@@ -266,6 +267,7 @@ bool flash_rom_set_speed(uint32_t speed)
     if (SPI_FLASH_RESULT_OK == spi_flash_read(0, (uint32 *)data, SPI_FLASH_SEC_SIZE))
     {
         ((SPIFlashInfo *)(&data[0]))->speed = speed_type;
+        NODE_DBG("\nflash_rom_set_speed(%u), was %u\n", speed_type, ((SPIFlashInfo *)(&data[0]))->speed );
         if (SPI_FLASH_RESULT_OK == spi_flash_erase_sector(0 * SPI_FLASH_SEC_SIZE))
         {
             NODE_DBG("\nERASE SUCCESS\n");

app/platform/platform.c

@@ -879,7 +879,7 @@ uint32_t platform_s_flash_write( const void *from, uint32_t toaddr, uint32_t siz
   if(SPI_FLASH_RESULT_OK == r)
     return size;
   else{
-    NODE_ERR( "ERROR in flash_write: r=%d at %p\n", ( int )r, ( unsigned )toaddr);
+    NODE_ERR( "ERROR in flash_write: r=%d at %p\n", r, toaddr);
     return 0;
   }
 }
@@ -917,35 +917,33 @@ uint32_t platform_s_flash_read( void *to, uint32_t fromaddr, uint32_t size )
   if(SPI_FLASH_RESULT_OK == r)
     return size;
   else{
-    NODE_ERR( "ERROR in flash_read: r=%d at %p\n", ( int )r, ( unsigned )fromaddr);
+    NODE_ERR( "ERROR in flash_read: r=%d at %p\n", r, fromaddr);
     return 0;
   }
 }
 
 int platform_flash_erase_sector( uint32_t sector_id )
 {
+  NODE_DBG( "flash_erase_sector(%u)\n", sector_id);
   system_soft_wdt_feed ();
   return flash_erase( sector_id ) == SPI_FLASH_RESULT_OK ? PLATFORM_OK : PLATFORM_ERR;
 }
 
-static uint32_t flash_map_meg_offset ()
+static uint32_t flash_map_meg_offset (void) {
-{
   uint32_t cache_ctrl = READ_PERI_REG(CACHE_FLASH_CTRL_REG);
   if (!(cache_ctrl & CACHE_FLASH_ACTIVE))
     return -1;
-  bool b0 = (cache_ctrl & CACHE_FLASH_MAPPED0) ? 1 : 0;
+  uint32_t m0 = (cache_ctrl & CACHE_FLASH_MAPPED0) ? 0x100000 : 0;
-  bool b1 = (cache_ctrl & CACHE_FLASH_MAPPED1) ? 1 : 0;
+  uint32_t m1 = (cache_ctrl & CACHE_FLASH_MAPPED1) ? 0x200000 : 0;
-  return ((b1 << 1) | b0) * 0x100000;
+  return m0 + m1;
 }
 
-uint32_t platform_flash_mapped2phys (uint32_t mapped_addr)
+uint32_t platform_flash_mapped2phys (uint32_t mapped_addr)  {
-{
   uint32_t meg = flash_map_meg_offset();
   return (meg&1) ? -1 : mapped_addr - INTERNAL_FLASH_MAPPED_ADDRESS + meg ;
 }
 
-uint32_t platform_flash_phys2mapped (uint32_t phys_addr)
+uint32_t platform_flash_phys2mapped (uint32_t phys_addr) {
-{
   uint32_t meg = flash_map_meg_offset();
   return (meg&1) ? -1 : phys_addr + INTERNAL_FLASH_MAPPED_ADDRESS - meg;
 }

app/platform/platform.h

@@ -268,11 +268,8 @@ uint32_t platform_eth_get_elapsed_time(void);
 // *****************************************************************************
 // Internal flash erase/write functions
 
-uint32_t platform_flash_reserve_section( uint32_t regsize, uint32_t *start );
 uint32_t platform_flash_get_first_free_block_address( uint32_t *psect );
 uint32_t platform_flash_get_sector_of_address( uint32_t addr );
-uint32_t platform_flash_mapped2phys (uint32_t mapped_addr);
-uint32_t platform_flash_phys2mapped (uint32_t phys_addr);
 uint32_t platform_flash_write( const void *from, uint32_t toaddr, uint32_t size );
 uint32_t platform_flash_read( void *to, uint32_t fromaddr, uint32_t size );
 uint32_t platform_s_flash_write( const void *from, uint32_t toaddr, uint32_t size );
@@ -282,13 +279,14 @@ int platform_flash_erase_sector( uint32_t sector_id );
 
 /**
  * Translated a mapped address to a physical flash address, based on the
- * current flash cache mapping.
+ * current flash cache mapping, and v.v.
  * @param mapped_addr Address to translate (>= INTERNAL_FLASH_MAPPED_ADDRESS)
  * @return the corresponding physical flash address, or -1 if flash cache is
  *  not currently active.
  * @see Cache_Read_Enable.
  */
 uint32_t platform_flash_mapped2phys (uint32_t mapped_addr);
+uint32_t platform_flash_phys2mapped (uint32_t phys_addr);
 
 // *****************************************************************************
 // Allocator support

app/user/user_main.c

@@ -129,8 +129,7 @@ bool user_process_input(bool force) {
     return task_post_low(input_sig, force);
 }
 
-void nodemcu_init(void)
+void nodemcu_init(void) {
-{
     NODE_ERR("\n");
     // Initialize platform first for lua modules.
     if( platform_init() != PLATFORM_OK )
@@ -139,11 +138,13 @@ void nodemcu_init(void)
         NODE_DBG("Can not init platform for modules.\n");
         return;
     }
-
+    uint32_t size_detected = flash_detect_size_byte();
-    if( flash_detect_size_byte() != flash_rom_get_size_byte() ) {
+    uint32_t size_from_rom = flash_rom_get_size_byte();
-        NODE_ERR("Self adjust flash size.\n");
+    if( size_detected != size_from_rom ) {
+        NODE_ERR("Self adjust flash size. 0x%x (ROM) -> 0x%x (Detected)\n", 
+                 size_from_rom, size_detected);
         // Fit hardware real flash size.
-        flash_rom_set_size_byte(flash_detect_size_byte());
+        flash_rom_set_size_byte(size_detected);
 
         system_restart ();
         // Don't post the start_lua task, we're about to reboot...
@@ -240,7 +241,6 @@ user_rf_cal_sector_set(void)
     return rf_cal_sec;
 }
 
-extern void luaN_user_init(void);
 /******************************************************************************
  * FunctionName : user_init
  * Description  : entry of user application, init user function here
@@ -261,10 +261,5 @@ void user_init(void)
 #ifndef NODE_DEBUG
     system_set_os_print(0);
 #endif
-
-#ifdef LUA_FLASH_STORE
-    luaN_user_init();
-#endif
-
     system_init_done_cb(nodemcu_init);
 }

docs/en/lcd.md

@@ -0,0 +1,101 @@
+## Lua Compact Debug (LCD)
+
+LCD (Lua Compact Debug) was developed in Sept 2015 by Terry Ellison as a patch to the Lua system to decrease the RAM usage of Lua scripts. This makes it possible to run larger Lua scripts on systems with limited RAM.  Its use is most typically for eLua-type applications, and in this version it targets the **NodeMCU** implementation for the ESP8266 chipsets.
+
+This section gives a full description of LCD. If you are writing **NodeMCU** Lua modules, then this paper will be of interest to you, as it shows how to use LCD in an easy to configure way.   *Note that the default `user_config.h` has enabled LCD at a level 2 stripdebug since mid-2016*.
+
+### Motivation
+
+The main issue that led me to write this patch is the relatively high Lua memory consumption of its embedded debug information, as this typically results in a 60% memory increase for most Lua code.  This information is generated when any Lua source is complied because the Lua parser uses this as meta information during the compilation process.  It is then retained by default for use in generating debug information.  The only standard method of removing this information is to use the “strip” option when precompiling source using a standard eLua **luac.cross** on the host, or (in the case of NodeMCU) using the `node.compile()` function on the target environment.
+
+Most application developers that are new to embedded development simply live with this overhead, because either they aren't familiar with these advanced techniques, or they want to keep the source line information in error messages for debugging.
+
+The standard Lua compiler generates fixed 4 byte instructions which are interpreted by the Lua VM during execution. The debug information consists of a map from instruction count to source line number (4 bytes per instruction) and two tables keyed by the names of each local and upvalue. These tables contain metadata on these variables used in the function. This information can be accessed to enable symbolic debugging of Lua source (which isn't supported on **NodeMCU** platforms anyway), and the line number information is also used to generate error messages.
+
+This overhead is sufficient large on limited RAM systems to replace this scheme by making two changes which optimize for space rather than time:
+
+-  The encoding scheme used in this patch typically uses 1 byte per source line instead of 4 bytes per instruction, and this represents a 10 to 20-fold reduction in the size of this vector.  The access time during compile is still **O(1)**, and **O(N)** during runtime error handling, where **N** is number of non-blank lines in the function.  In practice this might add a few microseconds to the time take to generate the error message for typical embedded functions.
+
+-  The line number, local and upvalue information is needed during the compilation of a given compilation unit (a source file or source string), but its only use after this is for debugging and so can be discarded. (This is what the `luac -s` option and `node.compile()` do).  The line number information if available is used in error reporting.  An extra API call has therefore been added to discarded this debug information on completion of the compilation.
+
+To minimise the impact within the C source code for the Lua system, an extra system define **LUA_OPTIMIZE_DEBUG** can be set in the `user_config.h` file to configure a given firmware build. This define sets the default value for all compiles and can take one of four values:
+
+1.  (or not defined) use the default Lua scheme.
+2.  Use compact line encoding scheme and retain all debug information.
+3.  Use compact line encoding scheme and only retain line number debug information.
+4.  Discard all debug information on completion of compile.
+
+Building the firmware with the 0 option compiles to the pre-patch version. Options 1-3 generate the `strip_debug()` function, which allows this default value to be set at runtime.
+
+_Note that options 2 and 3 can also change the default behaviour of the `loadstring()` function in that any functions declared within the string cannot inherited any outer locals within the parent hierarchy as upvalues if these have been stripped of the locals and upvalues information._
+
+### Details
+
+There are various API calls which compile and load Lua source code. During compilation each variable name is parsed, and is then resolved in the following order:
+
+-  Against the list of local variables declared so far in the current scope is scanned for a match.
+
+-  Against the local variable lists for each of the lexically parent functions are then scanned for a match, and if found the variable is tagged as an _upvalue_.
+
+-  If unmatched against either of these local scopes then the variable defaults to being a global reference.
+
+The parser and code generator must therefore access the line mapping, upvalues, and locals information tables maintained in each function Prototype header during source compilation.  This scoping scheme works because function compilation is recursive: if function A contains the definition of function B which contains the definition of function C, then the compilation of A is paused to compile B and this is in turn paused to compile C; then B complete and then A completes.
+
+The variable meta information is stored in standard Lua tables which are allocated using the standard Lua doubling algorithm and hence they can contain a lot of unused space.  The parser therefore calls `close_func()` once compilation of a function has been completed to trim these vectors to the final sizes.
+
+The patch makes the following if `LUA_OPTIMIZE_DEBUG` > 0. (The existing functionality is preserved if this define is zero or undefined.)
+
+-  It adds an extra API call: `stripdebug([level[, function]])`  as discussed below.
+
+-  It extends the trim logic in `close_func()` to replace this trim action by deleting the information according to the current default debug optimization level.
+
+-  The `lineinfo` vector associated with each function is replaced by a `packedlineinfo` string using a run length encoding scheme that uses a repeat of an optional line number delta (this is omitted if the line offset is zero) and a count of the number of instruction generated for that source line.  This scheme uses roughly an **M** byte vector where **M** is the number of non-blank source lines, as opposed to a **4N** byte vector where **N** is the number of VM instruction.  This vector is built sequentially during code generation so it is this patch conditionally replaces the current map with an algorithm to generate the packed version on the fly.
+
+The `stripdebug([level[, function]])` call is processed as follows:
+
+-  If both arguments are omitted then the function returns the current default strip level.
+
+-  If the function parameter is omitted, then the level is used as the default setting for future compiles.  The level must be 1-3 corresponding to the above debug optimization settings.  Hence if `stripdebug(3)` is included in **init.lua**, then all debug information will be stripped out of subsequently compiled functions. 
+
+-  The function parameter if present is parsed in the same way as the function argument in `setfenv()` (except that the integer 0 level is not permitted, and this function tree corresponding to this scope is walked to implement this debug optimization level.
+
+The `packedlineinfo` encoding scheme is as follows:
+
+-  It comprises a repeat of (optional) line delta + VM instruction count (IC) for that line starting from a base line number of zero.  The line deltas are optional because line deltas of +1 are assumed as default and therefore not emitted.
+
+-  ICs are stored as a single byte with the high bit set to zero.  Sequences longer than 126 instructions for a single sequence are rare, but can be are encoded using a multi byte sequence using 0 line deltas, e.g. 126 (0) 24 for a line generating 150 VM instructions.  The high bit is always unset, and note that this scheme reserves the code 0x7F as discussed below.
+
+-  Line deltas are stored with the high bit set and are variable (little-endian) in length.  Since deltas are always delimited by an IC which has the top bit unset, the length of each delta can be determined from these delimiters.  Deltas are stored as signed ones-compliment with the sign bit in the second bit of low order byte, that is in the format (in binary) `1snnnnnnn [1nnnnnnn]*`, with `s` denoting the sign and `n…n` the value element using the following map.  This means that a single byte is used encode line deltas in the range -63 … 65; two bytes used to encode line deltas in the range -8191 … 8193, etc..
+```C
+  value = (sign == 1) ? -delta : delta - 2
+```
+
+-  This approach has no arbitrary limits, in that it can accommodate any line delta or IC count. Though in practice, most deltas are omitted and multi-byte sequences are rarely generated.
+
+-  The codes 0x00 and 0x7F are reserved in this scheme.  This is because Lua allocates such growing vectors on a size-doubling basis.  The line info vector is always null terminated so that the standard **strlen()** function can be used to determine its length.  Any unused bytes between the last IC and the terminating null are filled with 0x7F.
+
+The current mapping scheme has **O(1)** access, but with a code-space overhead of some 140%. This alternative approach has been designed to be space optimized rather than time optimized.  It requires the actual IC to line number map to be computed by linearly enumerating the string from the low instruction end during execution, resulting in an **O(N)** access cost, where **N** is the number of bytes in the encoded vector.  However, code generation builds this information incrementally, and so only appends to it (or occasionally updates the last element's line number), and the patch adds a couple of fields to the parser `FuncState` record to enable efficient **O(1)**  access during compilation.
+
+### Testing
+
+Essentially testing any eLua compiler or runtime changes are a total pain, because eLua is designed to be build against a **newlib**-based ELF.  Newlib uses a stripped down set of headers and libraries that are intended for embedded use (rather than being ran over a standard operating system).  Gdb support is effectively non-existent, so I found it just easier first to develop this code on a standard Lua build running under Linux (and therefore with full gdb support), and then port the patch to NodeMCU once tested and working.
+
+I tested my patch in standard Lua built with "make generic" and against the [Lua 5.1 suite](http://lua-users.org/lists/lua-l/2006-03/msg00723.html). The test suite was an excellent testing tool, and it revealed a number of cases that exposed logic flaws in my approach, resulting from Lua's approach of not carrying out inline status testing by instead implementing a throw / catch strategy.  In fact I realised that I had to redesign the vector generation algorithm to handle this robustly.
+
+As with all eLua builds the patch assumes Lua will not be executing in a multithreaded environment with OS threads running different lua_States. (This is also the case for the NodeMCU firmware). It executes the full test suite cleanly as maximum test levels and I also added some specific tests to cover new **stripdebug** usecases.
+
+Once this testing was completed, I then ported the patch to the NodeMCU build.  This was pretty straight forward as this code is essentially independent of the NodeMCU functional changes. The only real issue as to ensure that the NodeMCU `c_strlen()` calls replaced the standard `strlen()`, etc.
+
+I then built both `luac.cross` and firmware images with the patch disable to ensure binary compatibility with the non-patched version and then with the patch enabled at optimization level 3.
+
+In use there is little noticeable difference other than the code size during development are pretty much the same as when running with `node.compile()` stripped code.  The new option 2 (retaining packed line info only) has such a minimal size impact that its worth using this all the time.  I've also added a separate patch to NodeMCU (which this assumes) so that errors now generate a full traceback.
+
+### How to enable LCD
+
+Enabling LCD is simple: all you need is a patched version and define `LUA_OPTIMIZE_DEBUG` at the default level that you want in `app/include/user_config.h` and do a normal make. 
+
+Without this define enabled, the unpatched version is generated.
+
+Note that since `node.compile()` strips all debug information, old **.lc** files generated by this command will still run under the patched firmware, but binary files which retain debug information will not work across patched and non-patched versions.
+
+Other than optionally including a `node.stripdebug(N)` or whatever in your **init.lua**, the patch is otherwise transparent at an application level.

docs/en/lfs.md

@@ -0,0 +1,246 @@
+# Lua Flash Store (LFS)
+
+## Background
+
+An IoT device such as the ESP8266 has very different processor characteristics from the CPU in a typical PC:
+
+-  Conventional CPUs have a lot of RAM, typically more than 1 Gb, that is used to store both code and data.  IoT processors like the ESP variants use a [modified Harvard architecture](https://en.wikipedia.org/wiki/Modified_Harvard_architecture) where code can also be executed out of flash memory that is mapped into a address region separate from the limited RAM.
+
+-  Conventional CPU motherboards include RAM and a lot of support chips. ESP modules are postage stamp-sized and typically comprise one ESP [SoC](https://en.wikipedia.org/wiki/System_on_a_chip) and a flash memory chip used to store firmware and a limited file system.
+
+Lua was originally designed as a general embeddable extension language for applications that would typically run on systems such as a PC, but its design goals of speed, portability, small kernel size, extensibility and ease-of-use also make Lua a good choice for embedded use on an IoT platform.  Our NodeMCU firmware implementation was therefore constrained by the standard Lua core runtime system (**RTS**) that assumes a conventional CPU architecture with both Lua code and data in RAM; however ESP8266 modules only have approximately 48Kb RAM for application use, even though the firmware itself executes out of the larger flash-based program memory.
+
+This Lua Flash Store (**LFS**) patch modifies the NodeMCU Lua RTS to allow Lua code and its associated constant data to be executed directly out of flash-memory, just as the firmware itself is executed. This now enables NodeMCU Lua developers to create Lua applications with up to 256Kb Lua code and read-only (**RO**) constants executing out of flash, so that all of the RAM is available for read-write (**RW**) data.
+
+Though the ESP architecture does allow RW operations to flash, these are constrained by the write limitations of NAND flash architecture, as writing involves the block erasing of 4Kb pages and then overwriting each pach with new content.  Whilst it is possible (as with SPIFFS) to develop R/W file systems working within this constraint, memory-mapped read access to flash is cached through a RAM cache in order to accelerate code execution, and this makes it practically impossible to modify executable code pages on the fly. Hence the LFS patch must work within a reflash-and-restart paradigm for reloading the LFS.
+
+The LFS patch does this by adding two API new calls to the `node` module: one to reflash the LFS and restart the processor, and one to access the LFS store once loaded.  Under the hood, it also addresses all of the technical issues to make this magic happen.
+
+The remainder of this paper is split into two sections:
+
+-  The first section provides an overview the issues that a Lua developer needs to understand at an application level to use LFS effectively.
+
+-  The second gives more details on the technical issues that were addressed in order to implement the patch. This is a good overview for those that are interested, but many application programmers won't care how the magic happens, just that it does.
+
+
+## Using LFS
+
+### Selecting the firmware
+
+Power developers might want to use Docker or their own build environment as per our [Building the firmware](https://nodemcu.readthedocs.io/en/master/en/build/) documentation, and so `app/include/user_config.h` has now been updated to include the necessary documentation on how to select the configuration options to make an LFS firmware build.
+
+However, most Lua developers seem to prefer the convenience of our [Cloud Build Service](https://nodemcu-build.com/), so we will add two extra menu options to facilitate building LFS images:
+
+Variable | Option
+---------|------------
+LFS size | (none, 32Kb, 64Kb, 94Kb) The default is none. Selecting a numeric value builds in the corresponding LFS.
+SPIFFS size | (default or a multiple of 64Kb) The cloud build will base the SPIFFS at 1Mb if an explicit size is specified.
+
+You must choose an explicit (non-default) LFS size to enable the use of LFS.  Whilst you can use a default (maximal) SPIFFS configuration, most developers find it more useful to work with a fixed SPIFFS that has been sized to match their application reqirements.
+
+### Choosing your development lifecycle
+
+ The build environment for generating the firmware images is Linux-based, but as you can use our cloud build service to generate these, you can develop NodeMCU applications on pretty much any platform including Windows and MacOS. Unfortunately LFS images must be built off-ESP on a host platform, so you must be able to run the `luac.cross` cross compiler on your development machine to build LFS images.
+
+-  For Windows 10 developers, the easiest method of achieving this is to install the [Windows Subsystem for Linux](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux).  Most WSL users install the Ubuntu Bash shell as well; note that this is just a shell and some core GNU utilities (somewhat similar to Cygwin) rather than a full Ubuntu OS, as WSL extends the NT kernel to support the direct execution of Linux ELF images. WSL can directly run the `luac.cross` and `spiffsimg` that are build as part of the firmware. You will also need the `esptool.py` tool but `python.org` already provides Python releases for Windows.
+
+-  Linux users can just use these tools natively.
+
+-  In principle, only the build environment components needed to support `luac.cross` and `spiffsimg` are the `app/lua/lua_cross` and `tools/spifsimg` subdirectory makes. It should be straight forward to get these working under any environment which provides POSIX runtime support, including MacOS and Cygwin (for windows versions prior to Win10), but suitable developer effort is required to generate suitable executables; any volunteers?
+
+Most Lua developers seem to start with the [ESPlorer](https://github.com/4refr0nt/ESPlorer) tool, a 'simple to use' IDE that enables beginning Lua developers to get started.  However, ESPlorer relies on a UART connection and this can be slow and cumbersome, and it doesn't scale well for larger ESP application.
+
+So many experienced Lua developers switch to a rapid development cycle where they use a development machine to maintain your master Lua source. Going this route will allow you use your favourite program editor and source control, with one of various techniques for compiling the lua on-host and downloading the compiled code to the ESP:
+
+-  If you use a fixed SPIFFS image (I find 128Kb is enough for most of my applications), then you can script recompiling your LC files, rebuilding a SPIFFS image and loading it onto the ESP using `esptool.py` in less than 60 sec. You can either embed the LFS.img in the SPIFFS, or you can use the `luac.cross -a` option to directly load the new LFS image into the LFS region within the firmware.
+
+-  I now have an LFS aware version of my LuaOTA provisioning system (see `lua_examples/luaOTA`).  This handles all of the incremental compiling and LFS reloads transparently. This is typically integrated into the ESP application.
+
+-  Another option would be to include the FTP and Telnet modules in the base LFS image and to use telnet and FTP to update your system.  (Given that a 64Kb LFS can store thousands of lines of Lua, doing this isn't much of an issue.)
+
+My current practice is to use a small bootstrap `init.lua` file in SPIFFS to load the `_init` module from LFS, and this does all of the actual application initialisation. My `init.lua`:
+
+-  Is really a Lua binary (`.lc`) file renamed to a `.lua` extension.  Using a binary init file avoids loading the Lua compiler.  This works because even though the firmware looks for `init.lua`, the file extension itself is a just a convention; it is treated as a Lua binary if it has the correct Lua binary header.
+
+-  Includes a 1 sec delay before connecting to the Wifi.  This is a "just in case" when I am developing.  This is enough to allow me to paste a `file.remove'init.lua'` into the UART if I want to do different development paths.
+
+No doubt some standard usecase / templates will be developed by the community over the next six months.
+
+### Programming Techniques and approachs
+
+I have found that moving code into LFS has changed my coding style, as I tend to use larger modules and I don't worry about in-memory code size. This facilitates a more 'keep it simple stupid' coding style, so my ESP Lua code now looks more similar to host-based Lua code. I still prefer to keep the module that I am currently testing in SPIFFS, and only move modules into LFS once they are stable. However if you use `require` to load modules then this can all be handled by the require loader.
+
+Here is the code fragment that I use in my `_init` module to do this magic:
+
+```Lua
+do
+  local index = node.flashindex
+  -- Setup the LFS object
+  local lfs_t = {
+    __index = function(_, name)
+        local fn_ut, ba, ma, size, modules = index(name)
+        if not ba then
+          return fn_ut
+        elseif name == '_time' then
+          return fn_ut
+        elseif name == '_config' then
+          local fs_ma, fs_size = file.fscfg()
+          return {ba, ma, fs_ma, size, fs_size}
+        elseif name == '_list' then
+          return modules
+        else
+          return nil
+        end
+            end,
+    __newindex = function(_, name, value)
+        error("LFS is readonly. Invalid write to LFS." .. name, 2)
+      end
+  }
+  rawset(getfenv(),'LFS', setmetatable(lfs_t,lfs_t))
+  -- And add LFS to the require path list
+  local function loader_flash(module)
+    local fn, ba = index(module)
+    return ba and "Module not in LFS" or fn
+  end
+  package.loaders[3] = loader_flash
+
+end
+```
+
+Once this has been executed, if you have a function module `func1` in LFS, then `LFS.func1(x,y,z)` just works as you would expect.  The LFS properties `_time`, `_config` and `_list` can be used to access the other LFS metadata that you need.
+
+Of course, if you use Lua modules to build your application then `require "some_module"` will automatically path in and load your modules from LFS.  Note that SPIFFS is still ahead of LFS in this search list, so if you have a dev version in SPIFFS, say, then this will be loaded first. However, if you want to want to swap this search order so that the LFS is searched first, then set `package.loaders[1] = loader_flash` in your `_init` code.  If you need to swap the search order temporarily for development or debugging, then do this after you've run the `_init` code:
+
+```Lua
+do local pl = package.loaders; pl[1],pl[3] = pl[3],pl[1]; end
+```
+
+Whilst LFS is primarily used to store compiled modules, it also includes its own string table and any strings loaded into this can be used in your Lua application without taking any space in RAM. Hence, you might also want to preload any other frequently used strings into LFS as this will both save RAM use and reduced the Lua-custom Garbage Collector (**LGC**) overheads.
+
+The patch adds an extra debug function `getstrings()` function to help you determine what strings are worth adding to LFS.  This takes an optional string argument `'RAM'` (the default) or `'ROM'`, and returns a list of the strings in the corresponding table. So the following example can be used to get a listing of the strings in RAM.  You can enter the following Lua at the interactive prompt or call it as a debug function during a running application in order to generate this string list.
+
+```Lua
+do
+  local a=debug.getstrings'RAM'
+  for i =1, #a do a[i] = ('%q'):format(a[i]) end
+  print ('local preload='..table.concat(a,','))
+end
+```
+
+If you then create a file, say `LFS_dummy_strings.lua`, and put these `local preload` lines in it, and include this file in your `luac.cross -f`, then the cross compiler will generate a ROM string table that includes all strings referenced in this dummy module. You never need to call this module; just it's inclusion in the LFS build is enough to add the strings to the ROM table. Once in the ROM table, then you can use them subsequently in your application without incurring any RAM or LGC overhead. The following example is a useful starting point, but if needed then you can add to this
+for your application.
+
+```Lua
+local preload = "?.lc;?.lua", "@init.lua", "_G", "_LOADED", "_LOADLIB", "__add",
+"__call", "__concat", "__div", "__eq", "__gc", "__index", "__le", "__len", "__lt",
+"__mod", "__mode", "__mul", "__newindex", "__pow", "__sub", "__tostring", "__unm",
+"collectgarbage", "cpath", "debug", "file", "file.obj", "file.vol", "flash",
+"getstrings", "index", "ipairs", "list", "loaded", "loader", "loaders", "loadlib",
+"module", "net.tcpserver", "net.tcpsocket", "net.udpsocket", "newproxy", "package",
+"pairs", "path", "preload", "reload", "require", "seeall", "wdclr"
+```
+
+## Technical Issues
+
+Whilst memory capacity isn't a material constraint on most conventional machines, the Lua RTS still embeds some features to minimise overall memory usage. In particular:
+
+-  The more resource intensive data types are know as _collectable objects_, and the RTS includes a LGC which regularly scans these collectable resources to determine which are no longer in use, so that their associated memory can be reclaimed and reused.
+
+-  The Lua RTS also treats strings and compiled function code as collectable objects, so that these can also be LGCed when no longer referenced
+
+The compiled code, as executed by Lua RTS, internally comprises one or more function prototypes (which use a `Proto` structure type) plus their associated vectors (constants, instructions and meta data for debug). Most of these compiled constant types are basic (e.g. numbers) and the only collectable constant data type are strings. The other collectable types such as arrays are actually created at runtime by executing Lua compiled instructions to build each resource dynamically.
+
+Currently, when any Lua file is loaded into an ESP application, the RTS loads the corresponding compiled version into RAM. Each compiled function has its own Proto structure hierarchy, but this hierarchy is not exposed directly to the running application; instead the compiler generate `CLOSURE` instruction which is executed at runtime to bind the Proto to a Lua function value thus creating a [closure](https://en.wikipedia.org/wiki/Closure_(computer_programming)). Since this occurs at runtime, any `Proto` can be bound to multiple closures. A Lua closure can have multiple RW [Upvalues](https://www.lua.org/pil/27.3.3.html) bound to it, and so function value is much like a Lua object in that it is refering to something that can contain RW state, even though the `Proto` hierarchy itself is intrinsically RO.
+
+Whilst advanced ESP Lua programmers can use overlay techniques to ensure that only active functions are loaded into RAM and thus increase the effective application size, this adds to runtime and program complexity. Moving Lua "program" resources into ESP Flash addressable memory typically doubles the effective RAM available, and largley removes the need to complicate applications code to facilitate overlaying.
+
+Any RO resources that are relocated to a flash address space:
+
+-  Must not be collected. Also RW references to RO resources must be robustly handled by the LGC.
+-  Cannot reference to any volatile RW data elements (though RW resources can refer to RO resources).
+
+All strings in Lua are [interned](https://en.wikipedia.org/wiki/String_interning), so that only one copy of any string is kept in memory and most string manipulation uses the address of this single copy as a unique reference. This uniqueness and the LGC of strings is facilitated by using a global string table that is hooked into the Lua Global State. Under standard Lua, any new string is first resolved against RAM string table, with only the string-misses being added to the string table. The LFS patch adds a second RO string table in flash and which contains all strings used in LFS Protos. Maintaining integrity across the two RAM and RO string tables is simple and low-cost, with LFS resolution process extended across both the RAM and ROM string tables. Hence any strings already in the ROM string table will generate a unique string reference without the need to add an additional entry in the RAM table. This both significantly reduces the size of the RAM string table, and removes a lot of strings from the LCG scanning.
+
+Note that early development implementations of the LFS build process allowed on-target ESP builds. Unfortunately, we found in practice that the Lua compiler was so resource hungry that it was impractical to get this to scale to usable application sizes, and we therefore abandoned this approach, moving the LFS build process onto the development host machine by embedding this into `luac.cross`. This approach avoids all of the update integrity issues involved in building a new LFS which might require RO resources already referenced in the RW ones.
+
+Any LFS image can be loaded in the LFS store by one of two mechanisms:
+
+-  The image can be build on the host and then copied into SPIFFS. Calling the `node.flashreload()` API with this filename will load the image, and then schedule a restart to leave the ESP in normal application mode, but with an updated flash block.  This sequence is essentially atomic. Once called, the only exit is the reboot.
+
+-  The second option is to build the LFS image using the `-a` option to base it at the correct absolute address of the LFS store for a given firmware image.  The LFS can then be flashed to the ESP along with the firmware image.
+
+The LFS store is a fixed size for any given firmware build (configurable by the a pplication developer through `user_config.h`) and is at a build-specific base address within the `ICACHE_FLASH` address space.  This is used to store the ROM string table and the set of `Proto` hierarchies corresponding to a list of Lua files in the loaded image.
+
+A separate `node.flashindex()` function creates a new Lua closure based on a module loaded into LFS and more specfically its flash-based prototype; whilst this access function is not transparent at a coding level, this is no different functionally than already having to handle `lua` and `lc` files and the existing range of load functions (`load`,`loadfile`, `loadstring`). Either way, creating a closure on flash-based prototype is _fast_ in terms of runtime. (It is basically a single instruction rather than a compile, and it has minimal RAM impact.)
+
+### Basic approach
+
+This **LFS** patch uses two string tables: the standard Lua RAM-based table (`RWstrt`) and a second RO flash-based one (`ROstrt`). The `RWstrt` is searched first when resolving new string requests, and then the `ROstrt`. Any string not already in either table is then added to the `RWstrt`, so this means that the RAM-based string table only contains application strings that are not already defined in the `ROstrt`.
+
+Any Lua file compiled into the LFS image includes its main function prototype and all the child resources that are linked in its `Proto` structure; so all of these resources are compiled into the LFS image with this entire hierarchy self-consistently within the flash memory.
+
+```C
+   TValue        *k;         Constants used by the function
+   Instruction   *code       The Lua VM instuction codes
+   struct Proto **p;         Functions defined inside the function
+   int           *lineinfo;  Debug map from opcodes to source lines
+   struct LocVar *locvars;   Debug information about local variables
+   TString      **upvalues   Debug information about upvalue names
+   TString       *source     String name associated with source file
+ ```
+
+Such LFS images are created by `luac.cross` using the `-f` option, and this builds a flash image based on the list of modules provided but with a master "main" function of the form:
+
+```Lua
+local n = ...,1518283691  -- The Unix Time of the compile
+if n == "module1" then return module1 end
+if n == "module2" then return module2 end
+-- and so on
+if n == "moduleN" then return module2 end
+return 1518283691,"module1","module2", --[[ ... ]] ""moduleN"
+```
+
+You can't actually code this Lua because the modules are in separate compilation units, but the compiler being a compiler can just emit the compiled code directly. (See `app/lua/luac_cross/luac.c` for the details.)
+
+The deep cross-copy of the `Proto` hierarchy is also complicated because current hosts are typically 64bit whereas the ESPs are 32bit, so the structures need repacking. (See `app/lua/luac_cross/luac.c` for the details.)
+
+With this patch, the `luac.cross` build has been moved into the overall application hierarchy and is now simply a part of the NodeMCU make. The old Lua script has been removed from the `tools` directory, together with the need to have Lua preinstalled on the host.
+
+The LFS image is by default position independent, so is independent of the actual NodeMCU target image. You just have to copy it to the target file system and execute a `reload` to copy this to the correct location, relocating all address to the correct base. (See `app/lua/lflash.c` for the details.) This process is fast.  However, -a `luac.cross -a` also allows absolute address images to be built for direct flashing into the LFS store during provisioning.
+
+### Impact of the Lua Garbage Collector
+
+The LGC applies to what the Lua VM classifies as collectable objects (strings, tables, functions, userdata, threads -- known collectively as `GCObjects`). A simple two "colour" LGC was used in previous Lua versions, but Lua 5.1 introduced the Dijkstra's 3-colour (*white*, *grey*, *black*) variant that enabled the LGC to operate in an incremental mode. This permits smaller LGC steps interspersed by LGC pause, and is very useful for larger scale Lua implementations. Whilst this is probably not really needed for IoT devices, NodeMCU follows this standard Lua 5.1 implementation, albeit with the `elua` EGC changes.
+
+In fact, two *white* flavours are used to support incremental working (so this 3-colour algorithm really uses 4). All newly allocated collectable objects are marked as the current *white*, and include a link in their header to enable scanning through all such Lua objects. They may also be referenced directly or indirectly via one of the Lua application's *roots*: the global environment, the Lua registry and the stack. The LGC operates two broad phases: **mark** and **sweep**.
+
+The LGC algorithm is quite complex and assumes that all GCObjects are RW so that a flag byte within each object can be updated during the mark and sweep processing. LFS introduces GCObjects that are actually stored in RO memory and are therefore truly RO.  Any attempt to update their content during LGC will result in the firmware crashing with a memory exception, so the LFS patch must therefore modify the LGC processing to avoid such potential updates whilst maintaining its integrity, and the remainder of this
+section provides further detail on how this was achieved.
+
+The **mark** phase walks collectable objects by a recursive walk starting at at the LGC roots. (This is referred to as _traverse_.) Any object that is visited in this walk has its colour flipped from *white* to *grey* to denote that it is in use, and it is relinked into a grey list. The grey list is iteratively processed, removing one grey object at a time. Such objects can reference other objects (e.g. a table has many keys and values which can also be collectable objects), so each one is then also traversed and all objects reachable from it are marked, as above. After an object has been traversed, it's turned from grey to black. The LGC will walks all RW collectable objects, traversing the dependents of each in turn.  As RW objects can now refer to RO ones, the traverse routines has additinal tests to skip trying to mark any RO LFS references.
+
+The white flavour is flipped just before entering the **sweep** phase. This phase then loops over all collectable objects. Any objects found with previous white are no longer in user, and so can be freed. The 'current' white are kept; this prevents any new objected created during a paused sweep from being accidentally collected before being marked, but this means that it takes two sweeps to free all unused objects. There are other subtleties introduced in this 3-colour algorithm such as barriers and back-tracking to maintain integrity of the LGC, and these also needed extra rules to handle RO GCObjects correclty, but detailed explanation of these is really outside the scope of this paper.
+
+As well as standard collectable GCOobjets:
+
+-  Standard Lua has the concept of **fixed** objects. (E.g. the main thread). These won't be collected by the LGC, but they may refer to objects that aren't fixed, so the LGC still has to walk through an fixed objects.
+
+-  eLua added the the concept of **readonly** objects, which confusingly are a hybrid RW/RO implementation, where the underlying string resource is stored as a program constant in flash memory but the `TSstring` structure which points to this is still kept in RAM and can by GCed, except that in this case the LGC does not free the RO string constant itself.
+
+-  LFS introduces a third variant **flash** object for `LUA_TPROTO` and `LUA_TSTRING` types. Flash objects can only refer to other flash objects and are entirely located in the LFS area in flash memory.
+
+The LGC already processed the _fixed_ and _readonly_ object, albeit as special cases. In the case of _flash_ GCObjects, the `mark` flag is in read-only memory and therefore the LGC clearly can't use this as a RW flag in its mark and sweep processing. So the LGC skips any marking operations for flash objects. Likewise, where all other GCObjects are linked into one of a number of sweeplists using the object's `gclist` field. In the case of flash objects, the compiler presets the `mark` and `gclist` fields with the fixed and readonly mark bits set, and the list pointer to `NULL` during the compile process.
+
+As far as the LGC algorithm is concerned, encountering any _flash_ object in a sweep is a dead end, so that branch of the walk of the GCObject hierarchy can be terminated on encountering a flash object. This in practice all _flash_ objects are entirely removed from the LGC process, without compromising collection of RW resources.
+
+### General comments
+
+- **Reboot implementation**. Whilst the application initiated LFS reload might seem an overhead, it typically only adds a few seconds per reboot. We may also consider the future enhancement of the `esptool.py` to enable the inclusion of an LFS image into the unified application flash image.
+
+- **LGC reduction**. Since the cost of LGC is directly related to the size of the LGC sweep lists, moving RO resources into LFS memory removes them from the LGC scope and therefore reduces LGC runtime accordingly.
+
+- **Typical Usecase**.  The rebuilding of a store is an occasional step in the development cycle. (Say up to 10 times a day in a typical intensive development process). Modules and source files under development would typically be executed from SPIFFS in `.lua` format. The developer is free to reorder the `package.loaders` and load any SPIFFS files in preference to Flash ones. And if stable code is moved into Flash, then there is little to be gained in storing development Lua code in SPIFFS in `lc` compiled format.
+
+- **Flash caching coherency**.  The ESP chipset employs hardware enabled caching of the `ICACHE_FLASH` address space, and writing to the flash does not flush this cache.  However, in this restart model, the CPU is always restarted before any updates are read programmatically, so this (lack of) coherence isn't an issue.
+
+- **Failsafe reversion**. Since the entire image is precompiled, the chances of failure during reload are small. The loader uses the Flash NAND rules to write the flash header flag in two parts: one at start of the load and again at the end. If on reboot, the flag in on incostent state, then the LFS is cleared and disabled until the next reload.

docs/en/modules/node.md

@@ -150,7 +150,7 @@ flash ID (number)
 Returns the function reference for a function in the LFS (Lua Flash Store).
 
 #### Syntax
-`node.flashindex()`
+`node.flashindex(modulename)`
 
 #### Parameters
 `modulename`  The name of the module to be loaded.  If this is `nil` or invalid then an info list is returned
@@ -160,6 +160,10 @@ Returns the function reference for a function in the LFS (Lua Flash Store).
 -  If the LFS is loaded and the function is called with the name of a valid module in the LFS, then the function is returned in the same way the `load()` and the other Lua load functions do.
 -  Otherwise an extended info list is returned: the Unix time of the LFS build, the flash and mapped base addresses of the LFS and its current length, and an array of the valid module names in the LFS.
 
+#### Example
+
+The `node.flashindex()` is a low level API call that is normally wrapped using standard Lua code to present a simpler application API.  See the module `_init.lua` in the `lua_examples/lfs` directory for an example of how to do this.
+
 ## node.flashreload()
 
 Reload the LFS (Lua Flash Store) with the flash image provided. Flash images are generated on the host machine using the `luac.cross`commnad.

ld/nodemcu.ld

@@ -245,10 +245,13 @@ SECTIONS
     */libc.a:*.o(.text* .literal*)
     /* end libc functions */
 
+    /* Reserved areas, flash page aligned and last */
+    . = ALIGN(4096);
+    KEEP(*(.irom.reserved .irom.reserved.*))
+
     _irom0_text_end = ABSOLUTE(.);
     _flash_used_end = ABSOLUTE(.);
   } >irom0_0_seg :irom0_0_phdr =0xffffffff
-
 }
 
 /* get ROM code address */

lua_examples/lfs/_init.lua

@@ -0,0 +1,83 @@
+--
+-- File: _init.lua
+--[[
+
+  This is a template for the LFS equivalent of the SPIFFS init.lua.  
+
+  It is a good idea to such an _init.lua module to your LFS and do most of the LFS
+  module related initialisaion in this. This example uses standard Lua features to
+  simplify the LFS API.  
+
+  The first section adds a 'LFS' table to _G and uses the __index metamethod to
+  resolve functions in the LFS, so you can execute the main function of module
+  'fred' by executing LFS.fred(params), etc. It also implements some standard
+  readonly properties:
+
+  LFS._time    The Unix Timestamp when the luac.cross was executed.  This can be
+               used as a version identifier.
+
+  LFS._config  This returns a table of useful configuration parameters, hence
+                 print (("0x%6x"):format(LFS._config.lfs_base))
+               gives you the parameter to use in the luac.cross -a option.
+
+  LFS._list    This returns a table of the LFS modules, hence
+                 print(table.concat(LFS._list),'\n')
+               gives you a single column listing of all modules in the LFS.
+
+---------------------------------------------------------------------------------]]               
+
+local index = node.flashindex
+
+local lfs_t = {
+  __index = function(_, name)
+      local fn_ut, ba, ma, size, modules = index(name)
+      if not ba then 
+        return fn_ut
+      elseif name == '_time' then
+        return fn_ut
+      elseif name == '_config' then
+        local fs_ma, fs_size = file.fscfg()
+        return {lfs_base = ba, lfs_mapped = ma, lfs_size = size, 
+                fs_mapped = fs_ma, fs_size = fs_size}
+      elseif name == '_list' then
+        return modules
+      else
+        return nil
+      end
+    end,
+
+  __newindex = function(_, name, value)
+      error("LFS is readonly. Invalid write to LFS." .. name, 2)
+    end,
+ 
+  }
+
+local G=getfenv()
+G.LFS = setmetatable(lfs_t,lfs_t)
+
+--[[-------------------------------------------------------------------------------
+  The second section adds the LFS to the require searchlist, so that you can
+  require a Lua module 'jean' in the LFS by simply doing require "jean". However
+  note that this is at the search entry following the FS searcher, so if you also
+  have jean.lc or jean.lua in SPIFFS, then this SPIFFS version will get loaded into 
+  RAM instead of using. (Useful, for development).
+
+  Note that if you want LFS to take a higher priority than SPIFFS, the use the [2]
+  slot for loaders.  If you want to reverse these in your init.lua or interactively
+  for debugging, then use
+
+  do local pl = package.loaders; pl[2],pl[4] = pl[4],pl[2]; end
+---------------------------------------------------------------------------------]]
+
+package.loaders[4] = function(module) -- loader_flash
+  local fn, ba = index(module)
+  return ba and "Module not in LFS" or fn 
+end
+
+--[[-------------------------------------------------------------------------------
+  You can add any other initialisation here, for example a couple of the globals
+  are never used, so setting them to nil saves a couple of global entries
+---------------------------------------------------------------------------------]]
+ 
+G.module       = nil    -- disable Lua 5.0 style modules to save RAM
+package.seeall = nil 

lua_examples/lfs/dummy_strings.lua

@@ -0,0 +1,37 @@
+--
+-- File: LFS_dummy_strings.lua
+--[[
+  luac.cross -f generates a ROM string table which is part of the compiled LFS
+  image. This table includes all strings referenced in the loaded modules. 
+
+  If you want to preload other string constants, then one way to achieve this is
+  to include a dummy module in the LFS that references the strings that you want
+  to load. You never need to call this module; it's inclusion in the LFS image is
+  enough to add the strings to the ROM table. Your application can use any strings
+  in the ROM table without incuring any RAM or Lua Garbage Collector (LGC)
+  overhead. 
+
+  The local preload example is a useful starting point. However, if you call the
+  following code in your application during testing, then this will provide a 
+  listing of the current RAM string table.  
+
+do
+  local a=debug.getstrings'RAM'
+  for i =1, #a do a[i] = ('%q'):format(a[i]) end
+  print ('local preload='..table.concat(a,','))
+end
+
+  This will exclude any strings already in the ROM table, so the output is the list
+  of putative strings that you should consider adding to LFS ROM table.
+
+---------------------------------------------------------------------------------]]
+
+local preload = "?.lc;?.lua", "/\n;\n?\n!\n-", "@init.lua", "_G", "_LOADED",
+"_LOADLIB", "__add", "__call", "__concat", "__div", "__eq", "__gc", "__index",
+"__le", "__len", "__lt", "__mod", "__mode", "__mul", "__newindex", "__pow",
+"__sub", "__tostring", "__unm", "collectgarbage", "cpath", "debug", "file",
+"file.obj", "file.vol", "flash", "getstrings", "index", "ipairs", "list", "loaded",
+"loader", "loaders", "loadlib", "module", "net.tcpserver", "net.tcpsocket",
+"net.udpsocket", "newproxy", "package", "pairs", "path", "preload", "reload",
+"require", "seeall", "wdclr", "not enough memory", "sjson.decoder","sjson.encoder", 
+"tmr.timer"

lua_examples/lfs/lfs_fragments.lua

@@ -4,8 +4,12 @@
 -- image for the first time bare, that is without either LFS or SPIFFS preloaded
 -- then enter the following commands interactively through the UART:
 --
-local _,mapa,fa=node.flashindex(); return ('0x%x, 0x%x, 0x%x'):format(
+do
-  mapa,fa,file.fscfg())
+  local _,ma,fa=node.flashindex()
+  for n,v in pairs{LFS_MAPPED=ma, LFS_BASE=fa, SPIFFS_BASE=sa} do
+    print(('export %s=""0x%x"'):format(n, v)
+  end
+end
 --
 -- This will print out 3 hex constants: the absolute address used in the 
 -- 'luac.cross -a' options and the flash adresses of the LFS and SPIFFS.
@@ -33,37 +37,6 @@ $ESPTOOL --port $USB --baud 460800  write_flash -fm dio 0x100000 \
 # and now you are good to go
 ]]
 
------------------------------------------------------------------------------------
---
--- It is a good idea to add an _init.lua module to your LFS and do most of the 
--- LFS module related initialisaion in this.  This example uses standard Lua
--- features to simplify the LFS API.  
---
--- The first adds a 'LFS' table to _G and uses the __index metamethod to resolve
--- functions in the LFS, so you can execute the main function of module 'fred'
--- by doing LFS.fred(params)
---
--- The second adds the LFS to the require searchlist so that you can require a
--- Lua module 'jean' in the LFS by simply doing require "jean".  However not that
--- this is at the search entry following the FS searcher, so if you also have
--- jean.lc or jean.lua in SPIFFS, then this will get preferentially loaded,
--- albeit into RAM.  (Useful, for development).
---
-do
-  local index = node.flashindex 
-  local lfs_t = { __index = function(_, name)
-      local fn, ba = index(name)
-      if not ba then return fn end -- or return nil implied
-    end}
-  getfenv().LFS = setmetatable(lfs_t,lfs_t)
-
-  local function loader_flash(module)
-    local fn, ba = index(module)
-    return ba and "Module not in LFS" or fn 
-  end
-  package.loaders[3] = loader_flash
-
-end
 
 -----------------------------------------------------------------------------------
 --
@@ -73,8 +46,9 @@ end
 -- module in LFS.  Here is an example.  It's a good idea either to use a timer 
 -- delay or a GPIO pin during development, so that you as developer can break into 
 -- the boot sequence if there is a problem with the _init bootstrap that is causing
--- a panic loop.  Here is one example of how you might do this.  You have a second to
+-- a panic loop.  Here is one example of how you might do this.  You have a second
--- inject tmr.stop(0) into UART0.  Extend if your reactions can't meet this.
+-- to inject tmr.stop(0) into UART0.  Extend this dealy if your reactions can't 
+-- meet this.
 --
 -- You also want to do autoload the LFS, for example by adding the following:
 --
@@ -87,38 +61,3 @@ tmr.alarm(0, 1000, tmr.ALARM_SINGLE,
     local fi=node.flashindex; return pcall(fi and fi'_init')
   end)
 
------------------------------------------------------------------------------------
---
--- The debug.getstrings function can be used to get a listing of the RAM (or ROM
--- if LFS is loaded), as per the following example, so you can do this at the 
--- interactive prompt or call it as a debug function during a running application. 
---
-do
-  local a=debug.getstrings'RAM'
-  for i =1, #a do a[i] = ('%q'):format(a[i]) end
-  print ('local preload='..table.concat(a,','))
-end
-
------------------------------------------------------------------------------------
---
--- File: LFS_dummy_strings.lua
---
--- luac.cross -f will generate a ROM string table that includes all strings 
--- referenced in the loaded modules.  If you want to preload other string constants
--- hen the trick is to include a dummy module in the LFS.  You never need to call
--- this.  It's inclusion is enough to add the strings to the ROM table.  Once in
--- the ROM table, then you can use them in your application without incuring any 
--- RAM or Lua Garbage Collector (LGC) overhead.  Here is a useful starting point,
--- but you can add to this for your application.  
---
--- The trick is to build the LFS as normal then run the previous example from your
--- running application and append these lines to this file.
---
-local preload = "?.lc;?.lua", "@init.lua", "_G", "_LOADED", "_LOADLIB", "__add", 
-"__call", "__concat", "__div", "__eq", "__gc", "__index", "__le", "__len", "__lt", 
-"__mod", "__mode", "__mul", "__newindex", "__pow", "__sub", "__tostring", "__unm", 
-"collectgarbage", "cpath", "debug", "file", "file.obj", "file.vol", "flash", 
-"getstrings", "index", "ipairs", "list", "loaded", "loader", "loaders", "loadlib", 
-"module", "net.tcpserver", "net.tcpsocket", "net.udpsocket", "newproxy", "package", 
-"pairs", "path", "preload", "reload", "require", "seeall", "wdclr"
-

mkdocs.yml

@@ -24,13 +24,16 @@ pages:
     - Home: 'en/index.md'
     - Building the firmware: 'en/build.md'
     - Flashing the firmware: 'en/flash.md'
-    - Internal filesystem notes: 'en/spiffs.md'
-    - Filesystem on SD card: 'en/sdcard.md'
     - Uploading code: 'en/upload.md'
     - FAQs:
         - Lua Developer FAQ: 'en/lua-developer-faq.md'
         - Extension Developer FAQ: 'en/extn-developer-faq.md'
         - Hardware FAQ: 'en/hardware-faq.md'
+    - Whitepapers:
+        - Filesystem on SD card: 'en/sdcard.md'
+        - Internal filesystem notes: 'en/spiffs.md'
+        - Lua Compact Debug(LCD): 'en/lfs.md'
+        - Lua Flash Store(LFS): 'en/lcd.md'
     - Support: 'en/support.md'
     - Modules:
         - 'adc': 'en/modules/adc.md'

tools/Makefile

@@ -61,7 +61,7 @@ spiffsimg: spiffsimg/spiffsimg
 	@echo Built spiffsimg in spiffsimg/spiffsimg
 
 spiffsimg/spiffsimg: 
-	@$(MAKE) -C spiffsimg CC=$(HOSTCC)
+	@$(MAKE) -C spiffsimg 
 
 spiffsscript: remove-image LFSimage spiffsimg/spiffsimg
 	rm -f ./spiffsimg/spiffs.lst