Lua 5.1 / 5.3 alignment and document (#3193)

.gitignore

@@ -1,4 +1,6 @@
 .gdb_history
+app/lua/.std
+app/lua53/.std
 sdk/
 cache/
 .ccache/

app/coap/coap.h

@@ -7,7 +7,6 @@ extern "C" {
 
 #include <stdint.h>
 #include <stddef.h>
-#include "lualib.h"
 #include "lauxlib.h"
 
 #define MAXOPT 16

app/coap/endpoints.c

@@ -5,7 +5,6 @@
 
 #include "lua.h"
 #include "lauxlib.h"
-#include "lualib.h"
 
 #include "os_type.h"
 #include "user_interface.h"

app/http/httpclient.c

@@ -14,12 +14,12 @@
 
 #include "osapi.h"
 #include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
 #include "user_interface.h"
 #include "espconn.h"
 #include "mem.h"
-#include "limits.h"
 #include "httpclient.h"
-#include "stdlib.h"
 #include "pm/swtimer.h"
 
 #define REDIRECTION_FOLLOW_MAX 20

app/include/user_config.h

@@ -247,7 +247,8 @@
 
 #define LUA_TASK_PRIO             USER_TASK_PRIO_0
 #define LUA_PROCESS_LINE_SIG      2
-#define LUA_OPTIMIZE_DEBUG        2
+// LUAI_OPTIMIZE_DEBUG 0 = Keep all debug; 1 = keep line number info; 2 = remove all debug
+#define LUAI_OPTIMIZE_DEBUG       1
 #define READLINE_INTERVAL        80
 #define STRBUF_DEFAULT_INCREMENT  3
 #define LUA_USE_BUILTIN_DEBUG_MINIMAL // for debug.getregistry() and debug.traceback()

app/lua/lapi.c

@@ -152,6 +152,16 @@ LUA_API lua_State *lua_newthread (lua_State *L) {
 */
 
 
+/*
+** convert an acceptable stack index into an absolute index
+*/
+LUA_API int lua_absindex (lua_State *L, int idx) {
+  return (idx > 0 || idx <= LUA_REGISTRYINDEX)
+         ? idx
+         : cast_int(L->top - L->base) - 1 + idx;
+}
+
+
 LUA_API int lua_gettop (lua_State *L) {
   return cast_int(L->top - L->base);
 }
@@ -241,7 +251,7 @@ LUA_API void lua_pushvalue (lua_State *L, int idx) {
 
 LUA_API int lua_type (lua_State *L, int idx) {
   StkId o = index2adr(L, idx);
-  return (o == luaO_nilobject) ? LUA_TNONE : basettype(o);
+  return (o == luaO_nilobject) ? LUA_TNONE : ttnov(o);
 }
 
 
@@ -253,7 +263,7 @@ LUA_API int lua_fulltype (lua_State *L, int idx) {
 
 LUA_API const char *lua_typename (lua_State *L, int t) {
   UNUSED(L);
-  return (t == LUA_TNONE) ? "no value" : luaT_typenames[t];
+  return (t == LUA_TNONE || t >= LUA_NUMTAGS) ? "no value" : luaT_typenames[t];
 }
 
 
@@ -290,32 +300,25 @@ LUA_API int lua_rawequal (lua_State *L, int index1, int index2) {
 }
 
 
-LUA_API int lua_equal (lua_State *L, int index1, int index2) {
+LUA_API int lua_compare (lua_State *L, int index1, int index2, int op) {
   StkId o1, o2;
-  int i;
+  int i = 0;
   lua_lock(L);  /* may call tag method */
   o1 = index2adr(L, index1);
   o2 = index2adr(L, index2);
-  i = (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0 : equalobj(L, o1, o2);
+  if (o1 != luaO_nilobject && o2 != luaO_nilobject) {
+    switch (op) {
+      case LUA_OPEQ: i = luaV_equalval(L, o1, o2); break;
+      case LUA_OPLT: i = luaV_lessthan(L, o1, o2); break;
+      case LUA_OPLE: i = luaV_lessequal(L, o1, o2); break;
+      default: api_check(L, 0);
+    }
+  }
   lua_unlock(L);
   return i;
 }
 
 
-LUA_API int lua_lessthan (lua_State *L, int index1, int index2) {
-  StkId o1, o2;
-  int i;
-  lua_lock(L);  /* may call tag method */
-  o1 = index2adr(L, index1);
-  o2 = index2adr(L, index2);
-  i = (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0
-       : luaV_lessthan(L, o1, o2);
-  lua_unlock(L);
-  return i;
-}
-
-
-
 LUA_API lua_Number lua_tonumber (lua_State *L, int idx) {
   TValue n;
   const TValue *o = index2adr(L, idx);
@@ -366,7 +369,7 @@ LUA_API const char *lua_tolstring (lua_State *L, int idx, size_t *len) {
 
 LUA_API size_t lua_objlen (lua_State *L, int idx) {
   StkId o = index2adr(L, idx);
-  switch (basettype(o)) {
+  switch (ttnov(o)) {
     case LUA_TSTRING: return tsvalue(o)->len;
     case LUA_TUSERDATA: return uvalue(o)->len;
     case LUA_TTABLE: return luaH_getn(hvalue(o));
@@ -552,16 +555,17 @@ LUA_API int lua_pushthread (lua_State *L) {
 */
 
 
-LUA_API void lua_gettable (lua_State *L, int idx) {
+LUA_API int lua_gettable (lua_State *L, int idx) {
   StkId t;
   lua_lock(L);
   t = index2adr(L, idx);
   api_checkvalidindex(L, t);
   luaV_gettable(L, t, L->top - 1, L->top - 1);
   lua_unlock(L);
+  return ttnov(L->top - 1);
 }
 
-LUA_API void lua_getfield (lua_State *L, int idx, const char *k) {
+LUA_API int lua_getfield (lua_State *L, int idx, const char *k) {
   StkId t;
   TValue key;
   lua_lock(L);
@@ -573,20 +577,36 @@ LUA_API void lua_getfield (lua_State *L, int idx, const char *k) {
   luaV_gettable(L, t, &key, L->top);
   api_incr_top(L);
   lua_unlock(L);
+  return ttnov(L->top - 1);
 }
 
+LUA_API int lua_geti (lua_State *L, int idx, int n) {
+  StkId t;
+  TValue key;
+  lua_lock(L);
+  t = index2adr(L, idx);
+  api_checkvalidindex(L, t);
+  fixedstack(L);
+  setnvalue(&key, n);
+  unfixedstack(L);
+  luaV_gettable(L, t, &key, L->top);
+  api_incr_top(L);
+  lua_unlock(L);
+  return ttnov(L->top - 1);
+}
 
-LUA_API void lua_rawget (lua_State *L, int idx) {
+LUA_API int lua_rawget (lua_State *L, int idx) {
   StkId t;
   lua_lock(L);
   t = index2adr(L, idx);
   api_check(L, ttistable(t));
   setobj2s(L, L->top - 1, luaH_get(hvalue(t), L->top - 1));
   lua_unlock(L);
+  return ttnov(L->top - 1);
 }
 
 
-LUA_API void lua_rawgeti (lua_State *L, int idx, int n) {
+LUA_API int lua_rawgeti (lua_State *L, int idx, int n) {
   StkId o;
   lua_lock(L);
   o = index2adr(L, idx);
@@ -594,6 +614,21 @@ LUA_API void lua_rawgeti (lua_State *L, int idx, int n) {
   setobj2s(L, L->top, luaH_getnum(hvalue(o), n));
   api_incr_top(L);
   lua_unlock(L);
+  return ttnov(L->top - 1);
+}
+
+
+LUA_API int lua_rawgetp (lua_State *L, int idx, const void *p) {
+  StkId t;
+  TValue k;
+  lua_lock(L);
+  t = index2adr(L, idx);
+  api_check(L, ttistable(t));
+  setpvalue(&k, cast(void *, p));
+  setobj2s(L, L->top, luaH_get(hvalue(t), &k));
+  api_incr_top(L);
+  lua_unlock(L);
+  return ttnov(L->top - 1);
 }
 
 
@@ -612,7 +647,7 @@ LUA_API int lua_getmetatable (lua_State *L, int objindex) {
   int res;
   lua_lock(L);
   obj = index2adr(L, objindex);
-  switch (basettype(obj)) {
+  switch (ttnov(obj)) {
     case LUA_TTABLE:
       mt = hvalue(obj)->metatable;
       break;
@@ -620,7 +655,7 @@ LUA_API int lua_getmetatable (lua_State *L, int objindex) {
       mt = uvalue(obj)->metatable;
       break;
     default:
-      mt = G(L)->mt[basettype(obj)];
+      mt = G(L)->mt[ttnov(obj)];
       break;
   }
   if (mt == NULL)
@@ -720,6 +755,23 @@ LUA_API void lua_rawseti (lua_State *L, int idx, int n) {
 }
 
 
+LUA_API void lua_rawsetp (lua_State *L, int idx, const void *p) {
+  StkId o;
+  TValue k;
+  lua_lock(L);
+  api_checknelems(L, 1);
+  o = index2adr(L, idx);
+  api_check(L, ttistable(o));
+  fixedstack(L);
+  setpvalue(&k, cast(void *, p))
+  setobj2t(L, luaH_set(L, hvalue(o), &k), L->top-1);
+  unfixedstack(L);
+  luaC_barriert(L, hvalue(o), L->top-1);
+  L->top--;
+  lua_unlock(L);
+}
+
+
 LUA_API int lua_setmetatable (lua_State *L, int objindex) {
   TValue *obj;
   Table *mt;
@@ -750,7 +802,7 @@ LUA_API int lua_setmetatable (lua_State *L, int objindex) {
       break;
     }
     default: {
-      G(L)->mt[basettype(obj)] = mt;
+      G(L)->mt[ttnov(obj)] = mt;
       break;
     }
   }
@@ -901,7 +953,7 @@ LUA_API int lua_load (lua_State *L, lua_Reader reader, void *data,
 }
 
 
-LUA_API int lua_dumpEx (lua_State *L, lua_Writer writer, void *data, int stripping) {
+LUA_API int lua_dump (lua_State *L, lua_Writer writer, void *data, int stripping) {
   int status;
   TValue *o;
   lua_lock(L);
@@ -916,6 +968,30 @@ LUA_API int lua_dumpEx (lua_State *L, lua_Writer writer, void *data, int strippi
 }
 
 
+LUA_API int lua_stripdebug (lua_State *L, int stripping){
+  TValue *o = L->top - 1; 
+  Proto *p = NULL;
+  int res = -1;
+  lua_lock(L);
+  api_checknelems(L, 1);
+  if (isLfunction(o)) {
+    p = clvalue(o)->l.p;
+    if (p && !isLFSobject(p) && (unsigned) stripping < 3 ) {
+    // found a valid proto to strip
+      res = luaG_stripdebug(L, p, stripping, 1);
+    }
+  } else if (ttisnil(L->top - 1)) {
+    // get or set the default strip level
+    if ((unsigned) stripping < 3)
+      G(L)->stripdefault = stripping;
+    res = G(L)->stripdefault;
+  }
+  L->top--;
+  lua_unlock(L);
+  return res;
+}
+
+
 LUA_API int  lua_status (lua_State *L) {
   return L->status;
 }
@@ -1136,15 +1212,15 @@ LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n) {
   return name;
 }
 
-LUA_API void lua_setegcmode( lua_State *L, int mode, int limit) {
+LUA_API void lua_setegcmode ( lua_State *L, int mode, int limit) {
   G(L)->egcmode = mode;
   G(L)->memlimit = limit;
 }
 
-LUA_API void legc_set_mode(lua_State *L, int mode, int limit) {
+LUA_API void lua_getegcinfo (lua_State *L, int *totals) {
-   global_State *g = G(L);
+  if (totals) {
-
+    totals[0] = G(L)->totalbytes;
-   g->egcmode = mode;
+    totals[1] = G(L)->estimate;
-   g->memlimit = limit;
+  }
 }
 

app/lua/lauxlib.c

@@ -25,9 +25,9 @@
 */
 
 #define lauxlib_c
+#include "lauxlib.h"
 #define LUA_LIB
 
-#include "lauxlib.h"
 #include "lgc.h"
 #include "ldo.h"
 #include "lobject.h"

app/lua/lauxlib.h

@@ -9,6 +9,9 @@
 #define lauxlib_h
 
 #include "lua.h"
+#ifdef LUA_LIB
+#include "lnodemcu.h"
+#endif
 
 #include <stdio.h>
 
@@ -73,7 +76,7 @@ LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def,
 
 LUALIB_API int (luaL_ref) (lua_State *L, int t);
 LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref);
-#define luaL_unref2(l,t,r) luaL_unref(L, (t), (r)); r = LUA_NOREF
+#define luaL_unref2(l,t,r) do {luaL_unref(L, (t), (r)); r = LUA_NOREF;} while (0)
 LUALIB_API void (luaL_reref) (lua_State *L, int t, int *ref);
 
 LUALIB_API int (luaL_loadfile) (lua_State *L, const char *filename);
@@ -108,8 +111,8 @@ LUALIB_API void luaL_assertfail(const char *file, int line, const char *message)
 #define luaL_optint(L,n,d)	((int)luaL_optinteger(L, (n), (d)))
 #define luaL_checklong(L,n)	((long)luaL_checkinteger(L, (n)))
 #define luaL_optlong(L,n,d)	((long)luaL_optinteger(L, (n), (d)))
-#define luaL_checktable(L,n)	luaL_checktype(L, (n), LUA_TTABLE);
+#define luaL_checktable(L,n)	luaL_checktype(L, (n), LUA_TTABLE)
-#define luaL_checkfunction(L,n)	luaL_checktype(L, (n), LUA_TFUNCTION);
+#define luaL_checkfunction(L,n)	luaL_checktype(L, (n), LUA_TFUNCTION)
 
 #define luaL_typename(L,i)	lua_typename(L, lua_type(L,(i)))
 
@@ -162,8 +165,16 @@ LUALIB_API void (luaL_pushresult) (luaL_Buffer *B);
 
 /* }====================================================== */
 
-LUALIB_API int luaL_pcallx (lua_State *L, int narg, int nres);
+LUALIB_API int  (luaL_pushlfsmodules) (lua_State *L);
-LUALIB_API int luaL_posttask( lua_State* L, int prio );
+LUALIB_API int  (luaL_pushlfsmodule) (lua_State *L);
+LUALIB_API int  (luaL_pushlfsdts) (lua_State *L);
+
+LUALIB_API void (luaL_lfsreload) (lua_State *L);
+LUALIB_API int  (luaL_pcallx) (lua_State *L, int narg, int nres);
+LUALIB_API int  (luaL_posttask) ( lua_State* L, int prio );
+#define  LUA_TASK_LOW    0
+#define  LUA_TASK_MEDIUM 1
+#define  LUA_TASK_HIGH   2
 
 /* }====================================================== */
 

app/lua/lbaselib.c

@@ -10,7 +10,6 @@
 #define LUA_LIB
 
 #include "lua.h"
-#include "lnodemcu.h"
 
 #include <stdio.h>
 #include <string.h>

app/lua/lcode.c

@@ -780,8 +780,6 @@ void luaK_posfix (FuncState *fs, BinOpr op, expdesc *e1, expdesc *e2) {
 }
 
 
-#ifdef LUA_OPTIMIZE_DEBUG
-
 /*
  * Attempted to write to last (null terminator) byte of lineinfo, so need
  * to grow the lineinfo vector and extend the fill bytes
@@ -828,10 +826,8 @@ static void generateInfoDeltaLine(FuncState *fs, int line) {
   fs->lastlineOffset = p - fs->f->packedlineinfo - 1;
 #undef addDLbyte
 }
-#endif
 
 void luaK_fixline (FuncState *fs, int line) {
-#ifdef LUA_OPTIMIZE_DEBUG
   /* The fixup line can be the same as existing one and in this case there's nothing to do */
   if (line != fs->lastline) {
     /* first remove the current line reference */
@@ -862,9 +858,6 @@ void luaK_fixline (FuncState *fs, int line) {
     /* Then add the new line reference */
     generateInfoDeltaLine(fs, line);
   }
-#else
-   fs->f->lineinfo[fs->pc - 1] = line;
-#endif
 }
 
 
@@ -876,7 +869,6 @@ static int luaK_code (FuncState *fs, Instruction i, int line) {
                   MAX_INT, "code size overflow");
   f->code[fs->pc] = i;
   /* save corresponding line information */
-#ifdef LUA_OPTIMIZE_DEBUG
   /* note that frst time fs->lastline==0 through, so the else branch is taken */
   if (fs->pc == fs->lineinfoLastPC+1) {
     if (line == fs->lastline && f->packedlineinfo[fs->lastlineOffset] < INFO_MAX_LINECNT) {
@@ -890,11 +882,6 @@ static int luaK_code (FuncState *fs, Instruction i, int line) {
     luaK_fixline(fs,line);
   }
   fs->lineinfoLastPC = fs->pc;
-#else
-  luaM_growvector(fs->L, f->lineinfo, fs->pc, f->sizelineinfo, int,
-                  MAX_INT, "code size overflow");
-  f->lineinfo[fs->pc] = line;
-#endif
   return fs->pc++;
 }
 

app/lua/ldblib.c

@@ -15,9 +15,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lstring.h"
-#include "lflash.h"
-#include "lnodemcu.h"
 
 #include "user_modules.h"
 
@@ -27,34 +24,16 @@ static int db_getregistry (lua_State *L) {
 }
 
 static int db_getstrings (lua_State *L) {
-  size_t i,n=0;
+  static const char *const opts[] = {"RAM","ROM",NULL};
-  stringtable *tb;
+  int opt = luaL_checkoption(L, 1, "RAM", opts);
-  GCObject *o;
+  if (lua_pushstringsarray(L, opt)) {
-#ifndef LUA_CROSS_COMPILER
+    if(lua_getglobal(L, "table") == LUA_TTABLE) {
-  const char *opt = lua_tolstring (L, 1, &n);
-  if (n==3 && memcmp(opt, "ROM", 4) == 0) {
-    if (G(L)->ROstrt.hash == NULL)
-      return 0;
-    tb = &G(L)->ROstrt;
-  }
-  else
-#endif
-  tb = &G(L)->strt;
-  lua_settop(L, 0);
-  lua_createtable(L, tb->nuse, 0);          /* create table the same size as the strt */
-  for (i=0, n=1; i<tb->size; i++) {
-    for(o = tb->hash[i]; o; o=o->gch.next) {
-      TString *ts =cast(TString *, o);
-      lua_pushnil(L);
-      setsvalue2s(L, L->top-1, ts);
-      lua_rawseti(L, -2, n++);             /* enumerate the strt, adding elements */
-    }
-  }
-  lua_getfield(L, LUA_GLOBALSINDEX, "table");
       lua_getfield(L, -1, "sort");    /* look up table.sort function */
       lua_replace(L, -2);             /* dump the table table */
       lua_pushvalue(L, -2);           /* duplicate the strt_copy ref */
       lua_call(L, 1, 0);              /* table.sort(strt_copy) */
+    }
+  }
   return 1;
 }
 

app/lua/ldebug.c

@@ -183,7 +183,6 @@ static void info_tailcall (lua_Debug *ar) {
 #  define INFO_MAX_LINECNT  126
 
     Table *t = luaH_new(L, 0, 0);
-#ifdef LUA_OPTIMIZE_DEBUG
     int line = 0;
     unsigned char *p = f->l.p->packedlineinfo;
     if (p) {
@@ -203,18 +202,11 @@ static void info_tailcall (lua_Debug *ar) {
        setbvalue(luaH_setnum(L, t, line), 1);
       }
     }
-#else
-     int *lineinfo = f->l.p->lineinfo;
-     int i;
-     for (i=0; i<f->l.p->sizelineinfo; i++)
-       setbvalue(luaH_setnum(L, t, lineinfo[i]), 1);
-#endif
     sethvalue(L, L->top, t);
    }
    incr_top(L);
  }
 
-#ifdef LUA_OPTIMIZE_DEBUG
 /*
  * This may seem expensive but this is only accessed frequently in traceexec
  * and the while loop will be executed roughly half the number of non-blank
@@ -249,18 +241,17 @@ int luaG_getline (const Proto *f, int pc) {
 
 static int stripdebug (lua_State *L, Proto *f, int level) {
   int len = 0, sizepackedlineinfo;
-  TString* dummy;
   switch (level) {
-    case 3:
+    case 2:
+      if (f->packedlineinfo) {
         sizepackedlineinfo = strlen(cast(char *, f->packedlineinfo))+1;
         f->packedlineinfo = luaM_freearray(L, f->packedlineinfo, sizepackedlineinfo, unsigned char);
         len += sizepackedlineinfo;
-    case 2:
+      }
-      len += f->sizelocvars * (sizeof(struct LocVar) + sizeof(dummy->tsv) + sizeof(struct LocVar *));
+    case 1:
       f->locvars = luaM_freearray(L, f->locvars, f->sizelocvars, struct LocVar);
       f->upvalues = luaM_freearray(L, f->upvalues, f->sizeupvalues, TString *);
-      len += f->sizelocvars * (sizeof(struct LocVar) + sizeof(dummy->tsv) + sizeof(struct LocVar *)) +
+      len += f->sizelocvars*sizeof(struct LocVar) + f->sizeupvalues*sizeof(TString *);
-             f->sizeupvalues * (sizeof(dummy->tsv) + sizeof(TString *));
       f->sizelocvars = 0;
       f->sizeupvalues = 0;
   }
@@ -276,7 +267,6 @@ LUA_API int luaG_stripdebug (lua_State *L, Proto *f, int level, int recv){
   len += stripdebug (L, f, level);
   return len;
 }
-#endif
 
 
 static int auxgetinfo (lua_State *L, const char *what, lua_Debug *ar,
@@ -379,9 +369,6 @@ static int precheck (const Proto *pt) {
   check(!(pt->is_vararg & VARARG_NEEDSARG) ||
               (pt->is_vararg & VARARG_HASARG));
   check(pt->sizeupvalues <= pt->nups);
-#ifndef LUA_OPTIMIZE_DEBUG
-  check(pt->sizelineinfo == pt->sizecode || pt->sizelineinfo == 0);
-#endif
   check(pt->sizecode > 0 && GET_OPCODE(pt->code[pt->sizecode-1]) == OP_RETURN);
   return 1;
 }
@@ -668,7 +655,7 @@ static int isinstack (CallInfo *ci, const TValue *o) {
 
 void luaG_typeerror (lua_State *L, const TValue *o, const char *op) {
   const char *name = NULL;
-  const char *t = luaT_typenames[basettype(o)];
+  const char *t = luaT_typenames[ttnov(o)];
   const char *kind = (isinstack(L->ci, o)) ?
                          getobjname(L, L->ci, cast_int(o - L->base), &name) :
                          NULL;
@@ -696,8 +683,8 @@ void luaG_aritherror (lua_State *L, const TValue *p1, const TValue *p2) {
 
 
 int luaG_ordererror (lua_State *L, const TValue *p1, const TValue *p2) {
-  const char *t1 = luaT_typenames[basettype(p1)];
+  const char *t1 = luaT_typenames[ttnov(p1)];
-  const char *t2 = luaT_typenames[basettype(p2)];
+  const char *t2 = luaT_typenames[ttnov(p2)];
   if (t1[2] == t2[2])
     luaG_runerror(L, "attempt to compare two %s values", t1);
   else

app/lua/ldebug.h

@@ -13,7 +13,6 @@
 
 #define pcRel(pc, p)	(cast(int, (pc) - (p)->code) - 1)
 
-#ifdef LUA_OPTIMIZE_DEBUG
 #  include "lvm.h"
 #  define getline(f,pc) (((f)->packedlineinfo) ? luaG_getline((f), pc) : 0)
 #  define INFO_FILL_BYTE   0x7F
@@ -23,9 +22,6 @@
 #  define INFO_DELTA_7BITS 0x7F
 #  define INFO_MAX_LINECNT  126
 #  define lineInfoTop(fs) ((fs)->f->packedlineinfo + (fs)->lastlineOffset)
-#else
-#  define getline(f,pc) (((f)->lineinfo) ? (f)->lineinfo[pc] : 0)
-#endif
 
 #define resethookcount(L)	(L->hookcount = L->basehookcount)
 
@@ -41,9 +37,7 @@ LUAI_FUNC void luaG_runerror (lua_State *L, const char *fmt, ...);
 LUAI_FUNC void luaG_errormsg (lua_State *L);
 LUAI_FUNC int luaG_checkcode (const Proto *pt);
 LUAI_FUNC int luaG_checkopenop (Instruction i);
-#ifdef LUA_OPTIMIZE_DEBUG
 LUAI_FUNC int luaG_getline (const Proto *f, int pc);
 LUAI_FUNC int luaG_stripdebug (lua_State *L, Proto *f, int level, int recv);
-#endif
 
 #endif

app/lua/ldump.c

@@ -228,7 +228,6 @@ static void DumpDebug(const Proto* f, DumpState* D)
 {
  int i,n;
 
-#ifdef LUA_OPTIMIZE_DEBUG
  n = (D->strip || f->packedlineinfo == NULL) ? 0: strlen(cast(char *,f->packedlineinfo))+1;
  DumpInt(n,D);
  Align4(D);
@@ -236,15 +235,6 @@ static void DumpDebug(const Proto* f, DumpState* D)
  {
   DumpBlock(f->packedlineinfo, n, D);
  }
-#else
- n= (D->strip) ? 0 : f->sizelineinfo;
- DumpInt(n,D);
- Align4(D);
- for (i=0; i<n; i++)
- {
-  DumpInt(f->lineinfo[i],D);
- }
- #endif
 
  n= (D->strip) ? 0 : f->sizelocvars;
  DumpInt(n,D);

app/lua/lflash.c

@@ -194,21 +194,20 @@ static int loadLFS (lua_State *L);
 static int loadLFSgc (lua_State *L);
 static void procFirstPass (void);
 
-/* lua_lfsreload() and lua_lfsindex() are exported via lua.h */
+/* luaL_lfsreload() is exported via lauxlib.h */
 
 /*
  * Library function called by node.flashreload(filename).
  */
-LUALIB_API int lua_lfsreload (lua_State *L) {
+LUALIB_API void luaL_lfsreload (lua_State *L) {
   const char *fn = lua_tostring(L, 1), *msg = "";
   int status;
 
   if (G(L)->LFSsize == 0) {
     lua_pushstring(L, "No LFS partition allocated");
-    return 1;
+    return;
   }
 
-
  /*
   * Do a protected call of loadLFS.
   *
@@ -237,7 +236,7 @@ LUALIB_API int lua_lfsreload (lua_State *L) {
     lua_cpcall(L, &loadLFSgc, NULL);
     lua_settop(L, 0);
     lua_pushstring(L, msg);
-    return 1;
+    return;
   }
 
   if (status == 0) {
@@ -257,60 +256,21 @@ LUALIB_API int lua_lfsreload (lua_State *L) {
   NODE_ERR(msg);
 
   while (1) {}  // Force WDT as the ROM software_reset() doesn't seem to work
-  return 0;
 }
 
 
-/*
+LUA_API void lua_getlfsconfig (lua_State *L, int *config) {
- * If the arg is a valid LFS module name then return the LClosure
+  if (!config)
- * pointing to it. Otherwise return:
+    return;
- *  -  The Unix time that the LFS was built
+  config[0] = (int) flashAddr;                   /* LFS region mapped address */
- *  -  The base address and length of the LFS
+  config[1] = flashAddrPhys;                 /* LFS region base flash address */
- *  -  An array of the module names in the LFS
+  config[2] = G(L)->LFSsize;                        /* LFS region actual size */
- */
+  config[3] = (G(L)->ROstrt.hash) ? cast(FlashHeader *, flashAddr)->flash_size : 0;
-LUAI_FUNC int lua_lfsindex (lua_State *L) {
+                                                           /* LFS region used */
-  int n = lua_gettop(L);
+  config[4] = 0;                                       /* Not used in Lua 5.1 */
-
-  /* Return nil + the LFS base address if the LFS size > 0 and it isn't loaded */
-  if (!(G(L)->ROpvmain)) {
-    lua_settop(L, 0);
-    lua_pushnil(L);
-    if (G(L)->LFSsize) {
-      lua_pushinteger(L, (lua_Integer) flashAddr);
-      lua_pushinteger(L, flashAddrPhys);
-      lua_pushinteger(L, G(L)->LFSsize);
-      return 4;
-    } else {
-      return 1;
-    }
-  }
-
-  /* Push the LClosure of the LFS index function */
-  Closure *cl = luaF_newLclosure(L, 0, hvalue(gt(L)));
-  cl->l.p = G(L)->ROpvmain;
-  lua_settop(L, n+1);
-  setclvalue(L, L->top-1, cl);
-
-  /* Move it infront of the arguments and call the index function */
-  lua_insert(L, 1);
-  lua_call(L, n, LUA_MULTRET);
-
-  /* Return it if the response if a single value (the function) */
-  if (lua_gettop(L) == 1)
-    return 1;
-
-  lua_assert(lua_gettop(L) == 2);
-
-  /* Otherwise add the base address of the LFS, and its size bewteen the */
-  /* Unix time and the module list, then return all 4 params. */
-  lua_pushinteger(L, (lua_Integer) flashAddr);
-  lua_insert(L, 2);
-  lua_pushinteger(L, flashAddrPhys);
-  lua_insert(L, 3);
-  lua_pushinteger(L, cast(FlashHeader *, flashAddr)->flash_size);
-  lua_insert(L, 4);
-  return 5;
 }
+
+
 /* =====================================================================================
  * The following routines use my uzlib which was based on pfalcon's inflate and
  * deflate routines.  The standard NodeMCU make also makes two host tools uz_zip
@@ -483,7 +443,7 @@ void procSecondPass (void) {
 }
 
 /*
- * loadLFS)() is protected called from luaN_reload_reboot so that it can recover
+ * loadLFS)() is protected called from luaL_lfsreload() so that it can recover
  * from out of memory and other thrown errors.  loadLFSgc() GCs any resources.
  */
 static int loadLFS (lua_State *L) {

app/lua/lflash.h

@@ -43,8 +43,5 @@ typedef struct {
 } FlashHeader;
 
 LUAI_FUNC void luaN_init (lua_State *L);
-LUAI_FUNC int  luaN_flashSetup (lua_State *L);
-LUAI_FUNC int  luaN_reload_reboot (lua_State *L);
-LUAI_FUNC int  luaN_index (lua_State *L);
 #endif
 

app/lua/lfunc.c

@@ -125,12 +125,7 @@ Proto *luaF_newproto (lua_State *L) {
   f->numparams = 0;
   f->is_vararg = 0;
   f->maxstacksize = 0;
-#ifdef LUA_OPTIMIZE_DEBUG
   f->packedlineinfo = NULL;
-#else
-  f->sizelineinfo = 0;
-  f->lineinfo = NULL;
-#endif
   f->sizelocvars = 0;
   f->locvars = NULL;
   f->linedefined = 0;
@@ -146,13 +141,9 @@ void luaF_freeproto (lua_State *L, Proto *f) {
   luaM_freearray(L, f->locvars, f->sizelocvars, struct LocVar);
   luaM_freearray(L, f->upvalues, f->sizeupvalues, TString *);
   luaM_freearray(L, f->code, f->sizecode, Instruction);
-#ifdef LUA_OPTIMIZE_DEBUG
   if (f->packedlineinfo) {
     luaM_freearray(L, f->packedlineinfo, strlen(cast(char *, f->packedlineinfo))+1, unsigned char);
   }
-#else
-  luaM_freearray(L, f->lineinfo, f->sizelineinfo, int);
-#endif
   luaM_free(L, f);
 }
 

app/lua/lgc.c

@@ -327,11 +327,7 @@ static l_mem propagatemark (global_State *g) {
                              sizeof(LocVar) * p->sizelocvars +
                              sizeof(TString *) * p->sizeupvalues +
                              sizeof(Instruction) * p->sizecode +
-#ifdef LUA_OPTIMIZE_DEBUG
                                (p->packedlineinfo ?  strlen(cast(char *, p->packedlineinfo))+1 : 0);
-#else
-                               sizeof(int) * p->sizelineinfo;
-#endif
     }
     default: lua_assert(0); return 0;
   }
@@ -515,7 +511,7 @@ void luaC_freeall (lua_State *L) {
 
 static void markmt (global_State *g) {
   int i;
-  for (i=0; i<NUM_TAGS; i++)
+  for (i=0; i<LUA_NUMTAGS; i++)
     if (g->mt[i] && isrwtable(g->mt[i])) markobject(g, g->mt[i]);
 }
 

app/lua/linit.c

@@ -61,7 +61,7 @@ extern LROT_TABLE(math);
 #if defined(LUA_CROSS_COMPILER)
 extern LROT_TABLE(base_func);
 LROT_BEGIN(rotables_meta, NULL, LROT_MASK_INDEX)
-  LROT_TABENTRY( _index, base_func)
+  LROT_TABENTRY( __index, base_func)
 LROT_END(rotables_meta, NULL, LROT_MASK_INDEX)
 
 extern LROT_TABLE(oslib);

app/lua/lmathlib.c

@@ -14,7 +14,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 #undef PI
 #define PI (3.14159265358979323846)

app/lua/lnodemcu.c

@@ -19,8 +19,10 @@
 #include "ltable.h"
 #include "ltm.h"
 #include "lnodemcu.h"
-#include "platform.h"
 
+//== NodeMCU lauxlib.h API extensions ========================================//
+#ifdef LUA_USE_ESP
+#include "platform.h"
 /*
 ** Error Reporting Task.  We can't pass a string parameter to the error reporter
 ** directly through the task interface the call is wrapped in a C closure with
@@ -99,7 +101,6 @@ static void do_task (platform_task_param_t task_fn_ref, uint8_t prio) {
 /*
 ** Schedule a Lua function for task execution
 */
-#include "lstate.h" /*DEBUG*/
 LUALIB_API int luaL_posttask( lua_State* L, int prio ) {          // [-1, +0, -]
   if (!task_handle)
     task_handle = platform_task_get_id(do_task);
@@ -115,6 +116,98 @@ LUALIB_API int luaL_posttask( lua_State* L, int prio ) {            // [-1, +0,
   }
   return task_fn_ref;
 }
+#else
+LUALIB_API int luaL_posttask( lua_State* L, int prio ) { 
+  return 0;
+} /* Dummy stub on host */
+#endif
+
+#ifdef LUA_USE_ESP
+/*
+ * Return an LFS function
+ */
+LUALIB_API int luaL_pushlfsmodule (lua_State *L) {
+  if (lua_pushlfsindex(L) == LUA_TNIL) {
+    lua_remove(L,-2);  /* dump the name to balance the stack */
+    return 0;          /* return nil if LFS not loaded */
+  }
+  lua_insert(L, -2);
+  lua_call(L, 1, 1);
+  if (!lua_isfunction(L, -1)) {
+    lua_pop(L, 1);
+    lua_pushnil(L);  /* replace DTS by nil */
+  }
+  return 1;
+}
+
+/*
+ * Return an array of functions in LFS
+ */
+LUALIB_API int luaL_pushlfsmodules (lua_State *L) {
+  if (lua_pushlfsindex(L) == LUA_TNIL)
+    return 0;              /* return nil if LFS not loaded */
+  lua_call(L, 0, 2);
+  lua_remove(L, -2);     /* remove DTS leaving array */
+  return 1;
+
+}
+
+/*
+ * Return the Unix timestamp of the LFS image creation
+ */
+LUALIB_API int luaL_pushlfsdts (lua_State *L) {
+  if (lua_pushlfsindex(L) == LUA_TNIL)
+    return 0;              /* return nil if LFS not loaded */
+  lua_call(L, 0, 1);
+  return 1;
+}
+#endif
+
+//== NodeMCU lua.h API extensions ============================================//
+
+LUA_API int lua_freeheap (void) {
+#ifdef LUA_USE_HOST
+  return MAX_INT;
+#else
+  return (int) platform_freeheap();
+#endif
+}
+
+#define api_incr_top(L)   {api_check(L, L->top < L->ci->top); L->top++;}
+LUA_API int lua_pushstringsarray(lua_State *L, int opt) {
+  stringtable *tb = NULL;
+  int i, j;
+
+  lua_lock(L);
+  if (opt == 0)
+    tb = &G(L)->strt;
+#ifdef LUA_USE_ESP
+  else if (opt == 1 && G(L)->ROstrt.hash)
+    tb = &G(L)->ROstrt;
+#endif
+  if (tb == NULL) {
+    setnilvalue(L->top);
+    api_incr_top(L);
+    lua_unlock(L);
+    return 0;
+  }
+
+  Table *t = luaH_new(L, tb->nuse, 0);
+  sethvalue(L, L->top, t);
+  api_incr_top(L);
+  luaC_checkGC(L);
+  lua_unlock(L);
+
+  for (i = 0, j = 1; i < tb->size; i++) {
+    GCObject *o;
+    for(o = tb->hash[i]; o; o = o->gch.next) {
+      TValue v;
+      setsvalue(L, &v, o); 
+      setobj2s(L, luaH_setnum(L, hvalue(L->top-1), j++), &v);
+    }
+  }
+  return 1;
+}
 
 LUA_API void lua_createrotable (lua_State *L, ROTable *t, const ROTable_entry *e, ROTable *mt) {
   int i, j;
@@ -142,3 +235,22 @@ LUA_API void lua_createrotable (lua_State *L, ROTable *t, const ROTable_entry *e
   t->entry     = cast(ROTable_entry *, e);
 }
 
+#ifdef LUA_USE_ESP
+#include "lfunc.h"
+
+/* Push the LClosure of the LFS index function */
+LUA_API int lua_pushlfsindex (lua_State *L) {
+  lua_lock(L);
+  Proto *p = G(L)->ROpvmain;
+  if (p) {
+    Closure *cl = luaF_newLclosure(L, 0, hvalue(gt(L)));
+    cl->l.p = p;
+    setclvalue(L, L->top, cl);
+  } else {
+    setnilvalue(L->top);
+  }
+  api_incr_top(L);
+  lua_unlock(L);
+  return p ? LUA_TFUNCTION : LUA_TNIL;
+}
+#endif

app/lua/loadlib.c

@@ -23,7 +23,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 /* prefix for open functions in C libraries */
 #define LUA_POF		"luaopen_"
@@ -474,7 +473,7 @@ static int ll_require (lua_State *L) {
   }
   /* Is this a readonly table? */
   lua_getfield(L, LUA_GLOBALSINDEX, name);
-  if(lua_isrotable(L,-1)) {
+  if(lua_istable(L,-1)) {
     return 1;
   } else {
     lua_pop(L, 1);
@@ -567,7 +566,7 @@ static int ll_module (lua_State *L) {
   const char *modname = luaL_checkstring(L, 1);
   /* Is this a readonly table? */
   lua_getfield(L, LUA_GLOBALSINDEX, modname);
-  if(lua_isrotable(L,-1)) {
+  if(lua_istable(L,-1)) {
     return 0;
   } else {
     lua_pop(L, 1);

app/lua/lobject.h

@@ -17,7 +17,7 @@
 /* tags for values visible from Lua */
 #define LAST_TAG	LUA_TTHREAD
 
-#define NUM_TAGS	(LAST_TAG+1)
+#define LUA_NUMTAGS	(LAST_TAG+1)
 
 #define READONLYMASK    (1<<7)      /* denormalised bitmask for READONLYBIT and */
 #define LFSMASK         (1<<6)      /* LFSBIT to avoid include proliferation */
@@ -111,7 +111,7 @@ typedef struct lua_TValue {
 #define ttisnil(o)		(ttype(o) == LUA_TNIL)
 #define ttisnumber(o)		(ttype(o) == LUA_TNUMBER)
 #define ttisstring(o)		(ttype(o) == LUA_TSTRING)
-#define ttistable(o)		(basettype(o) == LUA_TTABLE)
+#define ttistable(o)		(ttnov(o) == LUA_TTABLE)
 #define ttisrwtable(o)		(type(o) == LUA_TTABLE)
 #define ttisrotable(o)		(ttype(o) & LUA_TISROTABLE)
 #define ttisboolean(o)		(ttype(o) == LUA_TBOOLEAN)
@@ -120,12 +120,12 @@ typedef struct lua_TValue {
 #define ttislightuserdata(o)	(ttype(o) == LUA_TLIGHTUSERDATA)
 #define ttislightfunction(o)	(ttype(o) == LUA_TLIGHTFUNCTION)
 #define ttisclfunction(o)	(ttype(o) == LUA_TFUNCTION)
-#define ttisfunction(o)		(basettype(o) == LUA_TFUNCTION)
+#define ttisfunction(o)		(ttnov(o) == LUA_TFUNCTION)
 
 /* Macros to access values */
 
 #define ttype(o)	((void) (o)->value, (o)->tt)
-#define basettype(o)	((void) (o)->value, ((o)->tt & LUA_TMASK))
+#define ttnov(o)	((void) (o)->value, ((o)->tt & LUA_TMASK))
 #define gcvalue(o)	check_exp(iscollectable(o), (o)->value.gc)
 #define pvalue(o)	check_exp(ttislightuserdata(o), (o)->value.p)
 #define fvalue(o)	check_exp(ttislightfunction(o), (o)->value.p)
@@ -277,20 +277,13 @@ typedef struct Proto {
   TValue *k;  /* constants used by the function */
   Instruction *code;
   struct Proto **p;  /* functions defined inside the function */
-#ifdef LUA_OPTIMIZE_DEBUG
   unsigned char *packedlineinfo;
-#else
-  int *lineinfo;  /* map from opcodes to source lines */
-#endif
   struct LocVar *locvars;  /* information about local variables */
   TString **upvalues;  /* upvalue names */
   TString  *source;
   int sizeupvalues;
   int sizek;  /* size of `k' */
   int sizecode;
-#ifndef LUA_OPTIMIZE_DEBUG
-  int sizelineinfo;
-#endif
   int sizep;  /* size of `p' */
   int sizelocvars;
   int linedefined;

app/lua/lparser.c

@@ -343,12 +343,10 @@ static void open_func (LexState *ls, FuncState *fs) {
   fs->bl = NULL;
   f->source = ls->source;
   f->maxstacksize = 2;  /* registers 0/1 are always valid */
-#ifdef LUA_OPTIMIZE_DEBUG
   fs->packedlineinfoSize = 0;
   fs->lastline = 0;
   fs->lastlineOffset = 0;
   fs->lineinfoLastPC = -1;
-#endif
   fs->h = luaH_new(L, 0, 0);
   /* anchor table of constants and prototype (to avoid being collected) */
   sethvalue2s(L, L->top, fs->h);
@@ -366,15 +364,9 @@ static void close_func (LexState *ls) {
   luaK_ret(fs, 0, 0);  /* final return */
   luaM_reallocvector(L, f->code, f->sizecode, fs->pc, Instruction);
   f->sizecode = fs->pc;
-#ifdef LUA_OPTIMIZE_DEBUG
   f->packedlineinfo[fs->lastlineOffset+1]=0;
   luaM_reallocvector(L, f->packedlineinfo, fs->packedlineinfoSize,
                      fs->lastlineOffset+2, unsigned char);
-#else
-  luaM_reallocvector(L, f->lineinfo, f->sizelineinfo, fs->pc, int);
-  f->sizelineinfo = fs->pc;
-#endif
-
   luaM_reallocvector(L, f->k, f->sizek, fs->nk, TValue);
   f->sizek = fs->nk;
   luaM_reallocvector(L, f->p, f->sizep, fs->np, Proto *);
@@ -391,20 +383,11 @@ static void close_func (LexState *ls) {
   L->top -= 2;  /* remove table and prototype from the stack */
 }
 
-#ifdef LUA_OPTIMIZE_DEBUG
 static void compile_stripdebug(lua_State *L, Proto *f) {
-  int level;
+  int level =  G(L)->stripdefault;
-  lua_pushlightuserdata(L, &luaG_stripdebug );
+  if (level > 0)
-  lua_gettable(L, LUA_REGISTRYINDEX);
+    luaG_stripdebug(L, f, level, 1);
-  level = lua_isnil(L, -1) ? LUA_OPTIMIZE_DEBUG : lua_tointeger(L, -1);
-  lua_pop(L, 1);
-
-  if (level > 1) {
-    int len = luaG_stripdebug(L, f, level, 1);
-    UNUSED(len);
-  }
 }
-#endif
 
 
 Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, const char *name) {
@@ -421,9 +404,7 @@ Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, const char *name) {
   chunk(&lexstate);
   check(&lexstate, TK_EOS);
   close_func(&lexstate);
-#ifdef LUA_OPTIMIZE_DEBUG
   compile_stripdebug(L, funcstate.f);
-#endif
   L->top--; /* remove 'name' from stack */
   lua_assert(funcstate.prev == NULL);
   lua_assert(funcstate.f->nups == 0);

app/lua/lparser.h

@@ -72,12 +72,10 @@ typedef struct FuncState {
   lu_byte nactvar;  /* number of active local variables */
   upvaldesc upvalues[LUAI_MAXUPVALUES];  /* upvalues */
   unsigned short actvar[LUAI_MAXVARS];  /* declared-variable stack */
-#ifdef LUA_OPTIMIZE_DEBUG
   int packedlineinfoSize; /* only used during compilation for line info */
   int lastline;           /* ditto */
   int lastlineOffset;     /* ditto */
   int lineinfoLastPC;     /* ditto */
-#endif
 } FuncState;
 
 

app/lua/lstate.c

@@ -184,6 +184,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
   g->memlimit = 0;
   g->gcpause = LUAI_GCPAUSE;
   g->gcstepmul = LUAI_GCMUL;
+  g->stripdefault = LUAI_OPTIMIZE_DEBUG;
   g->gcdept = 0;
 #ifdef EGC_INITIAL_MODE
   g->egcmode = EGC_INITIAL_MODE;
@@ -203,7 +204,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
   g->LFSsize        = 0;
   g->error_reporter = 0;
 #endif
-  for (i=0; i<NUM_TAGS; i++) g->mt[i] = NULL;
+  for (i=0; i<LUA_NUMTAGS; i++) g->mt[i] = NULL;
   if (luaD_rawrunprotected(L, f_luaopen, NULL) != 0) {
     /* memory allocation error: free partial state */
     close_state(L);

app/lua/lstate.h

@@ -87,12 +87,13 @@ typedef struct global_State {
   lu_mem gcdept;  /* how much GC is `behind schedule' */
   int gcpause;  /* size of pause between successive GCs */
   int gcstepmul;  /* GC `granularity' */
+  int stripdefault;  /* default stripping level for compilation */
   int egcmode;    /* emergency garbage collection operation mode */
   lua_CFunction panic;  /* to be called in unprotected errors */
   TValue l_registry;
   struct lua_State *mainthread;
   UpVal uvhead;  /* head of double-linked list of all open upvalues */
-  struct Table *mt[NUM_TAGS];  /* metatables for basic types */
+  struct Table *mt[LUA_NUMTAGS];  /* metatables for basic types */
   TString *tmname[TM_N];  /* array with tag-method names */
 #ifndef LUA_CROSS_COMPILER
   stringtable ROstrt;  /* Flash-based hash table for RO strings */

app/lua/lstrlib.c

@@ -14,7 +14,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 /* macro to `unsign' a character */
 #define uchar(c)        ((unsigned char)(c))
@@ -141,17 +140,25 @@ static int writer (lua_State *L, const void* b, size_t size, void* B) {
 
 static int str_dump (lua_State *L) {
   luaL_Buffer b;
+  int strip, tstrip = lua_type(L, 2);
+  if (tstrip == LUA_TBOOLEAN) {
+    strip = lua_toboolean(L, 2) ? 2 : 0;
+  } else if (tstrip == LUA_TNONE || tstrip == LUA_TNIL) {
+    strip = -1;  /* This tells lua_dump to use the global strip default */
+  } else { 
+    strip = lua_tointeger(L, 2);
+    luaL_argcheck(L, (unsigned)(strip) < 3, 2, "strip out of range");
+  }
   luaL_checktype(L, 1, LUA_TFUNCTION);
   lua_settop(L, 1);
   luaL_buffinit(L,&b);
-  if (lua_dump(L, writer, &b) != 0)
+  if (lua_dump(L, writer, &b, strip) != 0)
-    luaL_error(L, "unable to dump given function");
+    return luaL_error(L, "unable to dump given function");
   luaL_pushresult(&b);
   return 1;
 }
 
 
-
 /*
 ** {======================================================
 ** PATTERN MATCHING

app/lua/ltable.h

@@ -20,21 +20,16 @@
 #define isrwtable(t) (gettt(t)==LUA_TTABLE)
 
 LUAI_FUNC const TValue *luaH_getnum (Table *t, int key);
-LUAI_FUNC const TValue *luaH_getnum_ro (void *t, int key);
 LUAI_FUNC TValue *luaH_setnum (lua_State *L, Table *t, int key);
 LUAI_FUNC const TValue *luaH_getstr (Table *t, TString *key);
-LUAI_FUNC const TValue *luaH_getstr_ro (void *t, TString *key);
 LUAI_FUNC TValue *luaH_setstr (lua_State *L, Table *t, TString *key);
 LUAI_FUNC const TValue *luaH_get (Table *t, const TValue *key);
-LUAI_FUNC const TValue *luaH_get_ro (void *t, const TValue *key);
 LUAI_FUNC TValue *luaH_set (lua_State *L, Table *t, const TValue *key);
 LUAI_FUNC Table *luaH_new (lua_State *L, int narray, int lnhash);
 LUAI_FUNC void luaH_resizearray (lua_State *L, Table *t, int nasize);
 LUAI_FUNC void luaH_free (lua_State *L, Table *t);
 LUAI_FUNC int luaH_next (lua_State *L, Table *t, StkId key);
-LUAI_FUNC int luaH_next_ro (lua_State *L, void *t, StkId key);
 LUAI_FUNC int luaH_getn (Table *t);
-LUAI_FUNC int luaH_getn_ro (void *t);
 LUAI_FUNC int luaH_isdummy (Node *n);
 
 #define LUA_MAX_ROTABLE_NAME  32

app/lua/ltablib.c

@@ -12,7 +12,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 
 #define aux_getn(L,n)	(luaL_checktype(L, n, LUA_TTABLE), luaL_getn(L, n))

app/lua/ltm.c

@@ -70,7 +70,7 @@ const TValue *luaT_gettmbyobj (lua_State *L, const TValue *o, TMS event) {
       mt = uvalue(o)->metatable;
       break;
     default:
-      mt = G(L)->mt[basettype(o)];
+      mt = G(L)->mt[ttnov(o)];
   }
   return (mt ? luaH_getstr(mt, G(L)->tmname[event]) : luaO_nilobject);
 }

app/lua/lua.h

@@ -9,9 +9,9 @@
 #ifndef lua_h
 #define lua_h
 #include <stdint.h>
-#include "stdarg.h"
+#include <stdarg.h>
-#include "stddef.h"
+#include <stddef.h>
-#include "ctype.h"
+#include <ctype.h>
 
 #include "luaconf.h"
 
@@ -99,8 +99,8 @@ typedef void * (*lua_Alloc) (void *ud, void *ptr, size_t osize, size_t nsize);
 #include LUA_USER_H
 #endif
 
-#if defined(LUA_OPTIMIZE_DEBUG) && LUA_OPTIMIZE_DEBUG == 0
+#ifndef LUAI_OPTIMIZE_DEBUG
-#undef LUA_OPTIMIZE_DEBUG
+#define LUAI_OPTIMIZE_DEBUG 2
 #endif
 
 /* type of numbers in Lua */
@@ -125,6 +125,7 @@ LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf);
 /*
 ** basic stack manipulation
 */
+LUA_API int   (lua_absindex) (lua_State *L, int idx);
 LUA_API int   (lua_gettop) (lua_State *L);
 LUA_API void  (lua_settop) (lua_State *L, int idx);
 LUA_API void  (lua_pushvalue) (lua_State *L, int idx);
@@ -148,9 +149,11 @@ LUA_API int             (lua_type) (lua_State *L, int idx);
 LUA_API int             (lua_fulltype) (lua_State *L, int idx);
 LUA_API const char     *(lua_typename) (lua_State *L, int tp);
 
-LUA_API int            (lua_equal) (lua_State *L, int idx1, int idx2);
 LUA_API int             (lua_rawequal) (lua_State *L, int idx1, int idx2);
-LUA_API int            (lua_lessthan) (lua_State *L, int idx1, int idx2);
+#define LUA_OPEQ    0
+#define LUA_OPLT    1
+#define LUA_OPLE    2
+LUA_API int             (lua_compare) (lua_State *L, int idx1, int idx2, int op);
 
 LUA_API lua_Number      (lua_tonumber) (lua_State *L, int idx);
 LUA_API lua_Integer     (lua_tointeger) (lua_State *L, int idx);
@@ -183,10 +186,11 @@ LUA_API int   (lua_pushthread) (lua_State *L);
 /*
 ** get functions (Lua -> stack)
 */
-LUA_API void  (lua_gettable) (lua_State *L, int idx);
+LUA_API int   (lua_gettable) (lua_State *L, int idx);
-LUA_API void  (lua_getfield) (lua_State *L, int idx, const char *k);
+LUA_API int   (lua_getfield) (lua_State *L, int idx, const char *k);
-LUA_API void  (lua_rawget) (lua_State *L, int idx);
+LUA_API int   (lua_rawget) (lua_State *L, int idx);
-LUA_API void  (lua_rawgeti) (lua_State *L, int idx, int n);
+LUA_API int   (lua_rawgeti) (lua_State *L, int idx, int n);
+LUA_API int   (lua_rawgetp) (lua_State *L, int idx, const void *p);
 LUA_API void  (lua_createtable) (lua_State *L, int narr, int nrec);
 LUA_API void *(lua_newuserdata) (lua_State *L, size_t sz);
 LUA_API int   (lua_getmetatable) (lua_State *L, int objindex);
@@ -200,6 +204,7 @@ LUA_API void  (lua_settable) (lua_State *L, int idx);
 LUA_API void  (lua_setfield) (lua_State *L, int idx, const char *k);
 LUA_API void  (lua_rawset) (lua_State *L, int idx);
 LUA_API void  (lua_rawseti) (lua_State *L, int idx, int n);
+LUA_API void  (lua_rawsetp) (lua_State *L, int idx, const void *p);
 LUA_API int   (lua_setmetatable) (lua_State *L, int objindex);
 LUA_API int   (lua_setfenv) (lua_State *L, int idx);
 
@@ -212,8 +217,8 @@ LUA_API int   (lua_pcall) (lua_State *L, int nargs, int nresults, int errfunc);
 LUA_API int   (lua_cpcall) (lua_State *L, lua_CFunction func, void *ud);
 LUA_API int   (lua_load) (lua_State *L, lua_Reader reader, void *dt,
                                         const char *chunkname);
-
+LUA_API int   (lua_dump) (lua_State *L, lua_Writer writer, void *data, int stripping);
-LUA_API int (lua_dumpEx) (lua_State *L, lua_Writer writer, void *data, int stripping);
+LUA_API int   (lua_stripdebug) (lua_State *L, int stripping);
 
 
 /*
@@ -273,9 +278,7 @@ LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud);
 #define lua_strlen(L,i)		lua_objlen(L, (i))
 
 #define lua_isfunction(L,n)	(lua_type(L, (n)) == LUA_TFUNCTION)
-#define lua_islightfunction(L,n) (lua_fulltype(L, (n)) == LUA_TLIGHTFUNCTION)
 #define lua_istable(L,n)	(lua_type(L, (n)) == LUA_TTABLE)
-#define lua_isrotable(L,n)	(lua_fulltype(L, (n)) == LUA_TROTABLE)
 #define lua_islightuserdata(L,n) (lua_type(L, (n)) == LUA_TLIGHTUSERDATA)
 #define lua_isnil(L,n)		(lua_type(L, (n)) == LUA_TNIL)
 #define lua_isboolean(L,n)	(lua_type(L, (n)) == LUA_TBOOLEAN)
@@ -291,9 +294,10 @@ LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud);
 
 #define lua_tostring(L,i)	lua_tolstring(L, (i), NULL)
 
-#define lua_dump(L,w,d)		lua_dumpEx(L,w,d,0)
+#define lua_equal(L,idx1,idx2)		lua_compare(L,(idx1),(idx2),LUA_OPEQ)
+#define lua_lessthan(L,idx1,idx2)	lua_compare(L,(idx1),(idx2),LUA_OPLT)
 
-/* error codes from cross-compiler returned by lua_dumpEx */
+/* error codes from cross-compiler returned by lua_dump */
 /* target integer is too small to hold a value */
 #define LUA_ERR_CC_INTOVERFLOW 101
 
@@ -382,14 +386,18 @@ struct lua_Debug {
 
 /* }====================================================================== */
 
+/* NodeMCU extensions to the standard API */
+
 typedef struct ROTable ROTable;
 typedef const struct ROTable_entry ROTable_entry;
 
 LUA_API void (lua_pushrotable) (lua_State *L, const ROTable *p);
 LUA_API void (lua_createrotable) (lua_State *L, ROTable *t, ROTable_entry *e, ROTable *mt);
+LUA_API int  (lua_pushstringsarray) (lua_State *L, int opt);
+LUA_API int  (lua_freeheap) (void);
 
-LUAI_FUNC int  lua_lfsreload (lua_State *L);
+LUA_API void (lua_getlfsconfig) (lua_State *L, int *);
-LUAI_FUNC int  lua_lfsindex (lua_State *L);
+LUA_API int  (lua_pushlfsindex) (lua_State *L);
 
 #define EGC_NOT_ACTIVE        0   // EGC disabled
 #define EGC_ON_ALLOC_FAILURE  1   // run EGC on allocation failure
@@ -400,15 +408,13 @@ LUAI_FUNC int  lua_lfsindex (lua_State *L);
 
 #define LUA_QUEUE_APP   0
 #define LUA_QUEUE_UART  1
-#define LUA_TASK_LOW    0
-#define LUA_TASK_MEDIUM 1
-#define LUA_TASK_HIGH   2
 
 /**DEBUG**/extern void dbg_printf(const char *fmt, ...)
                        __attribute__ ((format (printf, 1, 2)));
-#define luaN_freearray(L,b,l)  luaM_freearray(L,b,l,sizeof(*b));
+#define luaN_freearray(L,b,l)  luaM_freearray(L,b,l,sizeof(*b))
 
-LUA_API void lua_setegcmode(lua_State *L, int mode, int limit);
+LUA_API void (lua_setegcmode) (lua_State *L, int mode, int limit);
+LUA_API void (lua_getegcinfo) (lua_State *L, int *totals);
 
 #else
 

app/lua/luac_cross/Makefile

@@ -43,7 +43,7 @@ LUASRC  := lapi.c      lauxlib.c   lbaselib.c  lcode.c     ldblib.c    ldebug.c
            ldo.c       ldump.c     lfunc.c     lgc.c       linit.c     llex.c \
            lmathlib.c  lmem.c      loadlib.c   lobject.c   lopcodes.c  lparser.c \
            lstate.c    lstring.c   lstrlib.c   ltable.c    ltablib.c \
-           ltm.c       lundump.c   lvm.c       lzio.c
+           ltm.c       lundump.c   lvm.c       lzio.c      lnodemcu.c
 UZSRC   := uzlib_deflate.c crc32.c
 
 #

app/lua/luac_cross/lflashimg.c

@@ -186,10 +186,8 @@ static void scanProtoStrings(lua_State *L, const Proto* f) {
   if (f->source)
     addTS(L, f->source);
 
-#ifdef LUA_OPTIMIZE_DEBUG
   if (f->packedlineinfo)
     addTS(L, luaS_new(L, cast(const char *, f->packedlineinfo)));
-#endif
 
   for (i = 0; i < f->sizek; i++) {
     if (ttisstring(f->k + i))
@@ -355,11 +353,7 @@ static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) {
 }
 
 /* The debug optimised version has a different Proto layout */
-#ifdef LUA_OPTIMIZE_DEBUG
 #define PROTO_COPY_MASK  "AHAAAAAASIIIIIIIAI"
-#else
-#define PROTO_COPY_MASK  "AHAAAAAASIIIIIIIIAI"
-#endif
 
 /*
  * Do the actual prototype copy.
@@ -383,14 +377,10 @@ static void *functionToFlash(lua_State* L, const Proto* orig) {
   f.k = cast(TValue *, flashCopy(L, f.sizek, "V", f.k));
   f.code = cast(Instruction *, flashCopy(L, f.sizecode, "I", f.code));
 
-#ifdef LUA_OPTIMIZE_DEBUG
   if (f.packedlineinfo) {
     TString *ts=luaS_new(L, cast(const char *,f.packedlineinfo));
     f.packedlineinfo = cast(unsigned char *, resolveTString(L, ts)) + sizeof (FlashTS);
   }
-#else
-  f.lineinfo = cast(int *, flashCopy(L, f.sizelineinfo, "I", f.lineinfo));
-#endif
   f.locvars = cast(struct LocVar *, flashCopy(L, f.sizelocvars, "SII", f.locvars));
   f.upvalues = cast(TString **, flashCopy(L, f.sizeupvalues, "S", f.upvalues));
   return cast(void *, flashCopy(L, 1, PROTO_COPY_MASK, &f));

app/lua/luac_cross/mingw32-Makefile.mak

@@ -36,8 +36,8 @@ LUACSRC := luac.c      lflashimg.c liolib.c    loslib.c    print.c
 LUASRC  := lapi.c      lauxlib.c   lbaselib.c  lcode.c     ldblib.c    ldebug.c \
            ldo.c       ldump.c     lfunc.c     lgc.c       linit.c     llex.c \
            lmathlib.c  lmem.c      loadlib.c   lobject.c   lopcodes.c  lparser.c \
-           lrotable.c  lstate.c    lstring.c   lstrlib.c   ltable.c    ltablib.c \
+           lstate.c    lstring.c   lstrlib.c   ltable.c    ltablib.c \
-           ltm.c       lundump.c   lvm.c       lzio.c
+           ltm.c       lundump.c   lvm.c       lzio.c      lnodemcu.c
 UZSRC   := uzlib_deflate.c crc32.c
 #
 # This relies on the files being unique on the vpath

app/lua/lundump.c

@@ -223,18 +223,12 @@ static void LoadDebug(LoadState* S, Proto* f)
  n=LoadInt(S);
  Align4(S);
 
-#ifdef LUA_OPTIMIZE_DEBUG
  if(n) {
    f->packedlineinfo=luaM_newvector(S->L,n,unsigned char);
    LoadBlock(S,f->packedlineinfo,n);
  } else {
    f->packedlineinfo=NULL;
  }
-#else
- f->lineinfo=luaM_newvector(S->L,n,int);
- LoadVector(S,f->lineinfo,n,sizeof(int));
- f->sizelineinfo=n;
- #endif
  n=LoadInt(S);
  f->locvars=luaM_newvector(S->L,n,LocVar);
  f->sizelocvars=n;

app/lua/lvm.c

@@ -271,7 +271,7 @@ int luaV_lessthan (lua_State *L, const TValue *l, const TValue *r) {
 }
 
 
-static int lessequal (lua_State *L, const TValue *l, const TValue *r) {
+int luaV_lessequal (lua_State *L, const TValue *l, const TValue *r) {
   int res;
   if (ttype(l) != ttype(r))
     return luaG_ordererror(L, l, r);
@@ -570,7 +570,7 @@ void luaV_execute (lua_State *L, int nexeccalls) {
       }
       case OP_LEN: {
         const TValue *rb = RB(i);
-        switch (basettype(rb)) {
+        switch (ttnov(rb)) {
           case LUA_TTABLE: {
             setnvalue(ra, cast_num(luaH_getn(hvalue(rb))));
             break;
@@ -620,7 +620,7 @@ void luaV_execute (lua_State *L, int nexeccalls) {
       }
       case OP_LE: {
         Protect(
-          if (lessequal(L, RKB(i), RKC(i)) == GETARG_A(i))
+          if (luaV_lessequal(L, RKB(i), RKC(i)) == GETARG_A(i))
             dojump(L, pc, GETARG_sBx(*pc));
         )
         pc++;

app/lua/lvm.h

@@ -23,6 +23,7 @@
 
 
 LUAI_FUNC int luaV_lessthan (lua_State *L, const TValue *l, const TValue *r);
+LUAI_FUNC int luaV_lessequal (lua_State *L, const TValue *l, const TValue *r);
 LUAI_FUNC int luaV_equalval (lua_State *L, const TValue *t1, const TValue *t2);
 LUAI_FUNC const TValue *luaV_tonumber (const TValue *obj, TValue *n);
 LUAI_FUNC int luaV_tostring (lua_State *L, StkId obj);

app/lua53/lapi.c

@@ -1031,6 +1031,8 @@ LUA_API int lua_dump (lua_State *L, lua_Writer writer, void *data, int strip) {
   lua_lock(L);
   api_checknelems(L, 1);
   o = L->top - 1;
+  if (strip == -1)
+    strip = G(L)->stripdefault;
   if (isLfunction(o))
     status = luaU_dump(L, getproto(o), writer, data, strip);
   else
@@ -1040,6 +1042,30 @@ LUA_API int lua_dump (lua_State *L, lua_Writer writer, void *data, int strip) {
 }
 
 
+LUA_API int lua_stripdebug (lua_State *L, int stripping){
+  TValue *o = L->top - 1; 
+  Proto *p = NULL;
+  int res = -1;
+  lua_lock(L);
+  api_checknelems(L, 1);
+  if (isLfunction(o)) {
+    p = getproto(o);
+    if (p && !isLFSobj(p) && (unsigned) stripping < 3 ) {
+    // found a valid proto to strip
+      res = luaU_stripdebug(L, p, stripping, 1);
+    }
+  } else if (ttisnil(L->top - 1)) {
+    // get or set the default strip level
+    if ((unsigned) stripping < 3)
+      G(L)->stripdefault = stripping;
+    res = G(L)->stripdefault;
+  }
+  L->top--;
+  lua_unlock(L);
+  return res;
+}
+
+
 LUA_API int lua_status (lua_State *L) {
   return L->status;
 }

app/lua53/lauxlib.c

@@ -5,7 +5,6 @@
 */
 
 #define lauxlib_c
-#define LUA_LIB
 
 #include "lprefix.h"
 
@@ -29,6 +28,8 @@
 #include "lua.h"
 
 #include "lauxlib.h"
+#define LUA_LIB
+
 #ifdef LUA_USE_ESP
 #include "platform.h"
 #include "user_interface.h"
@@ -651,7 +652,6 @@ LUALIB_API void luaL_unref (lua_State *L, int t, int ref) {
 
 
 LUALIB_API void (luaL_reref) (lua_State *L, int t, int *ref) {
-  int reft;
 /*
  * If the ref is positive and the entry in table t exists then
  * overwrite the value otherwise fall through to luaL_ref()
@@ -659,8 +659,7 @@ LUALIB_API void (luaL_reref) (lua_State *L, int t, int *ref) {
   if (ref) {
     if (*ref >= 0) {
       t = lua_absindex(L, t);
-      lua_rawgeti(L, t, *ref);
+      int reft = lua_rawgeti(L, t, *ref);
-      reft = lua_type(L, -1);
       lua_pop(L, 1);
       if (reft != LUA_TNIL) {
         lua_rawseti(L, t, *ref);
@@ -706,7 +705,6 @@ typedef struct LoadF {
   char buff[BUFSIZ];  /* area for reading file */
 } LoadF;
 
-#include "llimits.h"
 static const char *getF (lua_State *L, void *ud, size_t *size) {
   LoadF *lf = (LoadF *)ud;
   (void)L;  /* not used */
@@ -1187,54 +1185,4 @@ LUALIB_API int luaL_pcallx (lua_State *L, int narg, int nres) {
     }
   return status;
 }
-
-extern void lua_main(void);
-/*
-** Task callback handler. Uses luaN_call to do a protected call with full traceback
-*/
-static void do_task (platform_task_param_t task_fn_ref, uint8_t prio) {
-  lua_State* L = lua_getstate();
-  if(task_fn_ref == (platform_task_param_t)~0 && prio == LUA_TASK_HIGH) {
-    lua_main();                   /* Undocumented hook for lua_main() restart */
-    return;
-  }
-  if (prio < LUA_TASK_LOW|| prio > LUA_TASK_HIGH)
-    luaL_error(L, "invalid posk task");
-/* Pop the CB func from the Reg */
-  lua_rawgeti(L, LUA_REGISTRYINDEX, (int) task_fn_ref);
-  luaL_checktype(L, -1, LUA_TFUNCTION);
-  luaL_unref(L, LUA_REGISTRYINDEX, (int) task_fn_ref);
-  lua_pushinteger(L, prio);
-  luaL_pcallx(L, 1, 0);
-}
-
-/*
-** Schedule a Lua function for task execution
-*/
-LUALIB_API int luaL_posttask ( lua_State* L, int prio ) {         // [-1, +0, -]
-  static platform_task_handle_t task_handle = 0;
-  if (!task_handle)
-    task_handle = platform_task_get_id(do_task);
-  if (L == NULL && prio == LUA_TASK_HIGH+1) { /* Undocumented hook for lua_main */
-    platform_post(LUA_TASK_HIGH, task_handle, (platform_task_param_t)~0);
-    return -1;
-  }
-  if (lua_isfunction(L, -1) && prio >= LUA_TASK_LOW && prio <= LUA_TASK_HIGH) {
-    int task_fn_ref = luaL_ref(L, LUA_REGISTRYINDEX);
-    if(!platform_post(prio, task_handle, (platform_task_param_t)task_fn_ref)) {
-      luaL_unref(L, LUA_REGISTRYINDEX, task_fn_ref);
-      luaL_error(L, "Task queue overflow. Task not posted");
-    }
-    return task_fn_ref;
-  } else {
-    return luaL_error(L, "invalid posk task");
-  }
-}
-#else
-/*
-** Task execution isn't supported on HOST builds so returns a -1 status
-*/
-LUALIB_API int luaL_posttask( lua_State* L, int prio ) {            // [-1, +0, -]
-  return -1;
-}
 #endif

app/lua53/lauxlib.h

@@ -14,6 +14,9 @@
 
 #include "lua.h"
 
+#ifdef LUA_LIB
+#include "lnodemcu.h"
+#endif
 
 
 /* extra error code for 'luaL_loadfilex' */
@@ -79,7 +82,7 @@ LUALIB_API int (luaL_execresult) (lua_State *L, int stat);
 
 LUALIB_API int (luaL_ref) (lua_State *L, int t);
 LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref);
-#define luaL_unref2(l,t,r) luaL_unref(L, (t), (r)); r = LUA_NOREF
+#define luaL_unref2(l,t,r) do {luaL_unref(L, (t), (r)); r = LUA_NOREF;} while (0)
 LUALIB_API void (luaL_reref) (lua_State *L, int t, int *ref);
 
 LUALIB_API int (luaL_loadfilex) (lua_State *L, const char *filename,
@@ -282,9 +285,14 @@ extern void dbg_printf(const char *fmt, ...);
 #define LUA_TASK_MEDIUM 1
 #define LUA_TASK_HIGH   2
 
+LUALIB_API int  (luaL_pushlfsmodules) (lua_State *L);
+LUALIB_API int  (luaL_pushlfsdts) (lua_State *L);
+LUALIB_API void (luaL_lfsreload) (lua_State *L);
 LUALIB_API int  (luaL_posttask) (lua_State* L, int prio);
 LUALIB_API int  (luaL_pcallx) (lua_State *L, int narg, int nres);
 
+#define luaL_pushlfsmodule(l) lua_pushlfsfunc(L)
+
 /* }============================================================ */
 
 #endif

app/lua53/lbaselib.c

@@ -18,7 +18,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 
 static int luaB_print (lua_State *L) {

app/lua53/lcorolib.c

@@ -16,7 +16,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 
 static lua_State *getco (lua_State *L) {

app/lua53/ldblib.c

@@ -18,7 +18,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 
 /*
@@ -48,16 +47,15 @@ static int db_getregistry (lua_State *L) {
 static int db_getstrings (lua_State *L) {
   static const char *const opts[] = {"RAM","ROM",NULL};
   int opt = luaL_checkoption(L, 1, "RAM", opts);
-  int st = lua_getstrings(L, opt);    /* return the relevant strt as an array */
+  if (lua_pushstringsarray(L, opt)) {
-  if (st) {
+    if(lua_getglobal(L, "table") == LUA_TTABLE) {
-    lua_pushvalue(L, -1);                             /* dup the array TValue */
-    lua_getglobal(L, "table");
       lua_getfield(L, -1, "sort");             /* look up table.sort function */
-    lua_replace(L, -2);                               /* dump the table entry */
+      lua_pushvalue(L, -3);                   /* dup the strings_table to ToS */
-    lua_insert(L, -2);               /* swap table/sort and the strings_table */
       lua_call(L, 1, 0);                         /* table.sort(strings_table) */
+      lua_pop(L, 1);                                  /* dump the table entry */
     }
-  return st ? 1 : 0;
+  }
+  return 1;
 }
 
 static int db_getmetatable (lua_State *L) {

app/lua53/ldump.c

@@ -17,6 +17,7 @@
 #include "lauxlib.h"
 #include "llex.h"
 #include "lgc.h"
+#include "lmem.h"
 #include "lobject.h"
 #include "lstate.h"
 #include "lstring.h"
@@ -263,6 +264,36 @@ int luaU_dump (lua_State *L, const Proto *f, lua_Writer w, void *data,
   return D.status;
 }
 
+static int stripdebug (lua_State *L, Proto *f, int level) {
+  int i, len = 0;
+  switch (level) {
+    case 2:
+      if (f->lineinfo) {
+        f->lineinfo = luaM_freearray(L, f->lineinfo, f->sizelineinfo);
+        len += f->sizelineinfo;
+      }
+    case 1:
+      for (i=0; i<f->sizeupvalues; i++)
+        f->upvalues[i].name = NULL;
+      f->locvars = luaM_freearray(L, f->locvars, f->sizelocvars);
+      len += f->sizelocvars * sizeof(LocVar);
+      f->sizelocvars = 0;
+  }
+  return len;
+}
+
+/* This is a recursive function so it's stack size has been kept to a minimum! */
+int luaU_stripdebug (lua_State *L, Proto *f, int level, int recv){
+  int len = 0, i;
+  if (recv != 0 && f->sizep != 0) {
+    for(i=0;i<f->sizep;i++) len += luaU_stripdebug(L, f->p[i], level, recv);
+  }
+  len += stripdebug (L, f, level);
+  return len;
+}
+
+
+
 /*============================================================================**
 **
 ** NodeMCU extensions for LFS support and dumping.  Note that to keep lua_lock
@@ -279,6 +310,8 @@ int luaU_dump (lua_State *L, const Proto *f, lua_Writer w, void *data,
 */
 static void addTS (TString *ts, DumpState *D) {
   lua_State *L = D->L;
+  if (!ts)
+    return;
   if (ttisnil(luaH_getstr(D->stringIndex, ts))) {
     TValue k, v, *slot;
     gettt(ts)<=LUA_TSHRSTR ? D->sTScnt++ : D->lTScnt++;

app/lua53/lgc.c

@@ -1133,6 +1133,13 @@ static l_mem getdebt (global_State *g) {
 /*
 ** performs a basic GC step when collector is running
 */
+#ifdef LUA_USE_ESP8266 /*DEBUG*/
+extern void dbg_printf(const char *fmt, ...);
+#define CCOUNT_REG ({ int32_t r; asm volatile("rsr %0, ccount" : "=r"(r)); r;})
+#else                  /*DEBUG*/   
+#define dbg_printf(...)
+#define CCOUNT_REG 0
+#endif                 /*DEBUG*/
 void luaC_step (lua_State *L) {
   global_State *g = G(L);
   l_mem debt = getdebt(g);  /* GC deficit (be paid now) */
@@ -1141,15 +1148,19 @@ void luaC_step (lua_State *L) {
     return;
   }
   do {  /* repeat until pause or enough "credit" (negative debt) */
+/*DEBUG  int32_t start = CCOUNT_REG; */
     lu_mem work = singlestep(L);  /* perform one single step */
     debt -= work;
+/*DEBUG  dbg_printf("singlestep - %d, %d, %u \n", debt, lua_freeheap(), CCOUNT_REG-start); */
   } while (debt > -GCSTEPSIZE && g->gcstate != GCSpause);
   if (g->gcstate == GCSpause)
     setpause(g);  /* pause until next cycle */
   else {
+/*DEBUG  int32_t start = CCOUNT_REG; */
     debt = (debt / g->gcstepmul) * STEPMULADJ;  /* convert 'work units' to Kb */
     luaE_setdebt(g, debt);
     runafewfinalizers(L);
+/*DEBUG  dbg_printf("new debt - %d, %d, %u \n", debt, lua_freeheap(), CCOUNT_REG-start); */
   }
 }
 

app/lua53/linit.c

@@ -47,7 +47,6 @@ extern LROT_TABLE(dblib);
 extern LROT_TABLE(co_funcs);
 extern LROT_TABLE(mathlib);
 extern LROT_TABLE(utf8);
-extern LROT_TABLE(LFS);
 
 #define LROT_ROM_ENTRIES \
   LROT_TABENTRY( string, strlib ) \
@@ -56,7 +55,6 @@ extern LROT_TABLE(LFS);
   LROT_TABENTRY( coroutine, co_funcs ) \
   LROT_TABENTRY( math, mathlib ) \
   LROT_TABENTRY( utf8, utf8 ) \
-  LROT_TABENTRY( LFS, LFS ) \
   LROT_TABENTRY( ROM, rotables )
 
 #define LROT_LIB_ENTRIES \
@@ -75,7 +73,7 @@ extern LROT_TABLE(LFS);
 
 /* _G __index -> rotables __index -> base_func */
 extern LROT_TABLE(rotables_meta);
-LROT_TABLE(base_func);
+extern LROT_TABLE(base_func);
 
 LROT_BEGIN(rotables_meta, NULL, LROT_MASK_INDEX)
   LROT_TABENTRY( __index, base_func)

app/lua53/lmathlib.c

@@ -17,8 +17,7 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
+
-#include "ldebug.h"
 #undef PI
 #define PI	(l_mathop(3.141592653589793238462643383279502884))
 
@@ -135,7 +134,7 @@ static int math_fmod (lua_State *L) {
       lua_pushinteger(L, lua_tointeger(L, 1) % d);
   } else {
     lua_Number m, a=luaL_checknumber(L, 1), b=luaL_checknumber(L, 2);
-    if (b==0) luaG_runerror(L,"modulo by zero");
+    if (b==0) luaL_error(L,"modulo by zero");
     m = a/b;
     lua_pushnumber(L, a - b*(m > 0.0 ? floor(m) : ceil(m)));
   }

app/lua53/lnodemcu.c

@@ -30,8 +30,8 @@
 **    *  POSIX vs VFS file API abstraction
 **    *  Emulate Platform_XXX() API
 **    *  ESP and HOST lua_debugbreak() test stubs
-**    *  NodeMCU lua.h API extensions
+**    *  NodeMCU lua.h LUA_API extensions
-**    *  NodeMCU LFS Table emulator
+**    *  NodeMCU lauxlib.h LUALIB_API extensions
 **    *  NodeMCU bootstrap to set up and to reimage LFS resources
 **
 ** Just search down for //== or ==// to flip through the sections.
@@ -39,12 +39,12 @@
 
 #define byte_addr(p)    cast(char *,p)
 #define byteptr(p)      cast(lu_byte *, p)
-#define byteoffset(p,q) (byteptr(p) - byteptr(q))
+#define byteoffset(p,q) ((int) cast(ptrdiff_t, (byteptr(p) - byteptr(q))))
 #define wordptr(p)      cast(lu_int32 *, p)
 #define wordoffset(p,q) (wordptr(p) - wordptr(q))
 
 
-//== Wrap POSIX and VFS file API =============================================//
+//====================== Wrap POSIX and VFS file API =========================//
 #ifdef LUA_USE_ESP
 int luaopen_file(lua_State *L);
 #  define l_file(f)   int f
@@ -62,7 +62,6 @@ int luaopen_file(lua_State *L);
 #  define l_rewind(f) rewind(f)
 #endif
 
-//== Emulate Platform_XXX() API ==============================================//
 #ifdef LUA_USE_ESP
 
 extern void dbg_printf(const char *fmt, ...);                // DEBUG
@@ -79,6 +78,8 @@ extern void dbg_printf(const char *fmt, ...);                // DEBUG
 
 #else // LUA_USE_HOST
 
+//==== Emulate Platform_XXX() API within host luac.cross -e environement =====//
+
 #include<stdio.h>                                             // DEBUG
 /*
 ** The ESP implementation use a platform_XXX() API to provide a level of
@@ -124,9 +125,9 @@ extern char  *LFSimageName;
 
 #define platform_rcr_write(id,rec,l) (128)
 #define platform_flash_phys2mapped(n) \
-    (byteptr(LFSaddr) + (n) - LFSbase)
+    ((ptrdiff_t)(((size_t)LFSaddr) - LFSbase) + (n))
 #define platform_flash_mapped2phys(n) \
-    (byteoffset(n, LFSaddr) + LFSbase)
+    ((size_t)(n) - ((size_t)LFSaddr) + LFSbase)
 #define platform_flash_get_sector_of_address(n) ((n)>>12)
 #define platform_rcr_delete(id) LFSimageName = NULL
 #define platform_rcr_read(id,s) \
@@ -170,7 +171,7 @@ static void platform_s_flash_write(const void *from, lu_int32 to, lu_int32 len)
 
 #endif
 
-//== ESP and HOST lua_debugbreak() test stubs ================================//
+//============= ESP and HOST lua_debugbreak() test stubs =====================//
 
 #ifdef DEVELOPMENT_USE_GDB
 /*
@@ -204,7 +205,7 @@ LUALIB_API void lua_debugbreak(void) {
 }
 #endif
 
-//== NodeMCU lua.h API extensions ============================================//
+//===================== NodeMCU lua.h API extensions =========================//
 
 LUA_API int lua_freeheap (void) {
 #ifdef LUA_USE_HOST
@@ -214,34 +215,39 @@ LUA_API int lua_freeheap (void) {
 #endif
 }
 
-LUA_API int lua_getstrings(lua_State *L, int opt) {
+LUA_API int lua_pushstringsarray(lua_State *L, int opt) {
-  stringtable *tb = NULL;
+  stringtable *strt = NULL;
-  Table *t;
+  int i, j = 1;
-  int i, j, n = 0;
-
-  if (n == 0)
-    tb = &G(L)->strt;
-#ifdef LUA_USE_ESP
-  else if (n == 1 && G(L)->ROstrt.hash)
-    tb = &G(L)->ROstrt;
-#endif
-  if (tb == NULL)
-   return 0;
  
   lua_lock(L);
-  t = luaH_new(L);
+  if (opt == 0)
+    strt = &G(L)->strt;
+#ifdef LUA_USE_ESP
+  else if (opt == 1 && G(L)->ROstrt.hash)
+    strt = &G(L)->ROstrt;
+#endif
+  if (strt == NULL) {
+    setnilvalue(L->top);
+    api_incr_top(L);
+    lua_unlock(L);
+    return 0;
+  }
+
+  Table *t = luaH_new(L);
   sethvalue(L, L->top, t);
   api_incr_top(L);
-  luaH_resize(L, t, tb->nuse, 0);
+  luaH_resize(L, t, strt->nuse, 0);
   luaC_checkGC(L);
   lua_unlock(L);
 
-  for (i = 0, j = 1; i < tb->size; i++) {
+  /* loop around all strt hash entries */
-    TString *o;
+  for (i = 0, j = 1; i < strt->size; i++) {
-    for(o = tb->hash[i]; o; o = o->u.hnext) {
+    TString *e;
+    /* loop around all TStings in this entry's chain */
+    for(e = strt->hash[i]; e; e = e->u.hnext) {
       TValue s;
-      setsvalue(L, &s, o);
+      setsvalue(L, &s, e);
-      luaH_setint(L, hvalue(L->top-1), j++, &s);  /* table[n] = true */
+      luaH_setint(L, hvalue(L->top-1), j++, &s);
     }
   }
   return 1;
@@ -274,72 +280,91 @@ LUA_API void lua_createrotable (lua_State *L, ROTable *t,
   t->entry     = cast(ROTable_entry *, e);
 }
 
-//== NodeMCU LFS Table emulator ==============================================//
+LUA_API void lua_getlfsconfig (lua_State *L, int *config) {
-
+  global_State *g = G(L);
-static int lfs_func (lua_State* L);
+  LFSHeader *l = g->l_LFS;  
-
+  if (!config)
-LROT_BEGIN(LFS_meta, NULL, LROT_MASK_INDEX)
+    return;
-  LROT_FUNCENTRY( __index, lfs_func)
+  config[0] = (int) (size_t) l;                  /* LFS region mapped address */
-LROT_END(LFS_meta, NULL, LROT_MASK_INDEX)
+  config[1] = platform_flash_mapped2phys(config[0]);    /* ditto phys address */
-
+  config[2] = g->LFSsize;                           /* LFS region actual size */
-LROT_BEGIN(LFS, LROT_TABLEREF(LFS_meta), 0)
+  if (g->ROstrt.hash) {
-LROT_END(LFS, LROT_TABLEREF(LFS_meta), 0)
+    config[3] = l->flash_size;                             /* LFS region used */
-
+    config[4] = l->timestamp;                         /* LFS region timestamp */
-
+  } else { 
-static int lfs_func (lua_State* L) {           /*T[1] = LFS, T[2] = fieldname */
+    config[3] = config[4] = 0;
-  const char *name = lua_tostring(L, 2);
-  LFSHeader *fh = G(L)->l_LFS;
-  Proto *f;
-  LClosure *cl;
-  lua_settop(L,2);
-  if (!fh) {                                  /* return nil if LFS not loaded */
-    lua_pushnil(L);
-    return 1;
   }
-  if (!strcmp(name, "_config")) {
+}
-    size_t ba = cast(size_t, fh);
+
-    lua_createtable(L, 0, 3);
+LUA_API int  (lua_pushlfsindex) (lua_State *L) {
-    lua_pushinteger(L, cast(lua_Integer, ba));
-    lua_setfield(L, -2, "lfs_mapped");
-    lua_pushinteger(L, cast(lua_Integer, platform_flash_mapped2phys(ba)));
-    lua_setfield(L, -2, "lfs_base");
-    lua_pushinteger(L, G(L)->LFSsize);
-    lua_setfield(L, -2, "lfs_size");
-    return 1;
-  } else if (!strcmp(name, "_list")) {
-    int i = 1;
-    setobjs2s(L, L->top-2, &G(L)->LFStable);  /* overwrite T[1] with LSFtable */
-    lua_newtable(L);                                /* new list table at T[3] */
-    lua_pushnil(L);                                 /* first key (nil) at T4] */
-    while (lua_next(L, 1) != 0) {         /* loop over LSFtable k,v at T[4:5] */
-      lua_pop(L, 1);                                            /* dump value */
-      lua_pushvalue(L, -1);                                        /* dup key */
-      lua_rawseti(L, 3, i++);                                 /* table[i]=key */
-    }
-    return 1;
-  } else if (!strcmp(name, "_time")) {
-    lua_pushinteger(L, fh->timestamp);
-    return 1;
-  }
-  setobjs2s(L, L->top-2, &G(L)->LFStable);    /* overwrite T[1] with LSFtable */
-  if (lua_rawget(L,1) != LUA_TNIL) {                    /* get LFStable[name] */
-    lua_pushglobaltable(L);
-    f = cast(Proto *, lua_topointer(L,-2));
   lua_lock(L);
-    cl = luaF_newLclosure(L, f->sizeupvalues);
+  setobj2n(L, L->top, &G(L)->LFStable);
-    setclLvalue(L, L->top-2, cl);       /* overwrite f addr slot with closure */
+  api_incr_top(L);
-    cl->p = f;                                                /* bind f to it */
-    if (cl->nupvalues >= 1) {           /* does it have at least one upvalue? */
-      luaF_initupvals(L, cl );                            /* initialise upvals */
-      setobj(L, cl->upvals[0]->v, L->top-1);             /* set UV[1] to _ENV */
-    }
   lua_unlock(L);
-    lua_pop(L,1);                          /* pop _ENV leaving closure at ToS */
+  return  ttnov(L->top-1);
+}
+
+/*
+ * In Lua 5.3 luac.cross generates a top level Proto for each source file with 
+ * one upvalue that must be the set to the _ENV variable when its closure is
+ * created, and as such this parallels some ldo.c processing.
+ */
+LUA_API int  (lua_pushlfsfunc) (lua_State *L) {
+  lua_lock(L);
+  const TValue *t = &G(L)->LFStable;
+  if (ttisstring(L->top-1) && ttistable(t)) {
+    const TValue *v = luaH_getstr (hvalue(t), tsvalue(L->top-1));
+    if (ttislightuserdata(v)) {
+      Proto *f = pvalue(v);   /* The pvalue is a Proto * for the Lua function */
+      LClosure *cl = luaF_newLclosure(L, f->sizeupvalues);
+      setclLvalue(L, L->top-1, cl);
+      luaF_initupvals(L, cl);
+      cl->p = f;
+      if (cl->nupvalues >= 1) {                   /* does it have an upvalue? */
+        UpVal *uv1 = cl->upvals[0];
+        TValue *val = uv1->v;
+        /* set 1st upvalue as global env table from registry */
+        setobj(L, val, luaH_getint(hvalue(&G(L)->l_registry), LUA_RIDX_GLOBALS));
+        luaC_upvalbarrier(L, uv1);
       }
       return 1;
+    }
+  }
+  setnilvalue(L->top-1);
+  lua_unlock(L);
+  return 0;
+}
+
+
+//================ NodeMCU lauxlib.h LUALIB_API extensions ===================//
+
+/*
+ * Return an array of functions in LFS
+ */
+LUALIB_API int  (luaL_pushlfsmodules) (lua_State *L) {
+  int i = 1;
+  if (lua_pushlfsindex(L) == LUA_TNIL)
+    return 0;                                 /* return nil if LFS not loaded */
+  lua_newtable(L);      /* create dest table and move above LFS index ROTable */
+  lua_insert(L, -2);
+  lua_pushnil(L);
+  while (lua_next(L, -2) != 0) {
+    lua_pop(L, 1);                       /* dump the value (ptr to the Proto) */
+    lua_pushvalue(L, -1);                            /* dup key (module name) */
+    lua_rawseti(L, -4, i++);
+  }
+  lua_pop(L, 1);                                /* dump the LFS index ROTable */
+  return 1;
 }
 
-//== NodeMCU bootstrap to set up and to reimage LFS resources ================//
+LUALIB_API int  (luaL_pushlfsdts) (lua_State *L) {
+  int config[5];
+  lua_getlfsconfig(L, config);
+  lua_pushinteger(L, config[4]);
+  return 1;
+}
+
+//======== NodeMCU bootstrap to set up and to reimage LFS resources ==========//
 /*
 ** This processing uses 2 init hooks during the Lua startup. The first is
 ** called early in the Lua state setup to initialize the LFS if present. The
@@ -494,9 +519,9 @@ LUAI_FUNC int luaN_init (lua_State *L) {
       if (n < 0) {
         global_State *g = G(L);
         g->LFSsize     = F->size;
+        g->l_LFS       = fh;
       /* Set up LFS hooks on normal Entry */
         if (fh->flash_sig   == FLASH_SIG) {
-          g->l_LFS       = fh;
           g->seed        = fh->seed;
           g->ROstrt.hash = cast(TString **, F->addr + fh->oROhash);
           g->ROstrt.nuse = fh->nROuse ;
@@ -565,9 +590,9 @@ LUAI_FUNC int luaN_init (lua_State *L) {
 #define getfield(L,t,f) \
   lua_getglobal(L, #t); luaL_getmetafield( L, 1, #f ); lua_remove(L, -2);
 
-LUAI_FUNC int  lua_lfsreload (lua_State *L) {
+LUALIB_API void luaL_lfsreload (lua_State *L) {
-  int n = 0;
 #ifdef LUA_USE_ESP
+  int n = 0;
   size_t l;
   int off = 0;
   const char *img = lua_tolstring(L, 1, &l);
@@ -579,30 +604,72 @@ LUAI_FUNC int  lua_lfsreload (lua_State *L) {
   lua_getglobal(L, "file");
   if (lua_isnil(L, 2)) {
     lua_pushstring(L, "No file system mounted");
-    return 1;
+    return;
   }
   lua_getfield(L, 2, "exists");
   lua_pushstring(L, img + off);
   lua_call(L, 1, 1);
   if (G(L)->LFSsize == 0 || lua_toboolean(L, -1) == 0) {
     lua_pushstring(L, "No LFS partition allocated");
-    return 1;
+    return;
   }
   n = platform_rcr_write(PLATFORM_RCR_FLASHLFS, img, l+1);/* incl trailing \0 */
-  if (n>0)
+  if (n>0) {
     system_restart();
+    luaL_error(L, "system restarting");
+  }
 #endif
-  lua_pushboolean(L, n>0);
-  return 1;
 }
 
-LUAI_FUNC int  lua_lfsindex (lua_State *L) {
+
-  lua_settop(L,1);
+#ifdef LUA_USE_ESP
-  if (lua_isstring(L, 1)){
+extern void lua_main(void);
-    lua_getglobal(L, "LFS");
+/*
-    lua_getfield(L, 2, lua_tostring(L,1));
+** Task callback handler. Uses luaN_call to do a protected call with full traceback
-  } else {
+*/
-    lua_pushnil(L);
+static void do_task (platform_task_param_t task_fn_ref, uint8_t prio) {
+  lua_State* L = lua_getstate();
+  if(task_fn_ref == (platform_task_param_t)~0 && prio == LUA_TASK_HIGH) {
+    lua_main();                   /* Undocumented hook for lua_main() restart */
+    return;
   }
-  return 1;
+  if (prio < LUA_TASK_LOW|| prio > LUA_TASK_HIGH)
+    luaL_error(L, "invalid posk task");
+/* Pop the CB func from the Reg */
+  lua_rawgeti(L, LUA_REGISTRYINDEX, (int) task_fn_ref);
+  luaL_checktype(L, -1, LUA_TFUNCTION);
+  luaL_unref(L, LUA_REGISTRYINDEX, (int) task_fn_ref);
+  lua_pushinteger(L, prio);
+  luaL_pcallx(L, 1, 0);
 }
+
+/*
+** Schedule a Lua function for task execution
+*/
+LUALIB_API int luaL_posttask ( lua_State* L, int prio ) {         // [-1, +0, -]
+  static platform_task_handle_t task_handle = 0;
+  if (!task_handle)
+    task_handle = platform_task_get_id(do_task);
+  if (L == NULL && prio == LUA_TASK_HIGH+1) { /* Undocumented hook for lua_main */
+    platform_post(LUA_TASK_HIGH, task_handle, (platform_task_param_t)~0);
+    return -1;
+  }
+  if (lua_isfunction(L, -1) && prio >= LUA_TASK_LOW && prio <= LUA_TASK_HIGH) {
+    int task_fn_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+    if(!platform_post(prio, task_handle, (platform_task_param_t)task_fn_ref)) {
+      luaL_unref(L, LUA_REGISTRYINDEX, task_fn_ref);
+      luaL_error(L, "Task queue overflow. Task not posted");
+    }
+    return task_fn_ref;
+  } else {
+    return luaL_error(L, "invalid posk task");
+  }
+}
+#else
+/*
+** Task execution isn't supported on HOST builds so returns a -1 status
+*/
+LUALIB_API int luaL_posttask( lua_State* L, int prio ) {            // [-1, +0, -]
+  return -1;
+}
+#endif

app/lua53/lnodemcu.h

@@ -40,12 +40,12 @@
 #define LROT_ENTRYREF(rt)    (rt ##_entries)
 #define LROT_TABLEREF(rt)    (&rt ##_ROTable)
 #define LROT_BEGIN(rt,mt,f)  LROT_TABLE(rt); \
-  static const ROTable_entry rt ## _entries[] = {
+  static ROTable_entry rt ## _entries[] = {
 #define LROT_ENTRIES_IN_SECTION(rt,s) \
-  static const ROTable_entry LOCK_IN_SECTION(s) rt ## _entries[] = {
+  static ROTable_entry LOCK_IN_SECTION(s) rt ## _entries[] = {
 #define LROT_END(rt,mt,f)    {NULL, LRO_NILVAL} }; \
   const ROTable rt ## _ROTable = { \
-    (GCObject *)1,LUA_TTBLROF, LROT_MARKED, \
+    (GCObject *)1, LUA_TTBLROF, LROT_MARKED, \
     cast(lu_byte, ~(f)), (sizeof(rt ## _entries)/sizeof(ROTable_entry)) - 1, \
     cast(Table *, mt), cast(ROTable_entry *, rt ## _entries) };
 #define LROT_BREAK(rt)       };

app/lua53/loadlib.c

@@ -22,7 +22,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 #ifndef LUA_USE_HOST
 #include <fcntl.h>

app/lua53/lobject.h

@@ -584,12 +584,6 @@ typedef struct ROTable {
 */
 #define luaO_nilobject		(&luaO_nilobject_)
 
-/*
-** KeyCache used for resolution of ROTable entries and Cstrings
-*/
-typedef size_t KeyCache;
-typedef KeyCache KeyCacheLine[KEYCACHE_M];
-
 LUAI_DDEC const TValue luaO_nilobject_;
 
 /* size of buffer for 'luaO_utf8esc' function */

app/lua53/lparser.c

@@ -23,9 +23,9 @@
 #include "lobject.h"
 #include "lopcodes.h"
 #include "lparser.h"
-#include "lstate.h"
 #include "lstring.h"
 #include "ltable.h"
+#include "lundump.h"
 
 
 
@@ -1633,6 +1633,13 @@ static void mainfunc (LexState *ls, FuncState *fs) {
 }
 
 
+static void compile_stripdebug(lua_State *L, Proto *f) {
+  int level =  G(L)->stripdefault;
+  if (level > 0)
+    luaU_stripdebug(L, f, level, 1);
+}
+
+
 LClosure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff,
                        Dyndata *dyd, const char *name, int firstchar) {
   LexState lexstate;
@@ -1654,6 +1661,7 @@ LClosure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff,
   lua_assert(!funcstate.prev && funcstate.nups == 1 && !lexstate.fs);
   /* all scopes should be correctly finished */
   lua_assert(dyd->actvar.n == 0 && dyd->gt.n == 0 && dyd->label.n == 0);
+  compile_stripdebug(L, funcstate.f);
   L->top--;  /* remove scanner's table */
   return cl;  /* closure is on the stack, too */
 }

app/lua53/lstate.c

@@ -307,7 +307,7 @@ void luaE_freethread (lua_State *L, lua_State *L1) {
   luaM_free(L, l);
 }
 
-LUA_API KeyCache *(lua_getcache) (int lineno) {
+LUAI_FUNC KeyCache *luaE_getcache (int lineno) {
   return &G(L0)->cache[lineno][0];
 }
 
@@ -347,6 +347,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
   g->gcfinnum = 0;
   g->gcpause = LUAI_GCPAUSE;
   g->gcstepmul = LUAI_GCMUL;
+  g->stripdefault = LUAI_OPTIMIZE_DEBUG;
   g->ROstrt.size = 0;
   g->ROstrt.nuse = 0;
   g->ROstrt.hash = NULL;

app/lua53/lstate.h

@@ -133,6 +133,11 @@ typedef struct CallInfo {
 #define setoah(st,v)	((st) = ((st) & ~CIST_OAH) | (v))
 #define getoah(st)	((st) & CIST_OAH)
 
+/*
+** KeyCache used for resolution of ROTable entries and Cstrings
+*/
+typedef size_t KeyCache;
+typedef KeyCache KeyCacheLine[KEYCACHE_M];
 
 /*
 ** 'global state', shared by all threads of this state
@@ -168,6 +173,7 @@ typedef struct global_State {
   unsigned int gcfinnum;  /* number of finalizers to call in each GC step */
   int gcpause;  /* size of pause between successive GCs */
   int gcstepmul;  /* GC 'granularity' */
+  int stripdefault;  /* default stripping level for compilation */
   l_mem gcmemfreeboard;  /* Free board which triggers EGC */
   lua_CFunction panic;  /* to be called in unprotected errors */
   struct lua_State *mainthread;
@@ -229,8 +235,6 @@ union GCUnion {
   struct Proto p;
   struct lua_State th;  /* thread */
 };
-
-
 #define cast_u(o)	cast(union GCUnion *, (o))
 
 /* macros to convert a GCObject into a specific value */
@@ -262,7 +266,6 @@ LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1);
 LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L);
 LUAI_FUNC void luaE_freeCI (lua_State *L);
 LUAI_FUNC void luaE_shrinkCI (lua_State *L);
-
+LUAI_FUNC KeyCache *luaE_getcache (int cl);
 
 #endif
-

app/lua53/lstrlib.c

@@ -23,7 +23,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 
 /*
@@ -189,11 +188,16 @@ static int writer (lua_State *L, const void *b, size_t size, void *B) {
 
 static int str_dump (lua_State *L) {
   luaL_Buffer b;
-  int strip = lua_tointeger(L, 2);
+  int strip, tstrip = lua_type(L, 2);
-  if (lua_isboolean(L, 2) && lua_toboolean(L, 2))
+  if (tstrip == LUA_TBOOLEAN) {
-    strip = 2;
+    strip = lua_toboolean(L, 2) ? 2 : 0;
+  } else if (tstrip == LUA_TNONE || tstrip == LUA_TNIL) {
+    strip = -1;  /* This tells lua_dump to use the global strip default */
+  } else { 
+    strip = lua_tointeger(L, 2);
+    luaL_argcheck(L, (unsigned)(strip) < 3, 2, "strip out of range");
+  }
   luaL_checktype(L, 1, LUA_TFUNCTION);
-  luaL_argcheck(L, 3 > (unsigned)(strip), 1, "strip out of range");
   lua_settop(L, 1);
   luaL_buffinit(L,&b);
   if (lua_dump(L, writer, &b, strip) != 0)
@@ -203,7 +207,6 @@ static int str_dump (lua_State *L) {
 }
 
 
-
 /*
 ** {======================================================
 ** PATTERN MATCHING

app/lua53/ltable.c

@@ -739,7 +739,7 @@ static const TValue* rotable_findentry(ROTable *t, TString *key, unsigned *ppos)
   const int tl = getlsizenode(t);
   const char *strkey = getstr(key);
   const int hash = HASH(t, key);
-  KeyCache *cl = lua_getcache(hash);
+  KeyCache *cl = luaE_getcache(hash);
   int i, j = 1, l;
 
   if (!e || gettt(key) != LUA_TSHRSTR)

app/lua53/ltablib.c

@@ -18,7 +18,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 
 /*

app/lua53/lua.c

@@ -238,9 +238,20 @@ static int pmain (lua_State *L) {
   * then attempting the open will trigger a file system format.
   */
   platform_rcr_read(PLATFORM_RCR_INITSTR, (void**) &init);
+  if (init[0] == '!') { /* !module is a compile-free way of executing LFS module */
+    luaL_pushlfsmodule(L);
+    lua_pushstring(L, init+1);
+    lua_call(L, 1, 1);  /* return LFS.module or nil */
+    status = LUA_OK;
+    if (!lua_isfunction(L, -1)) {
+      lua_pushfstring(L, "cannot load LFS.%s", init+1);
+      status = LUA_ERRRUN;
+    }
+  } else {
     status = (init[0] == '@') ?
              luaL_loadfile(L, init+1) :
              luaL_loadbuffer(L, init, strlen(init), "=INIT");
+  }
   if (status == LUA_OK)
     status = docall(L, 0);
   if (status != LUA_OK)

app/lua53/lua.h

@@ -153,6 +153,10 @@ LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf);
 
 LUA_API const lua_Number *(lua_version) (lua_State *L);
 
+#ifndef LUAI_OPTIMIZE_DEBUG
+#define LUAI_OPTIMIZE_DEBUG 2
+#endif
+
 
 /*
 ** basic stack manipulation
@@ -283,6 +287,7 @@ LUA_API int   (lua_load) (lua_State *L, lua_Reader reader, void *dt,
                           const char *chunkname, const char *mode);
 
 LUA_API int   (lua_dump) (lua_State *L, lua_Writer writer, void *data, int strip);
+LUA_API int   (lua_stripdebug) (lua_State *L, int stripping);
 
 
 /*
@@ -465,17 +470,17 @@ struct lua_Debug {
 
 typedef struct ROTable ROTable;
 typedef const struct ROTable_entry ROTable_entry;
-typedef size_t KeyCache;
 
 LUA_API void (lua_pushrotable) (lua_State *L, const ROTable *p);
 LUA_API void (lua_createrotable) (lua_State *L, ROTable *t, const ROTable_entry *e, ROTable *mt);
 LUA_API lua_State *(lua_getstate) (void);
-LUA_API KeyCache *(lua_getcache) (int cl);
+LUA_API int (lua_pushstringsarray) (lua_State *L, int opt);
-LUA_API int (lua_getstrings) (lua_State *L, int opt);
 LUA_API int (lua_freeheap) (void);
 
-LUAI_FUNC int  lua_lfsreload (lua_State *L);
+LUA_API void (lua_getlfsconfig) (lua_State *L, int *);
-LUAI_FUNC int  lua_lfsindex (lua_State *L);
+LUA_API int  (lua_pushlfsindex) (lua_State *L);
+LUA_API int  (lua_pushlfsfunc) (lua_State *L);
+
 
 #define luaN_freearray(L,a,l) luaM_freearray(L,a,l)
 
@@ -487,14 +492,12 @@ typedef struct Proto Proto;
 
 #ifdef DEVELOPMENT_USE_GDB
 LUALIB_API void (lua_debugbreak)(void);
-#define ASSERT(s) if (!(s)) {lua_debugbreak();}
+#define ASSERT(s)  ((s) ? (void)0 : lua_debugbreak())
 #else
 #define lua_debugbreak() (void)(0)
 #define ASSERT(s) (void)(0)
 #endif
 
-LUAI_FUNC int luaG_stripdebug (lua_State *L, Proto *f, int level, int recv);
-
 
 // from undump.c
 #define LUA_ERR_CC_INTOVERFLOW 101

app/lua53/lundump.h

@@ -48,6 +48,7 @@ LUAI_FUNC int luaU_DumpAllProtos(lua_State *L, const Proto *m, lua_Writer w,
                          void *data, int strip);
 
 LUAI_FUNC int luaU_undumpLFS(lua_State *L, ZIO *Z, int isabs);
+LUAI_FUNC int luaU_stripdebug (lua_State *L, Proto *f, int level, int recv);
 
 typedef struct FlashHeader LFSHeader;
 /*

app/lua53/lutf8lib.c

@@ -19,7 +19,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lnodemcu.h"
 
 #define MAXUNICODE	0x10FFFF
 
@@ -249,7 +248,7 @@ LROT_BEGIN(utf8_meta, NULL, LROT_MASK_INDEX)
 LROT_END(utf8_meta, NULL, LROT_MASK_INDEX)
 
 
-LROT_BEGIN(utf8, NULL, 0)
+LROT_BEGIN(utf8, LROT_TABLEREF(utf8_meta), 0)
   LROT_FUNCENTRY( offset, byteoffset )
   LROT_FUNCENTRY( codepoint, codepoint )
   LROT_FUNCENTRY( char, utfchar )

app/modules/node.c

@@ -1,14 +1,9 @@
 // Module for interfacing with system
 #include "module.h"
 #include "lauxlib.h"
-#include "lstate.h"
 #include "lmem.h"
 
-
 #include "platform.h"
-#if LUA_VERSION_NUM == 501
-#include "lflash.h"
-#endif
 #include <stdint.h>
 #include <string.h>
 #include "user_interface.h"
@@ -163,14 +158,27 @@ static void add_string_field( lua_State* L, const char *s, const char *name) {
   lua_setfield(L, -2, name);
 }
 
-static int node_info( lua_State* L )
+static void get_lfs_config ( lua_State* L ){
-{
+    int config[5];
-  const char* options[] = {"hw", "sw_version", "build_config", "legacy", NULL};
+    lua_getlfsconfig(L, config);
-  int option = luaL_checkoption (L, 1, options[3], options);
+    lua_createtable(L, 0, 4);
+    add_int_field(L, config[0], "lfs_mapped");
+    add_int_field(L, config[1], "lfs_base");
+    add_int_field(L, config[2], "lfs_size");
+    add_int_field(L, config[3], "lfs_used");
+}
+
+static int node_info( lua_State* L ){
+  const char* options[] = {"lfs", "hw", "sw_version", "build_config", "legacy", NULL};
+  int option = luaL_checkoption (L, 1, options[4], options);
 
   switch (option) {
-    case 0: { // hw
+    case 0: { // lfs
-      lua_createtable (L, 0, 5);
+      get_lfs_config(L);
+      return 1;
+    }
+    case 1: { // hw
+      lua_createtable(L, 0, 5);
       add_int_field(L, system_get_chip_id(),             "chip_id");
       add_int_field(L, spi_flash_get_id(),               "flash_id");
       add_int_field(L, flash_rom_get_size_byte() / 1024, "flash_size");
@@ -178,8 +186,8 @@ static int node_info( lua_State* L )
       add_int_field(L, flash_rom_get_speed(),            "flash_speed");
       return 1;
     }
-    case 1: { // sw_version
+    case 2: { // sw_version
-      lua_createtable (L, 0, 7);     
+      lua_createtable(L, 0, 7);     
       add_int_field(L, NODE_VERSION_MAJOR,       "node_version_major");
       add_int_field(L, NODE_VERSION_MINOR,       "node_version_minor");
       add_int_field(L, NODE_VERSION_REVISION,    "node_version_revision");
@@ -189,8 +197,8 @@ static int node_info( lua_State* L )
       add_string_field(L, BUILDINFO_RELEASE_DTS, "git_commit_dts");
       return 1;
     }
-    case 2: { // build_config
+    case 3: { // build_config
-      lua_createtable (L, 0, 4);
+      lua_createtable(L, 0, 4);
       lua_pushboolean(L, BUILDINFO_SSL);
       lua_setfield(L, -2, "ssl");
       lua_pushnumber(L, BUILDINFO_LFS_SIZE);
@@ -199,8 +207,7 @@ static int node_info( lua_State* L )
       add_string_field(L, BUILDINFO_BUILD_TYPE, "number_type");
       return 1;
     }
-    default:
+    default: { // legacy
-    {
       platform_print_deprecation_note("node.info() without parameter", "in the next version");
       lua_pushinteger(L, NODE_VERSION_MAJOR);
       lua_pushinteger(L, NODE_VERSION_MINOR);
@@ -333,8 +340,6 @@ static int writer(lua_State* L, const void* p, size_t size, void* u)
 }
 
 #if LUA_VERSION_NUM == 501
-#undef lua_dump
-#define lua_dump lua_dumpEx
 #define getproto(o)	(clvalue(o)->l.p)
 #endif
 
@@ -461,8 +466,6 @@ static int node_restore (lua_State *L)
   return 0;
 }
 
-#if defined(LUA_OPTIMIZE_DEBUG) && LUA_VERSION_NUM == 501
-
 /* node.stripdebug([level[, function]]). 
  * level:    1 don't discard debug
  *           2 discard Local and Upvalue debug info
@@ -472,54 +475,36 @@ static int node_restore (lua_State *L)
  * If function is omitted, this is the default setting for future compiles
  * The function returns an estimated integer count of the bytes stripped.
  */
-LUA_API int luaG_stripdebug (lua_State *L, Proto *f, int level, int recv);
 static int node_stripdebug (lua_State *L) {
 
   int n = lua_gettop(L);
-  if (n == 0) {
+  int strip = 0;
-    lua_pushlightuserdata(L, &luaG_stripdebug );
-    lua_gettable(L, LUA_REGISTRYINDEX);
-    if (lua_isnil(L, -1)) {
-      lua_pop(L, 1);
-      lua_pushinteger(L, LUA_OPTIMIZE_DEBUG);
-    }
-    return 1;
-  }
 
-  int level = luaL_checkint(L, 1);
+  lua_settop(L, 2);
-  luaL_argcheck(L, level > 0 && level < 4, 1, "must in range 1-3");
+  if (!lua_isnil(L, 1)) {
-
+    strip = lua_tointeger(L, 1);
-  if (n == 1) {
+    luaL_argcheck(L, strip > 0 && strip < 4, 1, "Invalid strip level");
-    /* Store the default level in the registry if no function parameter */
-    lua_pushlightuserdata(L, &luaG_stripdebug);
-    lua_pushinteger(L, level);
-    lua_settable(L, LUA_REGISTRYINDEX);
-    return 0;
-  }
-
-  if (level == 1) {
-    lua_pushinteger(L, 0);
-    return 1;
   } 
 
   if (lua_isnumber(L, 2)) {
-    int scope = luaL_checkint(L, 2);
+    /* Use debug interface to replace stack level by corresponding function */
+    int scope = luaL_checkinteger(L, 2);
     if (scope > 0) { 
-      /* if the function parameter is a +ve integer then climb to find function */
       lua_Debug ar;
-      lua_pop(L, 1); /* pop level as getinfo will replace it by the function */
+      lua_pop(L, 1);
       if (lua_getstack(L, scope, &ar)) {
-        lua_getinfo(L, "f", &ar);
+        lua_getinfo(L, "f", &ar);  /* put function at [2] (ToS) */
       }
     }
   }
  
-  luaL_argcheck(L, lua_isfunction(L, 2), 2, "must be a Lua Function");
+  int isfunc = lua_isfunction(L, 2);
-  Proto *f = getproto(L->ci->func + 1);
+  luaL_argcheck(L, n < 2 || isfunc, 2, "not a valid function");
-  lua_pushinteger(L, luaG_stripdebug(L, f, level, 1));
+
+  /* return result of lua_stripdebug, adding 1 if this is get/set level) */ 
+  lua_pushinteger(L, lua_stripdebug(L, strip - 1) + (isfunc ? 0 : 1));
   return 1;
 }
-#endif
 
 
 #if LUA_VERSION_NUM == 501
@@ -539,9 +524,10 @@ static int node_egc_setmode(lua_State* L) {
 }
 // totalallocated, estimatedused = node.egc.meminfo()
 static int node_egc_meminfo(lua_State *L) {
-  global_State *g = G(L);
+  int totals[2];
-  lua_pushinteger(L, g->totalbytes);
+  lua_getegcinfo(L, totals);
-  lua_pushinteger(L, g->estimate);
+  lua_pushinteger(L, totals[0]);
+  lua_pushinteger(L, totals[1]);
   return 2;
 }
 #endif
@@ -651,6 +637,64 @@ static int node_writercr (lua_State *L) {
 }
 #endif
 
+// Lua: n = node.LFS.reload(lfsimage)
+static int node_lfsreload (lua_State *L) {
+  lua_settop(L, 1);
+  luaL_lfsreload(L);
+  return 1;
+}
+
+// Lua: n = node.flashindex(module)
+// Lua: n = node.LFS.get(module)
+static int node_lfsindex (lua_State *L) {
+  lua_settop(L, 1);
+  luaL_pushlfsmodule(L);
+  return 1;
+}
+
+// Lua: n = node.LFS.list([option])
+// Note that option is ignored in this release
+static int node_lfslist (lua_State *L) {
+  lua_settop(L, 1);
+  luaL_pushlfsmodules(L);
+  if (lua_istable(L, -1) && lua_getglobal(L, "table") == LUA_TTABLE) {
+    lua_getfield(L, -1, "sort");
+    lua_remove(L, -2);       /* remove table table */
+    lua_pushvalue(L, -2);    /* dup array of modules ref to ToS */
+    lua_call(L, 1, 0);
+  }
+  return 1;
+}
+
+//== node.LFS Table emulator ==============================================//
+
+static int node_lfs_func (lua_State* L) {      /*T[1] = LFS, T[2] = fieldname */
+  lua_remove(L, 1);
+  lua_settop(L, 1);
+  const char *name = lua_tostring(L, 1);
+  if (!name) {
+    lua_pushnil(L);
+  } else if (!strcmp(name, "config")) {
+    get_lfs_config(L);
+  } else if (!strcmp(name, "time")) {
+    luaL_pushlfsdts(L);
+  } else {
+    luaL_pushlfsmodule(L);
+  }
+  return 1;
+}
+
+LROT_BEGIN(node_lfs_meta, NULL, LROT_MASK_INDEX)
+  LROT_FUNCENTRY( __index, node_lfs_func)
+LROT_END(node_lfs_meta, NULL, LROT_MASK_INDEX)
+
+LROT_BEGIN(node_lfs, LROT_TABLEREF(node_lfs_meta), 0)
+  LROT_FUNCENTRY( list, node_lfslist)
+  LROT_FUNCENTRY( get, node_lfsindex)
+  LROT_FUNCENTRY( reload, node_lfsreload )
+LROT_END(node_lfs, LROT_TABLEREF(node_lfs_meta), 0)
+
+
 typedef enum pt_t { lfs_addr=0, lfs_size, spiffs_addr, spiffs_size, max_pt} pt_t;
 
 LROT_BEGIN(pt_map, NULL, 0)
@@ -826,8 +870,8 @@ LROT_BEGIN(node, NULL, 0)
   LROT_FUNCENTRY( heap, node_heap )
   LROT_FUNCENTRY( info, node_info )
   LROT_TABENTRY( task, node_task )
-  LROT_FUNCENTRY( flashreload, lua_lfsreload )
+  LROT_FUNCENTRY( flashindex, node_lfsindex )
-  LROT_FUNCENTRY( flashindex, lua_lfsindex )
+  LROT_TABENTRY( LFS, node_lfs )
   LROT_FUNCENTRY( setonerror, node_setonerror )
   LROT_FUNCENTRY( startupcommand, node_startupcommand )
   LROT_FUNCENTRY( restart, node_restart )
@@ -856,9 +900,7 @@ LROT_BEGIN(node, NULL, 0)
   LROT_FUNCENTRY( bootreason, node_bootreason )
   LROT_FUNCENTRY( restore, node_restore )
   LROT_FUNCENTRY( random, node_random )
-#if LUA_VERSION_NUM == 501 && defined(LUA_OPTIMIZE_DEBUG)
   LROT_FUNCENTRY( stripdebug, node_stripdebug )
-#endif
 #if LUA_VERSION_NUM == 501
   LROT_TABENTRY( egc, node_egc )
 #endif

app/modules/sjson.c

@@ -1,7 +1,6 @@
 #define LUA_LIB
 
 #include "lauxlib.h"
-#include "lstring.h"
 
 #ifndef LOCAL_LUA
 #include "module.h"

app/platform/platform.c

@@ -5,7 +5,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
-#include "llimits.h"
+#include <stdint.h>
 #include "gpio.h"
 #include "user_interface.h"
 #include "driver/gpio16.h"
@@ -215,7 +215,7 @@ static void ICACHE_RAM_ATTR platform_gpio_intr_dispatcher (void *dummy){
   uint32_t j=0;
   uint32_t gpio_status = GPIO_REG_READ(GPIO_STATUS_ADDRESS);
   uint32_t now = system_get_time();
-  UNUSED(dummy);
+  (void)(dummy);
 
 #ifdef GPIO_INTERRUPT_HOOK_ENABLE
   if (gpio_status & platform_gpio_hook->all_bits) {
@@ -986,13 +986,13 @@ uint32_t platform_rcr_read (uint8_t rec_id, void **rec) {
 }
 
 uint32_t platform_rcr_delete (uint8_t rec_id) {
-  void *rec = NULL;
+  uint32_t *rec = NULL;
-  platform_rcr_read (rec_id, &rec);
+  platform_rcr_read(rec_id, (void**)&rec);
   if (rec) {
-    uint32_t *pHdr = cast(uint32_t *,rec)-1;
+    uint32_t *pHdr = rec - 1;  /* the header is the word proceeding the rec */ 
     platform_rcr_t hdr = {.hdr = *pHdr};
     hdr.id = PLATFORM_RCR_DELETED;
-    platform_s_flash_write(&hdr, platform_flash_mapped2phys(cast(uint32_t, pHdr)), WORDSIZE);
+    platform_s_flash_write(&hdr, platform_flash_mapped2phys((uint32_t) pHdr), WORDSIZE);
     return 0;
   }
   return ~0;

app/websocket/websocketclient.c

@@ -26,7 +26,6 @@
 #include "user_interface.h"
 #include "espconn.h"
 #include "mem.h"
-#include "limits.h"
 #include <stdint.h>
 #include <stddef.h>
 #include <string.h>

app/websocket/websocketclient.h

@@ -29,8 +29,8 @@
 #include "user_interface.h"
 #include "espconn.h"
 #include "mem.h"
-#include "limits.h"
+#include <limits.h>
-#include "stdlib.h"
+#include <stdlib.h>
 
 #if defined(USES_SDK_BEFORE_V140)
 #define espconn_send espconn_sent

docs/flash.md

@@ -75,48 +75,15 @@ Formatting a file system on a large flash device (e.g. the 16MB parts) can take
 
 ## SDK Init Data
 
-!!! note
+NodeMCU versions are compiled against specific versions of the Espressif SDK. The SDK reserves space in flash that is used to store calibration and other data. Espressif refers to this area as "System Param" and it occupies four 4 Kb sectors of flash. A fifth 4 Kb sector is also reserved for RF calibration.
+-  With SDK version 2.x builds, these 5 sectors are located in the last pages at in the Flash memory.
+-  With SDK version 3.x builds, these 5 sectors are located in the otherwise unused pages at Flash offset 0x0B000-0x0FFFF, between the `bin/0x00000.bin` segment at 0x00000 and the `bin/0x10000.bin` to 0x10000.
 
-    Normally, NodeMCU will take care of writing the SDK init data when needed. Most users can ignore this section.
+If this data gets corrupted or you are upgrading major SDK versions, then the firmware may not boot correctly. Symptoms include messages like `rf_cal[0] !=0x05,is 0xFF`, or endless reboot loops and/or fast blinking module LEDs.  If you are seeing one or several of the above symptoms, ensure that your chip is fully erased before flashing, for example by using `esptool.py`.  The SDK version 3.x firmware builds detect if the RF calibration sector has been erased or corrupted, and will automatically initialise it with the correct content before restarting the processor. This works for all SDK supported flash sizes.
-
-NodeMCU versions are compiled against specific versions of the Espressif SDK. The SDK reserves space in flash that is used to store calibration and other data. This data changes between SDK versions, and if it is invalid or not present, the firmware may not boot correctly. Symptoms include messages like `rf_cal[0] !=0x05,is 0xFF`, or endless reboot loops and/or fast blinking module LEDs.
-
-!!! tip
-
-    If you are seeing one or several of the above symptoms, ensure that your chip is fully erased before flashing, for example:
-
-    `esptool.py --port <serial-port-of-ESP8266> erase_flash`
-
-    Also verify that you are using an up-to-date NodeMCU release, as some early releases of NodeMCU 1.5.4.1 did not write the SDK init data to a freshly erased chip.
-
-Espressif refers to this area as "System Param" and it resides in the last four 4&nbsp;kB sectors of flash. Since SDK 1.5.4.1 a fifth sector is reserved for RF calibration (and its placement is controlled by NodeMCU) as described by this [patch notice](http://bbs.espressif.com/viewtopic.php?f=46&t=2407). At minimum, Espressif states that the 4th sector from the end needs to be flashed with "init data", and the 2nd sector from the end should be blank.
-
-The default init data is provided as part of the SDK in the file `esp_init_data_default.bin`. NodeMCU will automatically flash this file to the right place on first boot if the sector appears to be empty.
-
-If you need to customize init data then first download the [Espressif SDK 2.2.0](https://github.com/espressif/ESP8266_NONOS_SDK/archive/v2.2.0.zip) and extract `esp_init_data_default.bin`. Then flash that file just like you'd flash the firmware. The correct address for the init data depends on the capacity of the flash chip.
-
-- `0x7c000` for 512 kB, modules like most ESP-01, -03, -07 etc.
-- `0xfc000` for 1 MB, modules like ESP8285, PSF-A85, some ESP-01, -03 etc.
-- `0x1fc000` for 2 MB
-- `0x3fc000` for 4 MB, modules like ESP-12E, NodeMCU devkit 1.0, WeMos D1 mini
-- `0x7fc000` for 8 MB
-- `0xffc000` for 16 MB, modules like WeMos D1 mini pro
-
-See "4.1 Non-FOTA Flash Map" and "6.3 RF Initialization Configuration" of the [ESP8266 Getting Started Guide](https://www.espressif.com/sites/default/files/documentation/2a-esp8266-sdk_getting_started_guide_en.pdf) for details on init data addresses and customization.
 
 ## Determine flash size
 
-To determine the capacity of the flash chip *before* a firmware is installed you can run
+The easiest way to determine the flash capacity is to load the firmware and then `print(node.info'hw'.flash_size)` which reports the flash size in Kb.  Alternatively, if you want to determine the capacity of the flash chip _before_ a firmware is installed then you can run the following command.  This will return a 2 hex digit **Manufacturer** ID and a 4 digit **Device** ID and the detected flash size.
 
 `esptool.py --port <serial-port> flash_id`
-
+The chip ID can then be looked up in [https://review.coreboot.org/cgit/flashrom.git/tree/flashchips.h](https://review.coreboot.org/cgit/flashrom.git/tree/flashchips.h).
-It will return a manufacturer ID and a chip ID like so:
-
-```
-Connecting...
-Manufacturer: e0
-Device: 4016
-```
-The chip ID can then be looked up in [https://review.coreboot.org/cgit/flashrom.git/tree/flashchips.h](https://review.coreboot.org/cgit/flashrom.git/tree/flashchips.h). This leads to a manufacturer name and a chip model name/number e.g. `AMIC_A25LQ032`. That information can then be fed into your favorite search engine to find chip descriptions and data sheets.
-
-By convention the last two or three digits in the module name denote the capacity in megabits. So, `A25LQ032` in the example above is a 32Mb(=4MB) module.

docs/lfs.md

@@ -1,34 +1,179 @@
 # Lua Flash Store (LFS)
 
+!!! note "Work in Progress"
+    The next PR will add extra LFS functionality and require some major changes to this document. Hence we will be updating this document in the next few weeks.
+
+## Some Acronym Definitions
+
+Before I begin, I think it useful to define some acronyms in the context used in this paper.  There are others like [IoT](https://en.wikipedia.org/wiki/Internet_of_things) which you might know and on first use are hyperlinked to an external (typically Wikipedia) definition.
+
+Acronym | What it means in this paper
+-----|--------
+**LFS** | _Lua Flash Store_, and if you want to understand more then read on.
+**RTS** | The Lua environment on the ESP chipset include a core [Runtime System](https://en.wikipedia.org/wiki/Runtime_system) includes a [VM](https://en.wikipedia.org/wiki/Virtual_machine) that executes compiled Lua code, handle resource management, and provides and [API](https://en.wikipedia.org/wiki/Application_programming_interface) for the integration of library modules.
+**ESP** | [Espressif](https://www.espressif.com/) Systems Processor.  A range of [SoC](https://en.wikipedia.org/wiki/System_on_a_chip) devices based on an [Xtensa core](https://en.wikipedia.org/wiki/Tensilica#Xtensa_configurable_cores).  NodeMCU currently offers Lua firmware builds for its ESP8266 and ESP32 families.
+**RAM** | [Random Access Memory](https://en.wikipedia.org/wiki/Random-access_memory) that can be directly read or written by the [CPU](https://en.wikipedia.org/wiki/Central_processing_unit) and can be used to hold both data and machine code. 
+**ROM** | [Read Only Memory](https://en.wikipedia.org/wiki/Read-only_memory), though for the purposes of this paper, this also includes a region of [Flash memory](https://en.wikipedia.org/wiki/Flash_memory).
+**GC** | [Garbage Collection](https://www.lua.org/manual/5.3/manual.html#2.5) refers to the automatic management resources to collect dead objects and make any associated memory available for reuse.
+
 ## Background
 
 Lua was originally designed as a general purpose embedded extension language for use in applications run on a conventional computer such as a PC, where the processor is mounted on a motherboard together with multiple Gb of RAM and a lot of other chips providing CPU and I/O support to connect to other devices.
 
-ESP8266 modules are on a very different scale: they cost a few dollars; they are postage stamp-sized and only mount two main components, an ESP [SoC](https://en.wikipedia.org/wiki/System_on_a_chip) and a flash memory chip.  The SoC includes limited on-chip RAM, but also provides hardware support to map part of the external flash memory into a separate memory address region so that firmware can be executed directly out of this flash memory — a type of [modified Harvard architecture](https://en.wikipedia.org/wiki/Modified_Harvard_architecture) found on many [IoT](https://en.wikipedia.org/wiki/Internet_of_things) devices.  Even so, Lua's design goals of speed, portability, small kernel size, extensibility and ease-of-use have made it a good choice for embedded use on an IoT platform, but with one major limitaton: the standard Lua core runtime system (**RTS**) assumes that both Lua data _and_ code are stored in RAM; this isn't a material constraint with a conventional computer, but it can be if your system only has some 48Kb RAM available for application use.
+An ESP8266 module is on a very different scale: it costs a few dollars; it is postage stamp-sized and only mounts two main components, an ESP SoC and a flash memory chip.  The SoC includes on-chip RAM and also provides hardware support to map part of the external flash memory into a ROM addressable region so that the [firmware](https://en.wikipedia.org/wiki/Firmware) code can be executed via an [L1 cache](https://en.wikipedia.org/wiki/CPU_cache) out of this memory.
 
-The Lua Flash Store (**LFS**) patch modifies the Lua RTS to support a modified Harvard architecture by allowing the Lua code and its associated constant data to be executed directly out of flash-memory (just as the NoceMCU firmware is itself executed). This now allows NodeMCU Lua developers to create Lua applications with up to 256Kb Lua code and read-only (**RO**) constants executing out of flash, with all of the RAM is available for read-write (**RW**) data.
+ESP SoCs also adopt a type of [modified Harvard architecture](https://en.wikipedia.org/wiki/Modified_Harvard_architecture) found on many IoT devices where separate address regions are used for RAM code and RAM data and ROM code.  ESPs also allow data constants to be loaded from code memory.  The ESP8266 has 96 Kb of data RAM, but half of this is used by the operating system, for stack and for device drivers such as for WiFi support;  typically **44 Kb** RAM is available as heap space for embedded applications. By contrast, the mapped flash ROM region can be up to **960 Kb**, that is over twenty times larger.  Even though flash ROM is read-only for normal execution, there is also a "back-door" file-like API for erasing flash pages and overwriting them (though some care has to be taken to ensure cache integrity after update). 
 
-Unfortunately, the ESP architecture provides very restricted write operations to flash memory (writing to NAND flash involves bulk erasing complete 4Kb memory pages, before overwriting each erased page with any new content).  Whilst it is possible to develop a R/W file system within this constraint (as SPIFFS demonstrates), this makes impractical to modify Lua code pages on the fly. Hence the LFS patch works within a reflash-and-restart paradigm for reloading the LFS, and does this by adding two API new calls: one to reflash the LFS and restart the processor, and one to access LFS stored functions.  The patch also addresses all of the technical issues 'under the hood' to make this magic happen.
+Lua's design goals of speed, portability, small kernel size, extensibility and ease-of-use make it a good choice for embedded use on an IoT platform, but with one major limitation: the standard Lua RTS assumes that both Lua data _and_ code are stored in RAM, and this is a material constraint on a device with perhaps 44Kb free RAM and 512Kb free program ROM.
 
-The remainder of this paper is for those who want to understand a little of how this magic happens, and gives more details on the technical issues that were addressed in order to implement the patch.
+The LFS feature modifies the Lua RTS to support a modified Harvard architecture by allowing the Lua code and its associated constant data to be executed directly out of flash ROM (just as the NoceMCU firmware is itself executed). This enables NodeMCU Lua developers to create Lua applications with a region of flash ROM allocated to Lua code and read-only constants. The entire RAM heap is then available for Lua read-write variables and data for applications where all Lua is executed from LFS.
 
-If you're just interested in learning how to quickly get started with LFS then please read the respective chapters in the [Getting Started](./getting-started.md) overview.
+The ESP architecture provides very restricted write operations to flash memory: any updates use [NAND flash rules](https://en.wikipedia.org/wiki/Flash_memory#Writing_and_erasing), so any updates typically first require bulk erasing of complete 4Kb memory pages. This makes it impractical to modify Lua code pages on the fly safely, and hence LFS uses a "reflash and restart" paradigm for reloading the LFS region.
 
-## Using LFS
+If you are just interested in learning how to quickly get started with LFS, then please read the chapters referenced in the [Getting Started](getting-started.md) overview.  The remainder of this paper is for those who want to understand a little of how this magic happens, and gives more details on the technical issues that were addressed in order to implement this feature in the following sections:
 
-### Selecting the firmware
+-  [Configuring LFS](#configuring-lfs)
+-  [An Overview of LFS Internals](#an-overview-of-lfs-internals)
+-  [Programming Techniques](#programming-techniques)
+-  [Compiling and Loading LFS Images](#compiling-and-loading-lfs-images)
 
-Power developers might want to use Docker or their own build environment as per our [Building the firmware](https://nodemcu.readthedocs.io/en/master/en/build/) documentation, and so `app/include/user_config.h` has now been updated to include the necessary documentation on how to select the configuration options to make an LFS firmware build.
+## Configuring LFS
 
-However, most Lua developers seem to prefer the convenience of our [Cloud Build Service](https://nodemcu-build.com/), so we have added extra LFS menu options to facilitate building LFS images:
+The LFS region can be allocated during firmware build:
 
-Variable | Option
+-  For Lua developers that prefer the convenience of our [Cloud Build Service](https://nodemcu-build.com/), the menu dialogue offers a range of drop down options to select the size of LFS region required.
----------|------------
+
-LFS size | (none, 32, 64, 96, 128 or 256Kb) The default is none. The default is none, in which case LFS is disabled. Selecting a numeric value enables LFS with the LFS region sized at this value.
+-  Some advanced developers might want to use [Docker](https://www.docker.com/) or their own build environment as per our [Building the firmware](build.md) documentation, and in this case the file [`app/include/user_config.h`](../app/include/user_config.h) is used and includes inline documentation on how to select the configuration options to make an LFS firmware build.
-SPIFFS base | If you have a 4Mb flash module then I suggest you choose the 1024Kb option as this will preserve the SPIFFS even if you reflash with a larger firmware image; otherwise leave this at the default 0.
+
-SPIFFS size | (default or various multiples of 64Kb) Choose the size that you need.  Larger FS require more time to format on first boot.
+You also have the option to reconfigure the LFS and SPIFFS regions at any time (typically after restart).  See the function [`node.getpartitiontable()`](modules/node/#nodegetpartitiontable), so for example executing the following on one of my test modules:
+```Lua
+do
+  local s,p={},node.getpartitiontable()
+  for _,k in ipairs{'lfs_addr','lfs_size','spiffs_addr','spiffs_size'} do
+    s[#s+1] ='%s = 0x%06x' % {k, p[k]}
+  end
+  print ('{ %s }' % table.concat(s,', '))
+end
+```
+prints out:
+```Lua
+{ lfs_addr = 0x096000, lfs_size = 0x010000, spiffs_addr = 0x100000, spiffs_size = 0x100000 }
+```
+which just the format of the array parameter for  [`node.setpartitiontable()`](modules/node/#nodesetpartitiontable).  By default the LFS region starts immediately after the firmware so bringing this forward can cause errors. Likewise I usually start my SPIFFS on the 1Mb boundary and only allocate what I think that I need as having a larger SPIFFS partition slow I/O performance.  However if I want to increase the partition, say to 128Kb then I can just patch this into a set partition command:
+```Lua
+> node.setpartitiontable{ lfs_addr = 0x096000, lfs_size = 0x020000, 
+>> spiffs_addr = 0x100000, spiffs_size = 0x100000 }
+```
+restarts the CPU and after startup, executing the above `do` `end` fragment returns:
+```Lua
+{ lfs_addr = 0x096000, lfs_size = 0x020000, spiffs_addr = 0x100000, spiffs_size = 0x100000 }
+```
+Job done.
+ 
+## An Overview of LFS Internals
+
+### LFS Internal Structure
+
+Whilst memory capacity isn't a material constraint on most conventional machines, the Lua RTS still includes some features to minimise overall memory usage. In particular:
+
+-  The more resource intensive data types are know as _collectable objects_, and the RTS includes a GC which regularly scans these collectable resources to determine which are no longer in use, so that their associated memory can be reclaimed and reused.
+
+-  The Lua RTS also treats strings and compiled function code as collectable objects, so that these can also be GCed when no longer referenced
+
+The compiled Lua code which is executed by the RTS internally comprises one or more function prototypes (which use a `Proto` structure type) plus their associated vectors (constants, instructions and meta data for debug). Most of these compiled constant types are basic (e.g. numbers); strings are the only collectable constant data type. Other collectable types such as arrays are actually created at runtime by executing Lua compiled code to build each resource dynamically.
+
+Overlay techniques can be used ensure that active functions are loaded into RAM on a just-in time basis and thus mitigate RAM limitations. Moving Lua "program" resources into flash ROM typically at least doubles the effective RAM available, and typically removes any need to complicate applications code by implementing overlaying.
+
+When any Lua file is loaded (without LFS) into an ESP application, the RTS loads the corresponding compiled version into RAM. Each compiled function has an associated own Proto structure hierarchy, but this hierarchy is not exposed directly to the running application; instead the compiler generates `CLOSURE` instruction which is executed at runtime to bind the Proto to a Lua function value thus creating a [closure](https://en.wikipedia.org/wiki/Closure_(computer_programming)). Since this occurs at runtime, any Proto can be bound to multiple closures. A Lua closure can also have multiple [Upvalues](https://www.lua.org/manual/5.3/manual.html#3.5) bound to it, and so function value is an updatable object in that it is referring to something that can contain writeable state, even though the Proto hierarchy itself is intrinsically read-only.
+
+As read-only resources that are store in flash ROM clearly cannot be collected by the GC, such ROM based resources cannot reference any volatile RAM-based data elements, though the converse can apply: updatable resources in RAM can reference read-only ones in ROM.
+
+All strings are stored internally in Lua with a header structure known as a `TString`, In Lua 5.1, all TStrings were [interned](https://en.wikipedia.org/wiki/String_interning), so that only one copy of any string is kept in memory, and most string manipulation uses the address of this single copy as a unique reference.  Lua 5.3 divides strings into **Short** and **Long** subtypes with short strings being handled in the same way as Lua 5.1. In contrast, long strings are created and copied by reference, but are _not_ guaranteed to be stored uniquely.  Guaranteeing uniqueness requires the string to be hashed and this can have a large runtime overhead for long strings, yet identical long strings are rarely generated by other than by copy-reference; hence the general runtime savings for not hashing long strings exceed the small chance of storage duplication.
+
+All of this complexity is hidden from running Lua applications, but does impact our LFS implementation.  The Lua global state links to two (short) string tables: `ROstrt` in LFS for short strings stored in LFS; and `strt` for short strings in RAM. Maintaining integrity across these two string tables at runtime is simple and low-cost, with LFS resolution process extended across both the RAM and ROM string tables. Hence any strings already in the ROM string table can be reused and this avoids the need to add an additional entry in the RAM table. This significantly reduces the size of the RAM string table, and also removes a lot of strings from the LCG scanning.
+
+Lua GC of both types is essentially the same (and skipped for LFS strings) except that collection of long strings does not need to update the `strt`.
+
+LFS internal layouts are different for our Lua 5.1 and Lua 5.3 version execution environments, with the Lua 5.3 version:
+
+-  Reflecting the changes to the Lua core that support the new Lua 5.2 and 5.3 language features (such the two string subtypes);
+-  Facilitating the additions of a second LFS region to allow separate System and Application LFS regions;  this will also add a second `RO2strt`;
+-  Facilitates the on-ESP building of LFS regions at runtime, thus optionally removing the need for host-based code compilation;
+-  Improving the scalability of LFS function resolution.
+ 
+Any Lua file compiled into the LFS image includes its main function prototype and all the child resources that are linked in its `Proto` structure, so all of these resources are compiled into the LFS image with this entire hierarchy self-consistently within the flash memory:
+
+-  A TString reference to the name of the function
+-  The vector of Lua VM instructions used to execute the function codes
+-  (optional) A packed map of instruction offset -> line number used for error tracebacks
+-  A vector of constants used by the function
+-  (optional) metadata on local variables
+-  metadata on local variables (some fields optional)
+-  a vector of references to other Protos compiled within the current one.
+
+The optional fields are include or not depending on the [stripdebug](modules/node.md#nodestripdebug) setting at the time of compilation. 
+
+Lua 5.1 LFS images include a (hidden) Lua index function which has an `if n == "moduleN" then return moduleN end` chain to resolve functions within the LFS.  This as been replaced by a [ROTable](nodemcu-lrm.md#values-and-types) in Lua 5.3 significantly reducing lookup times for larger LFS images.
+
+Lua uses a binary tokenised format for dumping compiled Lua code into a file based "compiled" format.  Unlike the loading of Lua source which involves on-demand compilation at runtime, "undumping" compiled Lua code is the reverse operation to "dump" and is 5-10× faster.
+
+With Lua 5.3 LFS image file formats are a minor extension to the dump format and loading the image file into an LFS is a variant of the "undump" process (which shares the same internal functions).
+
+### Impact on the Lua Garbage Collector
+
+The GC applies to what the Lua VM classifies as collectable objects (strings, tables, functions, userdata, threads -- known collectively as `GCObjects`). A simple two "colour" GC was used in previous Lua versions, but Lua 5.1 introduced the Dijkstra's 3-colour (*white*, *grey*, *black*) variant that enabled the GC to operate in an incremental mode. This permits smaller GC steps interspersed by pauses, and this is very useful for larger scale Lua implementations. Even though this incremental mode is less useful for small RAM IoT devices, NodeMCU retains this standard Lua implementation.
+
+In fact, two *white* flavours are used to support incremental working (so this 3-colour algorithm really uses 4). All newly allocated collectable objects are marked as the current *white*, and a link in `GCObject` header enables scanning through all such Lua objects. Collectable objects can be referenced directly or indirectly via one of the Lua application's *roots*: .
+
+The standard GC algorithm is quite complex and assumes that all GCObjects are in RAM and updatable. It operates in two broad phases: **mark** and **sweep** where the mark phase does a recursive walk starting at the GC _roots_ (the global environment, the Lua registry and the stack) and marks the collectable objects in use.  This process is complicated by the fact that the collector is incremental, that is its processing can be broken into packets of collection that can be interleaved within normal memory allocation actions.  Once a mark phase has been completed, the sweep phase chains down the linked list of GCobjects to detect unmarked objects and reclaim them.  Because LFS ROM GCObjects are treated as immutable, GC processing can still maintain overall object integrity, with the LFS modifications preventing the marking updates to ROM GCObjects (since any attempt to update any ROM-base structure during GC will result in the firmware crashing with a memory exception).
+
+There are all sorts of nuances needed because of the incremental nature and to allow Lua finalizers to take an active role in collection, and these finalizers can themselves trigger allocation actions.  Not for the faint hearted.  However, ROM GCOjects can still be handled robustly within this scheme because:
+-  Whilst a RAM object can refer to a ROM object, the converse in _not_true: a ROM object can never refer to a RAM one.  Hence the recursive mark phase can safe abort its recursive walk at any node when a ROM object is detected. 
+-  ROM objects are in a separate linked list to that used by the sweep process on RAM objects and so are never swept.
+
+Lastly note that the cost of GC is directly related to the size of the GC sweep lists.  Therefore moving resources into LFS ROM removes them from the GC scope, and therefore reduces GC runtime accordingly.
+
+
+### Programming Techniques
+
+A number of developers have commented that moving applications into LFS simplifies coding style, as you need to be far less concerned about dynamic loading of code just-in-time to mitigate RAM use, and source modules can be larger if that suits application structure.  This makes it easier to adopt a clearer coding style, so ESP Lua code can now looks more similar to host-based Lua development.  Also Lua code can still be loaded from SPIFFS, so you still have the option to keep test code in SPIFFS, and only move modules into LFS once they are stable.
+
+#### Accessing LFS functions and loading LFS modules
+
+The main interface to LFS is encapsulated in the `node.LFS` table.
+See [`lua_examples/lfs/_init.lua`](../lua_examples/lfs/_init.lua) for the code that I use in my `_init` module to do create a simple access API for LFS.  There are two parts to this.
+
+The first sets up a table in the global variable `LFS` with the `__index` and `__newindex` metamethods. The main purpose of the `__index()` is to resolve any names against the LFS using a `node.flashindex()` call, so that `LFS.someFunc(params)` does exactly what you would expect it to do: this will call `someFunc` with the specified parameters, if it exists in in the LFS.  The LFS properties `_time`, `_config` and `_list` can be used to access the other LFS metadata that you need.  See the code to understand what they do, but `LFS._list` is the array of all module names in the LFS.  The `__newindex` method makes `LFS` readonly.
+
+The second part uses standard Lua functionality to add the LFS to the require [package.loaders](http://pgl.yoyo.org/luai/i/package.loaders) list. (Read the link if you want more detail).  There are four standard loaders, which the require loader searches in turn.  NodeMCU only uses the second of these (the Lua loader from the file system), and since loaders 1,3 and 4 aren't used, we can simply replace the 1st or the 3rd by code to use `node.flashindex()` to return the LFS module.  The supplied `_init` puts the LFS loader at entry 3, so if the module is in both SPIFFS and LFS, then the SPIFFS version will be loaded. One result of this has burnt me during development: if there is an out of date version in SPIFFS, then it will still get loaded instead of the one if LFS.
+
+If you want to swap this search order so that the LFS is searched first, then SET `package.loaders[1] = loader_flash` in your `_init` code.  If you need to swap the search order temporarily for development or debugging, then do this after you've run the `_init` code:
+
+```Lua
+do local pl = package.loaders; pl[1],pl[3] = pl[3],pl[1]; end
+```
+
+#### Moving common string constants into LFS
+
+LFS is mainly used to store compiled modules, but it also includes its own string table and any strings loaded into this can be used in your Lua application without taking any space in RAM. Hence, you might also want to preload any other frequently used strings into LFS as this will both save RAM use and reduced the Lua Garbage Collector (**GC**) overheads.
+
+The new debug function `debug.getstrings()` can help you determine what strings are worth adding to LFS.  It takes an optional string argument `'RAM'` (the default) or `'ROM'`, and returns a list of the strings in the corresponding table. So the following example can be used to get a listing of the strings in RAM.
+```Lua
+do
+  local a=debug.getstrings'RAM'
+  for i =1, #a do a[i] = ('%q'):format(a[i]) end
+  print ('local preload='..table.concat(a,','))
+end
+```
+
+You can do this at the interactive prompt or call it as a debug function during a running application in order to generate this string list, (but note that calling this still creates the overhead of an array in RAM, so you do need to have enough "head room" to do the call).
+
+You can then create a file, say `LFS_dummy_strings.lua`, and insert these `local preload` lines into it.  By including this file in your `luac.cross` compile, then the cross compiler will also include all strings referenced in this dummy module in the generated ROM string table.  Note that you don''t need to call this module; it's inclusion in the LFS build is enough to add the strings to the ROM table. Once in the ROM table, then you can use them subsequently in your application without incurring any RAM or GC overhead.
+
+A useful starting point may be found in [lua_examples/lfs/dummy_strings.lua](../lua_examples/lfs/dummy_strings.lua); this saves about 4Kb of RAM by moving a lot of common compiler and Lua VM strings into ROM.
+
+Another good use of this technique is when you have resources such as CSS, HTML and JS fragments that you want to output over the internet.  Instead of having lots of small resource files, you can just use string assignments in an LFS module and this will keep these constants in LFS instead.
 
-You must choose an explicit (non-default) LFS size to enable the use of LFS.  Most developers find it more useful to work with a fixed SPIFFS size matched to their application requirements.
 
 ### Choosing your development life-cycle
 
@@ -45,7 +190,7 @@ Most Lua developers seem to start with the [ESPlorer](https://github.com/4refr0n
 -  If you use a fixed SPIFFS image (I find 128Kb is enough for most of my applications) and are developing on a UART-attached ESP module, then you can also recompile any LC files and LFS image, then rebuild a SPIFFS file system image before loading it onto the ESP using `esptool.py`; if you script this you will find that this cycle takes less than a minute. You can either embed the LFS.img in the SPIFFS.  You can also use the `luac.cross -a` option to build an absolute address format image that you can directly flash into the LFS region within the firmware.
 
 -  If you only need to update the Lua components, then you can work over-the-air (OTA).  For example see my
-[HTTP_OTA.lua](https://github.com/nodemcu/nodemcu-firmware/tree/dev/lua_examples/lfs/HTTP_OTA.lua), which pulls a new LFS image from a webservice and reloads it into the LFS region.  This only takes seconds, so I often use this in preference to UART-attached loading.
+[HTTP_OTA.lua](../lua_examples/lfs/HTTP_OTA.lua), which pulls a new LFS image from a webservice and reloads it into the LFS region.  This only takes seconds, so I often use this in preference to UART-attached loading.
 
 -  Another option would be to include the FTP and Telnet modules in the base LFS image and to use telnet and FTP to update your system.  (Given that a 64Kb LFS can store thousands of lines of Lua, doing this isn't much of an issue.)
 
@@ -55,149 +200,21 @@ Under rare circumstances, for example a power fail during the flashing process,
 
 No doubt some standard usecase / templates will be developed by the community over the next six months.
 
-### Programming Techniques and approachs
+A LFS image can be loaded in the LFS store by either during provisioning of the initial firmware image or programmatically at runtime as discussed further in [Compiling and Loading LFS Images](#compiling-and-loading-lfs-images) below.
-
+one of two mechanisms:
-I have found that moving code into LFS has changed my coding style, as I tend to use larger modules and I don't worry about in-memory code size. This make it a lot easier to adopt a clearer coding style, so my ESP Lua code now looks more similar to host-based Lua code.  Lua code can still be loaded from SPIFFS, so you still have the option to keep code under test  in SPIFFS, and only move modules into LFS once they are stable.
-
-#### Accessing LFS functions and loading LFS modules
-
-See [lua_examples/lfs/_init.lua](https://github.com/nodemcu/nodemcu-firmware/tree/dev/lua_examples/lfs/_init.lua) for the code that I use in my `_init` module to do create a simple access API for LFS.  There are two parts to this.
-
-The first sets up a table in the global variable `LFS` with the `__index` and `__newindex` metamethods. The main purpose of the `__index()` is to resolve any names against the LFS using a `node.flashindex()` call, so that `LFS.someFunc(params)` does exactly what you would expect it to do: this will call `someFunc` with the specified parameters, if it exists in in the LFS.  The LFS properties `_time`, `_config` and `_list` can be used to access the other LFS metadata that you need.  See the code to understand what they do, but `LFS._list` is the array of all module names in the LFS.  The `__newindex` method makes `LFS` readonly.
-
-The second part uses standard Lua functionality to add the LFS to the require [package.loaders](http://pgl.yoyo.org/luai/i/package.loaders) list. (Read the link if you want more detail).  There are four standard loaders, which the require loader searches in turn.  NodeMCU only uses the second of these (the Lua loader from the file system), and since loaders 1,3 and 4 aren't used, we can simply replace the 1st or the 3rd by code to use `node.flashindex()` to return the LFS module.  The supplied `_init` puts the LFS loader at entry 3, so if the module is in both SPIFFS and LFS, then the SPIFFS version will be loaded. One result of this has burnt me during development: if there is an out of date version in SPIFFS, then it will still get loaded instead of the one if LFS.
-
-If you want to swap this search order so that the LFS is searched first, then SET `package.loaders[1] = loader_flash` in your `_init` code.  If you need to swap the search order temporarily for development or debugging, then do this after you've run the `_init` code:
-
-```Lua
-do local pl = package.loaders; pl[1],pl[3] = pl[3],pl[1]; end
-```
-
-#### Moving common string constants into LFS
-
-LFS is mainly used to store compiled modules, but it also includes its own string table and any strings loaded into this can be used in your Lua application without taking any space in RAM. Hence, you might also want to preload any other frequently used strings into LFS as this will both save RAM use and reduced the Lua Garbage Collector (**LGC**) overheads.
-
-The new debug function `debug.getstrings()` can help you determine what strings are worth adding to LFS.  It takes an optional string argument `'RAM'` (the default) or `'ROM'`, and returns a list of the strings in the corresponding table. So the following example can be used to get a listing of the strings in RAM.
-```Lua
-do
-  local a=debug.getstrings'RAM'
-  for i =1, #a do a[i] = ('%q'):format(a[i]) end
-  print ('local preload='..table.concat(a,','))
-end
-```
-
-You can do this at the interactive prompt or call it as a debug function during a running application in order to generate this string list, (but note that calling this still creates the overhead of an array in RAM, so you do need to have enough "head room" to do the call).
-
-You can then create a file, say `LFS_dummy_strings.lua`, and insert these `local preload` lines into it.  By including this file in your `luac.cross` compile, then the cross compiler will also include all strings referenced in this dummy module in the generated ROM string table.  Note that you don''t need to call this module; it's inclusion in the LFS build is enough to add the strings to the ROM table. Once in the ROM table, then you can use them subsequently in your application without incurring any RAM or LGC overhead.
-
-A useful starting point may be found in [lua_examples/lfs/dummy_strings.lua](https://github.com/nodemcu/nodemcu-firmware/tree/dev/lua_examples/lfs/dummy_strings.lua); this saves about 4Kb of RAM by moving a lot of common compiler and Lua VM strings into ROM.
-
-Another good use of this technique is when you have resources such as CSS, HTML and JS fragments that you want to output over the internet.  Instead of having lots of small resource files, you can just use string assignments in an LFS module and this will keep these constants in LFS instead.
-
-
-## Technical issues
-
-Whilst memory capacity isn't a material constraint on most conventional machines, the Lua RTS still includes some features to minimise overall memory usage. In particular:
-
--  The more resource intensive data types are know as _collectable objects_, and the RTS includes a LGC which regularly scans these collectable resources to determine which are no longer in use, so that their associated memory can be reclaimed and reused.
-
--  The Lua RTS also treats strings and compiled function code as collectable objects, so that these can also be LGCed when no longer referenced
-
-The compiled code, as executed by Lua RTS, internally comprises one or more function prototypes (which use a `Proto` structure type) plus their associated vectors (constants, instructions and meta data for debug). Most of these compiled constant types are basic (e.g. numbers) and the only collectable constant data type are strings. The other collectable types such as arrays are actually created at runtime by executing Lua compiled instructions to build each resource dynamically.
-
-When any Lua file is loaded without LFS into an ESP application, the RTS loads the corresponding compiled version into RAM. Each compiled function has its own Proto structure hierarchy, but this hierarchy is not exposed directly to the running application; instead the compiler generates `CLOSURE` instruction which is executed at runtime to bind the `Proto` to a Lua function value thus creating a [closure](https://en.wikipedia.org/wiki/Closure_(computer_programming)). Since this occurs at runtime, any `Proto` can be bound to multiple closures. A Lua closure can also have multiple RW [Upvalues](https://www.lua.org/pil/27.3.3.html) bound to it, and so function value is a Lua RW object in that it is referring to something that can contain RW state, even though the `Proto` hierarchy itself is intrinsically RO.
-
-Whilst advanced ESP Lua programmers can use overlay techniques to ensure that only active functions are loaded into RAM and thus increase the effective application size, this adds to runtime and program complexity. Moving Lua "program" resources into ESP Flash addressable memory typically at least doubles the effective RAM available, and removes the need to complicate applications code by implementing overlaying.
-
-Any RO resources that are relocated to a flash address space:
-
--  Must not be collected. Also RW references to RO resources must be robustly handled by the LGC.
--  Cannot reference to any volatile RW data elements (though RW resources can refer to RO resources).
-
-All strings in Lua are [interned](https://en.wikipedia.org/wiki/String_interning), so that only one copy of any string is kept in memory, and most string manipulation uses the address of this single copy as a unique reference. This uniqueness and the LGC of strings is facilitated by using a global string table that is hooked into the Lua global state.  Within standard Lua VM, any new string is first resolved against RAM string table, so that only the string-misses are added to the string table.
-
-The LFS patch adds a second RO string table in flash and this contains all strings used in the LFS Protos. Maintaining integrity across the two string tables is simple and low-cost, with LFS resolution process extended across both the RAM and ROM string tables. Hence any strings already in the ROM string table already have a unique string reference avoiding the need to add an additional entry in the RAM table. This both significantly reduces the size of the RAM string table, and removes a lot of strings from the LCG scanning.
-
-Note that my early development implementations of the LFS build process allowed on-target ESP builds, but I found that the Lua compiler was too resource hungry for usable application sizes, and  it was impractical to get this approach to scale. So we abandoned this approach and moved the LFS build process onto the development host machine by embedding this into `luac.cross`. This approach also avoids all of the update integrity issues involved in building a new LFS which might require RO resources already referenced in the RW ones.
-
-A LFS image can be loaded in the LFS store by one of two mechanisms:
 
 -  The image can be build on the host and then copied into SPIFFS. Calling the `node.flashreload()` API with this filename will load the image, and then schedule a restart to leave the ESP in normal application mode, but with an updated flash block.  This sequence is essentially atomic. Once called, and the format of the LFS image has been valiated, then the only exit is the reboot.
 
 -  The second option is to build the LFS image using the `-a` option to base it at the correct absolute address of the LFS store for a given firmware image.  The LFS can then be flashed to the ESP along with the firmware image.
 
-The LFS store is a fixed size for any given firmware build (configurable by the a pplication developer through `user_config.h`) and is at a build-specific base address within the `ICACHE_FLASH` address space.  This is used to store the ROM string table and the set of `Proto` hierarchies corresponding to a list of Lua files in the loaded image.
+The LFS store is a fixed size for any given firmware build (configurable by the application developer through `user_config.h`) and is at a build-specific base address within the `ICACHE_FLASH` address space.  This is used to store the ROM string table and the set of `Proto` hierarchies corresponding to a list of Lua files in the loaded image.
 
 A separate `node.flashindex()` function creates a new Lua closure based on a module loaded into LFS and more specfically its flash-based prototype; whilst this access function is not transparent at a coding level, this is no different functionally than already having to handle `lua` and `lc` files and the existing range of load functions (`load`,`loadfile`, `loadstring`). Either way, creating a closure on flash-based prototype is _fast_ in terms of runtime. (It is basically a single instruction rather than a compile, and it has minimal RAM impact.)
 
-### Implementation details
-
-This **LFS** patch uses two string tables: the standard Lua RAM-based table (`RWstrt`) and a second RO flash-based one (`ROstrt`). The `RWstrt` is searched first when resolving new string requests, and then the `ROstrt`. Any string not already in either table is then added to the `RWstrt`, so this means that the RAM-based string table only contains application strings that are not already defined in the `ROstrt`.
-
-Any Lua file compiled into the LFS image includes its main function prototype and all the child resources that are linked in its `Proto` structure; so all of these resources are compiled into the LFS image with this entire hierarchy self-consistently within the flash memory.
-
-```C
-   TValue        *k;         Constants used by the function
-   Instruction   *code       The Lua VM instuction codes
-   struct Proto **p;         Functions defined inside the function
-   int           *lineinfo;  Debug map from opcodes to source lines
-   struct LocVar *locvars;   Debug information about local variables
-   TString      **upvalues   Debug information about upvalue names
-   TString       *source     String name associated with source file
- ```
-
-Such LFS images are created by `luac.cross` using the `-f` option, and this builds a flash image using the list of modules provided but with a master "main" function of the form:
-
-```Lua
-local n = ...,1518283691  -- The Unix Time of the compile
-if n == "module1" then return module1 end
-if n == "module2" then return module2 end
--- and so on
-if n == "moduleN" then return module2 end
-return 1518283691,"module1","module2", --[[ ... ]] ""moduleN"
-```
-
-Note that you can't actually code this Lua because the modules are in separate compilation units, but the compiler being a compiler can just emit the compiled code directly. (See `app/lua/luac_cross/luac.c` for the details.)
-
-The deep cross-copy of the compiled `Proto` hierarchy is also complicated because current hosts are typically 64bit whereas the ESPs are 32bit, so the structures need repacking. (See `app/lua/luac_cross/luac.c` for the details.)
-
-This patch moves the `luac.cross` build into the overall application make hierarchy and so it is now simply a part of the NodeMCU make. The old Lua script has been removed from the `tools` directory, together with the need to have Lua pre-installed on the host.
-
-The LFS image is by default position independent, so is independent of the actual NodeMCU target image. You just have to copy it to the target file system and execute a `flashreload` and this copies the image from SPIFSS to the correct flash location, relocating all address to the correct base. (See `app/lua/lflash.c` for the details.)  This process is fast.
-
-A `luac.cross -a` option also allows absolute address images to be built for direct flashing the LFS store onto the module during provisioning.
-
-### Impact of the Lua Garbage Collector
-
-The LGC applies to what the Lua VM classifies as collectable objects (strings, tables, functions, userdata, threads -- known collectively as `GCObjects`). A simple two "colour" LGC was used in previous Lua versions, but Lua 5.1 introduced the Dijkstra's 3-colour (*white*, *grey*, *black*) variant that enabled the LGC to operate in an incremental mode. This permits smaller LGC steps interspersed by LGC pause, and is very useful for larger scale Lua implementations. Whilst this is probably not really needed for IoT devices, NodeMCU follows this standard Lua 5.1 implementation, albeit with the `elua` EGC changes.
-
-In fact, two *white* flavours are used to support incremental working (so this 3-colour algorithm really uses 4). All newly allocated collectable objects are marked as the current *white*, and a link in `GCObject` header enables scanning through all such Lua objects. Collectable objects can be referenced directly or indirectly via one of the Lua application's *roots*: the global environment, the Lua registry and the stack.
-
-The standard LGC algorithm is quite complex and assumes that all GCObjects are RW so that a flag byte within each object can be updated during the mark and sweep processing. LFS introduces GCObjects that are stored in RO memory and are therefore truly RO.
-The LFS patch therefore modifies the LGC processing to avoid such updates to GCObjects in RO memory, whilst still maintaining overall object integrity, as any attempt to update their content during LGC will result in the firmware crashing with a memory exception; the remainder of this section provides further detail on how this was achieved.  The LGC operates two broad phases: **mark** and **sweep**
-
--  The **mark** phase walks collectable objects by a recursive walk starting at at the LGC roots. (This is referred to as _traverse_.) Any object that is visited in this walk has its colour flipped from *white* to *grey* to denote that it is in use, and it is relinked into a grey list. The grey list is iteratively processed, removing one grey object at a time. Such objects can reference other objects (e.g. a table has many keys and values which can also be collectable objects), so each one is then also traversed and all objects reachable from it are marked, as above. After an object has been traversed, it's turned from grey to black. The LGC will walks all RW collectable objects, traversing the dependents of each in turn.  As RW objects can now refer to RO ones, the traverse routines has additional tests to skip trying to mark any RO LFS references.
-
--  The white flavour is flipped just before entering the **sweep** phase. This phase then loops over all collectable objects. Any objects found with previous white are no longer in use, and so can be freed. The 'current' white are kept; this prevents any new objects created during a paused sweep from being accidentally collected before being marked, but this means that it takes two sweeps to free all unused objects. There are other subtleties introduced in this 3-colour algorithm such as barriers and back-tracking to maintain integrity of the LGC, and these also needed extra rules to handle RO GCObjects correclty, but detailed explanation of these is really outside the scope of this paper.
-
-As well as standard collectable GCOobjets:
-
--  Standard Lua has the concept of **fixed** objects. (E.g. the main thread). These won't be collected by the LGC, but they may refer to objects that aren't fixed, so the LGC still has to walk through an fixed objects.
-
--  eLua added the the concept of **readonly** objects, which confusingly are a hybrid RW/RO implementation, where the underlying string resource is stored as a program constant in flash memory but the `TSstring` structure which points to this is still kept in RAM and can by GCed, except that in this case the LGC does not free the RO string constant itself.
-
--  LFS introduces a third variant **flash** object for `LUA_TPROTO` and `LUA_TSTRING` types. Flash objects can only refer to other flash objects and are entirely located in the LFS area in flash memory.
-
-The LGC already processed the _fixed_ and _readonly_ object, albeit as special cases. In the case of _flash_ GCObjects, the `mark` flag is in read-only memory and therefore the LGC clearly can't use this as a RW flag in its mark and sweep processing. So the LGC skips any marking operations for flash objects. Likewise, where all other GCObjects are linked into one of a number of sweeplists using the object's `gclist` field. In the case of flash objects, the compiler presets the `mark` and `gclist` fields with the fixed and readonly mark bits set, and the list pointer to `NULL` during the compile process.
-
-As far as the LGC algorithm is concerned, encountering any _flash_ object in a sweep is a dead end, so that branch of the walk of the GCObject hierarchy can be terminated on encountering a flash object. This in practice all _flash_ objects are entirely removed from the LGC process, without compromising collection of RW resources.
-
 ### General comments
 
 - **Reboot implementation**. Whilst the application initiated LFS reload might seem an overhead, it typically only adds a few seconds per reboot.
 
-- **LGC reduction**. Since the cost of LGC is directly related to the size of the LGC sweep lists, moving RO resources into LFS memory removes them from the LGC scope and therefore reduces LGC runtime accordingly.
-
 - **Typical Usecase**.  The rebuilding of a store is an occasional step in the development cycle. (Say up to 10-20 times a day in a typical intensive development process). Modules and source files under development can also be executed from SPIFFS in `.lua` format. The developer is free to reorder the `package.loaders` and load any SPIFFS files in preference to Flash ones. And if stable code is moved into Flash, then there is little to be gained in storing development Lua code in SPIFFS in `lc` compiled format.
 
 - **Flash caching coherency**.  The ESP chipset employs hardware enabled caching of the `ICACHE_FLASH` address space, and writing to the flash does not flush this cache.  However, in this restart model, the CPU is always restarted before any updates are read programmatically, so this (lack of) coherence isn't an issue.

docs/lua53.md

@@ -0,0 +1,270 @@
+## Background and Objectives
+
+The NodeMCU firmware was historically based on the Lua 5.1.4 core with the `eLua` patch and other NodeMCU specific enhancements and optimisations ("**Lua51**").  This paper discusses the rebaselining of NodeMCU to the latest production Lua version 5.3.5 ("**Lua53**").  Our goals in this upgrade were:
+
+-  NodeMCU now offers a current Lua version, 5.3.5 which is as functionally complete as practical.
+
+-  Lua53 adopts a minimum change strategy against the standard Lua source code base, that is changes to the VM and runtime system will only be made where there is a compelling reasons for any change, for example Lua53 preserves some valuable NodeMCU enhancements previously introduced to Lua51, for example the addition of Lua VM support for constant Lua program and data being executable directly from Flash ROM in order to free up RAM for application use to mitigate the RAM limitations of ESP-class IoT devices
+
+-  NodeMCU provides a clear and stable migration path for both existing hardware libraries and ESP Lua applications being migrated from Lua51 to Lua53.
+
+-  The Lua53 implementation provides a common code base for ESP8266 and ESP32 architectures.  (The Lua51 implementation was forked with variant code bases for the two architectures.)
+
+## Specific Design Decisions
+
+-  The NodeMCU C module API is built on the standard Lua C API that is common across the Lua51 and Lua53 build environments but with limited changes needs to reflect our IoT changes.  Note that standard Lua 5.3 introduced some core functional and C API changes v.v 5.1; however, the use the standard Lua 5.3 compatibility modes largely hides these changes, though modules can make use of the `LUA_VERSION_NUM` define should version-specific code variants be needed.
+
+-  The historic NodeMCU C module API (following `eLua` precedent and) added some extensions that somewhat compromised the orthogonal design principles of the standard Lua API; these are that modules should only access the Lua runtime via the `lua_XXX` macros and and calls exported through the `lua.h` header (or the wrapped helper `luaL_XXXX` versions exported through the `lauxlib.h` header).  Such inconsistencies will be removed from the existing NodeMCU API and modules, so that all modules can be compiled and executed within either the Lua51 or the Lua53 environment.
+
+-  Lua publishes a [Lua Reference Manual](https://www.lua.org/manual/5.3/)(**LRM**) for the Language specification, core libraries and C APIs for each Lua version.  The Lua53 implementation includes a supplemental [NodeMCU Reference Manual](nodemcu-lrm.md)(**NRM**) to document the NodeMCU extensions to the core libraries and C APIs.  As this API is unified across Lua51 and Lua53, this also provides a common reference that can also be used for developing both Lua51 and Lua53 modules.
+
+-  The two Lua code bases are maintained within a common Git branch (in parallel NodeMCU sub-directories `app/lua` and `app/lua53`).  An optional make parameter `LUA=53` selects a Lua53 build based on `app/lua53`, thus generating a Lua 5.3 firmware image.  (At a later stage once Lua53 is proven and stable, we will swap the default to Lua53 and move the Lua51 tree into frozen support.)
+
+-  Many of the important features of the `eLua` changes to Lua51 used by NodeMCU have now been incorporated into core Lua53 and can continue to be used 'out of the box'.   Other NodeMCU **LFS**, **ROTable** and **LCD** functionality has been rewritten for NodeMCU, so the Lua53 code base no longer uses the `eLua` patch.
+
+-  Lua53 will ultimately support three build targets that correspond to the ESP8266, ESP32 and native host targets using a _common Lua53 source directory_.  The ESP build targets generate a firmware image for the corresponding ESP chip class, and the host target generates a host based `luac.cross` executable.  This last can either be built standalone or as a sub-make of the ESP builds.
+
+-  The Lua53 host build `luac.cross` executable will continue to extend the standard functionality by adding support for LFS image compilation and include a Lua runtime execution environment that can be invoked with the `-e` option.  An optional make target also adds the [Lua Test Suite](https://www.lua.org/tests/) to this environment to enable use of this test suite.
+
+-  Lua 5.3 introduces the concept of **subtypes** which are used for Numbers and Functions, and Lua53 follows this model adding an additional ROTables subtype for Tables.
+
+   -  **Lua numbers** have separate **Integer** and **Floating point** subtypes.  There is therefore no advantage in having separate Integer and Floating point build variants.  Lua53 therefore ignores the `LUA_NUMBER_INTEGRAL` build option.  However it will provide option to use 32 or 64 bit numeric values with floating point numbers being stored as single or double precision respectively as well as the current hybrid where integers are 32-bit and `double` for floating point.  Hence 32-bit integer only applications will have similar memory use and runtime performance as existing Lua51 Integer builds.
+
+   -  **Lua tables** have separate **Table** and **ROTable** subtypes.  The Lua53 implementation of this subtyping has been backported to Lua51 where these were previously separate types, so that the table access API is the same for both Lua51 and Lua53.
+
+   -  **Lua Functions** have separate **Lua**, **C** and **lightweight C** subtypes, with this last being a special case where the C function has no upvals and so it doesn't need an associated `Closure` structure.  It is also the same TValue type as our lightweight functions.
+
+-  Lua 5.3 also introduces another small number of other but significant Lua language and  core library / API changes.  Our Lua53 implementation does not limit these, though we have enabled the appropriate compatibility modes to limit the impact at a Lua developer level.  These are discussed in further detail in the following compatibility sections.
+
+-  Many standard OS services aren't available on a embedded IoT device, so Lua53 follows the Lua51 precedent by omitting the `os` library for target builds as the relevant functionally is largely replaced by the `node` library.
+
+-  Flash based code execution can incur runtime performance impacts if not mitigated. Some small code changes were required as the current GCC toolchain for the Xtensa processors doesn't handle all of these flash access mitigations during code generation.  For example, the ESP CPU only supports aligned word access to flash memory, so 'hot' byte constant accesses have been encapsulated in an inline assembler macro to remove non-aligned exceptions at a cost of one extra Xtensa instruction per access; the remaining byte accesses to flash use a software exception handler. 
+
+-  NodeMCU employs a single threaded event loop model (somewhat akin to Node.js), and this is supported by some task extensions to the C API that facilitate use of a callback mechanism.
+
+## Detailed Implementation Notes
+
+We follow two broad principles (1) everything other than the Lua source directory is common, and (2) only change the Lua core when there is a compelling reason.  The following sub-sections describe these issues / changes in detail.
+
+### Build variants and includes
+
+Lua53 supports three build targets which correspond to:
+
+-  The ESP8266 (and derivative ESP8385) architectures use the Espressif non-OS SDK and its GCC Xtensa tool-chain.  The macros `LUA_USE_ESP8266` and `LUA_USE_ESP` are defined for this target.
+
+-  The ESP32 architecture using the Espressif IDF and its GCC Xtensa toolchain.  The macros `LUA_USE_ESP32` and `LUA_USE_ESP` are defined for this target.
+
+-  A host architecture using the standard host C toolchain.  The macro `LUA_USE_HOST` is defined for this target.  We currently support any POSIX environment that supports the GCC toolchain and Windows builds using native MSVC, WSL, Cygwin and MinGW.
+
+`LUA_USE_HOST` and `LUA_USE_ESP` are in effect mutually exclusive: `LUA_USE_ESP` is defined for a target firmware build, and `LUA_USE_HOST` for a build of the host-based `luac.cross` executable for the host environment.  Note that `LUA_USE_HOST` also defines `LUA_CROSS_COMPILER` as used in Lua51.
+
+Our Lua51 source has been migrated to use `newlib` conformant headers and C runtime calls.  An example of this is that the standard SDK headers supplied by Espressif include `"c_string.h"` and use `c_strcmp()` as the string comparison function.  NodeMCU source files use `<string.h>` and `strcmp()`.
+
+_**Caution**: The NodeMCU source is _compliant_ rather than fully _conformant_ with the standard headers such as `<string.h>`; that is the current subset of these APIs used by the code will successfully compile, link and execute as an image, but code additions which attempt to use extra functions defined in the APIs might not; this at least minimises the need to change standard source code to compile and run on the ESP8266._
+
+As with Lua51, Lua53 heavily customises the `linit.c`, `lua.c` and `luac.c` files because of the demands of an embedded runtime environment.  Given the amount of change, these files are stripped of the functionally dead code.
+
+### TString types and implementation
+
+The Lua 5.3 version includes a significant modification to the treatment of strings, by dividing them into two separate subtypes based on the string length (at `LUAI_MAXSHORTLEN`, 40 in the current implementation).  This decision reflects two empirical observations based on a broad range of practical Lua applications: the longer the string, the less likely the application is to recreate it independently; and the cost of ensuring uniqueness increases linearly with the length of the string.  Hence Lua53 now treats the string type differently:
+
+-  **Short Strings** are stored uniquely using the `strt` and `ROstrt` string table as discussed below.  Two short TStrings are identical if and only if their addresses are the same.
+
+-  **Long Strings** are created and copied by reference, but are not guaranteed to be stored uniquely.
+
+Since short strings are stored uniquely, identity comparison is based comparing their TString address.  For long TStrings identify comparison is a little more complex:
+
+-  They are identical if their addresses are the same
+-  They are different if their lengths are different.
+-  Failing these short circuits, a full `memcmp()` must be carried out.
+
+Lua GC of both types is essentially the same, except that collection of long strings does not need to update the `strt`.
+
+Note that for real applications, identical long strings are rarely generated by other than by copy-reference and hence in general the runtime savings benefits exceed the small chance of storage duplication.  Also note that this and other sub-typing is hidden at the Lua C API level and is handled privately inside the Lua VM implementation.
+
+Whilst running Lua applications make heavy use of TStrings, the Lua VM itself makes little use of TStrings and typically pushes any string literals as CStrings.  The Lua53 VM introduced a key cache to avoid the runtime cost of hashing string and doing the `strt` lookup for this type of CString constant.  The NodeMCU implementation shares this key cache with ROTable field resolution.
+
+Short strings in the standard Lua VM are bound at runtime into the the RAM-based `strt` string table.  The LFS implementation adds a second LFS-based readonly `ROstrt` string table that is created during LFS image load and then referenced on subsequent CPU restarts. The Lua VM and C API resolves each new short string first against the `strt`, then the `ROstrt` string table, before adding any unresolved strings into the `strt`.  Hence short strings are interned across these two `strt` and `ROstrt` string tables.  Any runtime reference to strings already in the LFS `ROstrt` therefore do not create additional entries in RAM.  So applications are free include dummy resource functions (such as `dummy_strings.lua` in `lua_examples/lfs`) to preload additional strings into `ROstrt` and avoid needing RAM for these string constants.  Such dummy functions don't need to called; simply inclusion in the LFS build is sufficient.
+
+An LFS section below discusses further implementation details.
+
+### ROTables
+
+The ROTables concept was introduced in `eLua`, with the ROTable format designed to be compiled by being declarable with C source and so statically included in the firmware at built-time, rather taking up RAM.  This essential functionality has been preserved across both Lua51 and Lua53.  At an API level ROTables are handled as a table subtype within the Lua VM except that:
+-  ROTables are declared statically in C code.
+-  Only a subset of key and value types is supported for ROTables.
+-  Attempting to write to a ROTable field will raise an error.
+-  The C API provides a method to push a ROTable reference direct to the Lua stack, but other than this, the Lua API to read ROTables and Tables is the same.
+
+We have now completely replaced the `eLua` implementation for Lua53, and this implementation has been back-ported to Lua51.  Tables are now declared using `LROT` macros with the `LROT_END()` macro also generating a `ROTable` structure, which is a variant of the standard `Table` header and linking to the `luaR_entry` vector declared using the various `LROT_XXXXENTRY()` macros.  This has new implementation has some major advantages:
+
+-  ROTables are a separate subtype of Table, and so only minor code changes are needed within `ltable.c` to implement this, with the implementation now effectively hidden from the rest of the runtime and any library modules; this has enabled the removal of most of the ROTable code patches introduced by `eLua` into Lua51.
+
+-  The `luaR_entry` vector is a linear list, so (unlike a standard RAM Table) any ROTable has no associate hash table for fast key lookup.  However we have introduced a unified ROTable key cache to provide direct access into ROTable entries with a typical hit rate over 99% (key cache misses still require a linear key scan), and so average ROTable access is perhaps 30% slower than RAM Table access, unlike the `eLua` implementation which was 30-50 times slower. 
+
+-  The `ROTable` structure variants are not GC collectable and so one of its fields is tagged to allow the Lua GC to short-circuit GC sweeps across such RO nodes.  The `ROTable` structure variant also drops unused fields to save space, and again this is handled internally within `ltable.c`.
+
+-  As all tables have a header record that includes a valid `flags` field, the `fasttm()` optimisations for common metamethods now work for both `ROTables` and `Tables`.
+
+The same Lua51 ROTable functionality and limitations also apply to Lua53 in order to minimise migration impact for C module libraries:
+
+-  ROTables can only have string keys and a limited set of Lua value types (Numeric, Light CFunc, Light UserData, ROTable, string and Nil). In Lua 5.3 `Integer` and `Float` are now separate numeric subtypes, so `LROT_INTENTRY()` takes an integer value. The new `LROT_FLOATENTRY()` is used for a non-integer values. This isn't a migration issue as none of the current NodeMCU modules use floating point constants in ROTables declared in them. (There is the only one currently used and that is `math.PI`.)
+
+   -  For 5.1 builds, `LROT_FLOATENTRY()` is a synonym of `LROT_NUMENTRY()`.
+   -  For 5.3 builds, `LROT_NUMENTRY()` is a synonym of  `LROT_INTENTRY()`.
+
+-  Some ordering limitations apply: `luaR_entry` vectors can be unordered _except for any metafields_:  Any entry with a key name starting in "_" _must_ must be ordered and placed at the start of the vector.
+
+-  The `LROT_BEGIN()` and  `LROT_END()` take the same three parameters. (These are ignored in the case of the `LROT_BEGIN()` macro, but by convention these are the same to facilitate the begin / end pairing).
+    -  The first field is the table name.
+    -  The second field is used to reference the ROTable's metatable (or `NULL` if it doesn't have one).
+    -  The third field is 0 unless the table is a metatable, in which case it is a bit mask used to define the `fasttm()` `flags` field.  This must match any metafield entries for metafield lookup to work correctly.
+
+### Proto Structures
+
+Standard Lua 5.3 contains a new peep hole optimisation relating to closures: the Proto structure now contains one RW field pointing to the last closure created, and the GC adopts a lazy approach to recovering these closures. When a new closure is created, if the old one exists _and the upvals are the same_ then it is reused instead of creating a new one. This allows peephole optimisation of a usecase where a function closure is embedded in a do loop, so the higher cost closure creation is done once rather than `n` times.
+
+This reduces runtime at the cost of RAM overhead.  However for RAM limited IoTs this change introduced two major issues: first, LFS relies on Protos being read-only and this RW `cache` field breaks this assumption; second closures can now exist past their lifetime, and this delays their GC.  Memory constrained NodeMCU applications rely on the fact that dead closed upvals can be GCed once the closure is complete.  This optimisation changes this behaviour.  Not good.
+
+Lua53 **_removes_** this optimisation for all prototypes.
+
+### Locale support
+
+Standard Lua 5.3 introduces localisation support.  NodeMCU Lua53 disables this because IoT implementation doesn't have the appropriate OS support.
+
+### Memory Optimisations
+
+Some Lua structures have `double` fields which are align(8) by default.  There is no reason or performance benefit for doing align(8) on ESPs so all Lua code is compiled with the `-fpack-struct=4` option.
+
+Lua53 also reimplements the Lua51 LCD (Lua Compact Debug) patch. This replaces the `sizecode` `ìnt` vector giving line info with a packed byte array that is typically 15-30× smaller.  See the [LCD whitepaper](lcd.md) for more information on this algorithm.
+
+### Unaligned exception avoidance
+
+By default the GCC compiler emits a `l8ui` instruction to access byte fields on the ESP8266 and ESP32 Xtensa processors. This instruction will generate an unaligned fetch exception when this byte field is in Flash memory (as will accessing short fields).  These exceptions are handled  by emulating the instruction in software using an unaligned access handler; this allows execution to continue albeit with the runtime cost of handling the exception in software. We wish to avoid the performance hit of executing this handler for such exceptions.
+
+`lobject.h` now defines a `GET_BYTE_FN(name,t,wo,bo)` macro. In the case of host targets this macro generates the normal field access, but in the case of Xtensa targets uses of this macro define an `static inline` access function for each field.  These functions at the default `-O2` optimisation level cause the code generator to emit a pair of `l32i.n` + `extui` instructions replacing the single `l8ui` instruction.  This has the cost of an extra instruction execution for accessing RAM data, but also removes the 200+ clock overhead of the software exception handler in the case of flash memory accesses.
+
+There are 9 byte fields in the `GCObject`,`TString`, `Proto`, `ROTable` structures that can either be statically compiled as `const struct` into library code space or generated by the Lua cross compiler and loaded into the LFS region; the `GET_BYTE_FN` macro is used to create inline access functions for these fields, and read references of the form `(o)->tt` (for example) have been recoded using the access macro form `gettt(o)`.  There are 44 such changed access references in the source which together represent perhaps 99% of potential sources of this software exception within the Lua VM. 
+
+The access macro hasn't been used where access is guarded by a conditional that implies the field in a RAM structure and therefore the `l8ui` instruction is executed correctly in hardware.  Another exclusion is in modules such as `lcode.c` which are only used in compilation, and where the addition runtime penalty is acceptable.
+
+A wider review of `const char` initialisers and `-S` asm output from the compiler confirms that there are few other cases of character loads of constant data, largely because inline character constants such as '@' are loaded into a register as an immediate parameter to a `movi.n` instruction.  Ditto use of short fields.
+
+### Modulus and division operation avoidance
+
+The Lua runtime uses the modulus (`%`) and divide (`/`) operators in a number of computations. This isn't an issue for most uses where the divisor is an integer power of 2 since the **gcc** optimiser substitutes a fast machine code equivalent which typically executes 1-4 inline Xtensa instructions (ditto for many constant multiplies).  The compiler will also fold any used in constant expressions to avoid runtime evaluation. However the ESP Xtensa CPU doesn't implement modulus and divide operations in hardware, so these generate a call to a subroutine such as `_udivsi3()` which typically involves 500 instructions or so to evaluate.  A couple of frequent uses have been replaced.  (I have ensured that such uses are space delimited, so searching for " % " will locate these. `grep -P " (%|/) (?!(2|4|8|16))" app/lua53/*.[hc]` will list them off.)
+
+### Key cache
+
+Standard Lua 5.3 introduced a string key cache for constant CString to TString lookup.  In NodeMCU we also introduced a lookaside cache for ROTable fields access, and in practice this provides single probe access for over 99% of key hit accesses to ROTable entries.
+
+In Lua53 these two caching functions (for CString and ROTable key lookup) have been unified into a common key cache to provide both caching functions with the runtime overhead of a single cache table in RAM.  Folding these two lookups into a single key cache isn't ideal, but given our limited RAM this allows the cache use to be rebalanced at runtime reflecting the relative use of CString and ROTable key lookups.
+
+### Flash image generation and loading
+
+The current Lua51 `app/lua` implementation has two variants for dumping and loading Lua bytecode: (1) `ldump.c` + `lundump.c`; (2) `lflashimg.c` + `lflash.c`. These have been unified into a single load / unload mechanism in Lua53.  However, this mechanism must facilitate sequential loading into flash storage, which is straight forward _if_ with some small changes to the standard internal ordering of the LC file format.  The reason for this is that any `Proto` can embed other Proto definitions internally, creating a Proto hierarchy.  The standard Lua dump algorithm dumps some Proto header components, then recurses into any sub-Protos before completing the wrapping Proto dump.  As a result each Proto's resources get interleaved with those of its subordinate Proto hierarchy.  This means that resources get to written to RAM non-serially, which is bad news for writing serially to the LFS region.
+
+The NodeMCU Lua53 dump reorders the proto hierarchy tree walk, so that resources of the lowest protos in the hierarchy are loaded first:
+```
+dump_proto(p)
+  foreach subp in p
+    dump_proto(subp)
+  dump proto content
+end
+```
+This results in Proto references now being backwards references to Protos that have already been loaded, and this in turn enables the Proto resources to be allocated as a sequential contiguous allocation units, so the same code can be used for loading compiled Lua code into RAM and into LFS.
+
+The standard Lua 5.3 dump format embeds string constants in each proto as a `len`+`byte string` definition. NodeMCU needs to separate the collection of strings into an `ROstrt` for LFS loading, and this requires an extra processing pass either on dump or load.  By doing a preliminary Proto scan to collect tracking the strings used then dumping these as a prologue makes the load process on the ESP a single pass and avoids any need for string resolution tables in the ESP's RAM. The extra memory resources needed for this two-pass dump aren't a material issue in a PC environment.
+
+Changes to `lundump.c` facilitate the addition of LFS mode. Writing to flash uses a record oriented write-once API. Once the flash cache has been flushed when updating the LFS region, this data can be directly accesses using the memory-mapped RO flash window, the resources are written directly to Flash without any allocation in RAM.
+
+Both the `dump.c` and `lundump.c` are compiled into both the ESP firmware and the host-based luac cross compiler.  Both the host and ESP targets use the same integer and float formats (e.g. 32 bit, 32-bit IEEE) which simplifies loading and unloading.  However one complication is that the host `luac.cross` application might be compiled on either a 32 or 64 bit environment and must therefore accommodate either 4 or 8 byte address constants.  This is not an issue with the compiled Lua format since this uses the grammatical structure of the file format to derive resource relationships, rather than offsets or pointers.
+
+We also have a requirement to generate binary compatible absolute LFS images for linking into firmware builds.  The host mode is tweaked to achieve this.  In this case the write buffer function returns the correct absolute ESP address which are 32-bit; this doesn't cause any execution issue in luac since these addressed are never used for access within `luac.cross`.  On 64-bit execution environments, it also repacks the Proto and other record formats on copy by discarding the top 32-bits of any address reference.
+
+#### Handling embedded integers in the dump format
+
+A typical dump contains a lot of integer fields, not only for Integer constants, but also for repeat count and lengths.  Most of these integers are small, so rather than using a fixed 4-byte field in the file stream all integers are unsigned and represented by a big-endian multi-byte encoding, 7 bits per byte, with the high-bit used as a continuation flag.  This means that integers 0..127 encode in 1 byte, 128..32,767 in 2 etc.  This multi-byte scheme has minimal overhead but reduces the size of typical `.lc` and `.img` by 10% with minimal extra processing and less than the cost of reading that extra 10% of bytes from the file system.
+
+A separate dump type is used for negative integer constants where the constant `-x` is stored as `-(x+1)`.  Note that endianness isn't an issue since the stream is processed byte-wise, but using big-endian simplifies the load algorithm.
+
+#### Handling LFS-based strings
+
+The dump function for a individual Protos hierarchy for loading as an `.lc` file follows the standard convention of embedding strings inline as a `\<len>\<byte sequence>`.  Any LFS image contains a dump of all of the strings used in the LFS image Protos as an "all-strings" prologue; the Protos are then dumped into the image with string references using an index into the all-strings header.  This approach enables a fast one-pass algorithm for loading the LFS image; it is also a compact encoding strategy as string references typically use 1 or 2 byte integer offset in the image file.
+
+One complication here is that in the standard Lua runtime start-up adds a set of special fixed strings to the `strt` that are also tagged to prevent GC. This could cause problems with the LFS image if any of these constants is used in the code.  To remove this conflict the LFS image loader _always_ automatically includes these fixed strings in the `ROstrt`. (This also moves an extra ~2Kb string constants from RAM to Flash as a side-effect.)  These fixed strings are omitted from the "all-strings prologue", even though the code itself can still use them.  The `llex.c` and `ltm.c` initialisers loop over internal `char *` lists to register these fixed strings. NodeMCU adds a couple of access methods to `llex.c` and `ltm.c` to enable the dump and load functions to process these lists and resolve strings against them.
+
+#### Handling LFS top level functions
+
+Lua functions in standard Lua 5.1 are represented by two variant Closure headers (for C and Lua functions).  In the case of Lua functions with upvals, the internal Protos can validly be bound to multiple function instances. `eLua` and Lua 5.3 introduced the concept of lightweight C functions as a separate function subtype that doesn't require a Closure record.  Note that a function variable in a Lua exists as a TValue referencing either the C function address or a Closure record; this Closure is not the same as the CallInfo records which are chained to track the current call chain and stack usage.
+
+Whilst lightweight C functions can be declared statically as TValues in ROTables, there isn't a corresponding mechanism for declaring a ROTable containing LFS functions. This is because a Lua function TValue can only be created at runtime by executing a `CLOSURE` opcode within the Lua VM.  The Lua51 implementation avoids this issue by generating a top level Lua dispatch function that does the equivalent of emitting `if name == "moduleN" then return moduleN end` for each entry, and this takes 4 Lua opcodes per module entry. This lookup has an O(N) cost which becomes non-trivial as N grows large, and so Lua51 has a somewhat arbitrary limit of 50 for the maximum number modules in a LFS image.
+
+In the Lua53 LFS implementation, the undump loader appends a `ROTable` to the LFS region which contains a set of entries `"module name"= Proto_address`.  These table values are store as lightweight userdata pointer and as such are not directly accessible via Lua but the NodeMCU C function that does LFS lookup can still retrieve the required Proto address, execute the `CLOSURE` and return the corresponding TValue.  Since this approach uses the standard table access API, which is a lot more efficient than the 4×N opcode `if` chain implementation.
+
+### Garbage collection
+
+Lua51 includes the eLua emergency GC, plus the various EGC tuning parameters that seem to be rarely used. The default setting (which most users use) is `node.egc.ALWAYS` which triggers a full GC before every memory allocation so the VM spends maybe 90% of its time doing full GC sweeps.
+
+Standard Lua 5.3 has adopted the eLua EGC but without the EGC tuning parameters. (I have raised a separate GitHub issue to discuss this.) We extend the EGC with the functional equivalent of the `ON_MEM_LIMIT` setting with a negative parameter, that is only trigger the EGC with less than a preset free heap left. The runtime spends far less time in the GC and code typically runs perhaps 5× faster.
+
+### Panic Handling
+
+Standard Lua includes a throw / catch framework for handling errors.  (This has been slightly modified to enable yielding to work across C API calls, but this can be modification can ignored for the discussion of Panic handling.)  All calls to Lua execution are handled by `ldo.c` through one of two mechanisms:
+
+-  All protected calls are handled via `luaD_rawrunprotected()` which links its C stack frame into the  `struct lua_longjmp` chain updating the head pointer at `L->errorJmp`.  Any `luaD_throw()` will `longjmp` up to this entry in the C stack, hence as long as there is at least one protected call in the call chain, the C call stack can be properly unrolled to the correct frame.
+
+-  If no protected calls are on the Lua call stack, then `L->errorJmp` will be null and there is no established C stack level to unroll to.  In this case the `luaD_throw()` will directly call the `at_panic()` handler.  Since there is no valid stack frame to unroll to and execution cannot safely continue, so the only safe next step is to abort, which in our case restarts the processor.
+
+Any Lua calls directly initiated through `lua.c` interpreter loop or through `luac.cross` are protected.  However NodeMCU applications can also establish C callbacks that are called directly by the SDK / event dispatcher.  The current practice is that these invoke their associated Lua CB using an unprotected call and hence the only safe option is to restart the processor on error.  With Lua53 we have introduced a new `luaL_pcallx()` call variant as a NodeMCU extension; this is new call is designed to be used within library CBs that execute Lua CB functions, and it is argument compatible with `lua_call()`, except that in the case of caught errors it will also return a negative call status.   This function establishes an error handler to protect the called function, and this is invoked on error at the error's stack level to provide a stack trace.
+
+This error handler posts a task to a panic error handler (with the error string as an upval) before returning control to the invoking routine.  If the Lua registry entry `onerror` exists and is set to a function, then the handler calls this with the error string as an argument otherwise it calls standard `print` function by default.  This function can return `false` in which case the handler exits, otherwise it restarts the processor.  The application can use `node.setonerror()` to override the default "always restart" action if wanted (for example to write an error to a logfile or to a network syslog before restarting).  Note that `print` returns `nil` and this has the effect of printing the full error traceback before restarting the processor.
+
+Currently all (bar 1) of the cases of such Lua callbacks within the NodeMCU C modules used a simple `lua_call()`, with the result that any runtime error executed a panic on error and reboots the processor. These call have all been replaced by the `luaL_pcallx()` variant, so control is always returned to the C routine, and a later post task report the error and restarts the processor.  Note that substituting library uses of `lua_call()` by `luaL_pcallx()` does changes processing paths in the case of thrown errors.  If the library CB function immediately returns control to the SDK/event scheduler after the call, then this is the correct behaviour.  However, in a few cases, the routine performs post-call clean-up and this adapt the logic depending on the return status.
+
+### The `luac.cross` execution environment
+
+As with Lua51, the Lua53 host-build `luac.cross` executable extends the standard functionality by adding support for LFS image compilation and also includes a Lua runtime execution environment that can be invoked with the `-e` option.  This environment was added primarily to facilitate in host testing (albeit with some limitations) of the NodeMCU.
+
+The make target `TEST=1` also adds the [Lua Test Suite](https://www.lua.org/tests/) to the `luac.cross -e` execution environment to enable this test support.  Due to NodeMCU extensions some changes were required to the Test suite so [`app/lua53/host/tests`](../app/lua53/host/tests) includes the version regressed against our current build.  Note that we plan to add some variant capability to the ESP target firmware build in the future.
+
+Enabling the test suite also disables some compiler optimisations and hence increases the size of compiled Lua files, so this test option is _not_ enabled by default in the `luac.cross` make.
+
+The test configuration has some variations from the standard suite:
+-  NodeMCU lua and luac.cross do not support dynamic loading and the related dynamic loading tests are omitted.
+-  The tests adopt the Lua compatibility modes implemented in our builds.
+-  The standard Lua VM supports the initiation of multiple VM environments and this feature is used in some tests.  Our firmware supports multiple Lua threads but only one `lua_newstate()` instance.  So the host `luac.cross` make with the `TEST=1` option set also supports multiple VM environments.
+
+This execution environment also emulates LFS loading and execution using the `-F` option to load an LFS image before running the `-e` script.  On POSIX environments this allocates the LFS region using a kernel extension, a page-aligned allocator and it also uses the kernel API to turn off write access to this region except during the simulated write to flash operations.  In this way unintended writes to the LFS region throw a H/W exception in a manner parallel to the ESP environment.
+
+### API Compatibility for NodeMCU modules
+
+The Lua public API has largely been preserved across both Lua versions.  Having done a difference analysis of the two API and in particular the standard `lua.h` and `lauxlib.h` headers which contain the public API as documented in the LRM 5.1 and 5.3, these differences are grouped into the following categories:
+
+-  NodeMCU features that were in Lua51 and have also been added to Lua53 as part of this migration.
+-  Differences (additions / removals / API changes) that we are not using in our modules and which can therefore be effectively ignored for the purposes of migration.
+-  Some very nice feature enhancements in Lua53, for example the table access functions are now type `int` instead of type `void` and return the type of the accessed TValue.  These features have been back-ported to Lua51 so that modules can be coded using Lua53 best practice and still work in a Lua51 runtime environment.
+-  Source differences which can be encapsulated through common macros will be be removed by updating module code to use this common macro set.  In a very small number of cases module functionality will be recoded to employ this common API base.
+
+Both the LRM and PiL make quite clear that the public API for C modules is as documented in `lua.h` and all its definitions start with `lua_`. This API strives for economy and orthogonality. The supplementary functions provided by the auxiliary library (`lauxlib.c`)  access Lua services and functions through the `lua.h` interface and without other reference to the internals of Lua; this is exposed through `lauxlib.h` and all its definitions start with `luaL_`.  All Lua library modules (such as `lstrlib.c` which implements `string`) conform to this policy and only access the Lua runtime via the `lua.h` and `lauxlib.h` interfaces.
+
+There are significant changes to internal APIs between Lua51 and Lua53 and as exposed in the other "private" headers within the Lua source directory, and so any code using these APIs may fail to work across the two versions.
+
+Both Lua51 and Lua53 have a concept of Lua core files, and these set the `LUA_CORE` define. In order to enforce limited access to the 'private' internal APIs, `#ifdef LUA_CORE` guards have been added to all private Lua headers effectively hiding them from application library and other non-core access.
+
+One thing that this analysis has underline is that the project has been lax in the past about how we allow our modules to be implemented.  All existing modules have now been updated to use only the public API.  If any new or changed module required any of the 'internal' Lua headers to compile, then it is implemented incorrectly.
+
+
+### Lua Language and Libary Compatibility for NodeMCU Lua modules
+
+For the immediate future we will be supporting both builds based on both language variants, so Lua module writers either:
+ -  Avoid using Lua 5.3 language features and implement their module in the common subset (this is currently our preferred approach);
+ -  Or explicitly state any language constraints and include a test for `_VERSION=='Lua.5.3'` (or 5.1) in the module startup and explicitly error if incompatible.
+
+### Other Implementation Notes
+
+-  **Use of Linker Magic**. Lua51 introduced a set of linker-aware macros to allow NodeMCU C library modules to be marshalled by the GNU linker for firmware builds; Lua53 target builds maintain these to ensuring cross-version compatibility. However, the lua53 `luac.cross` build does all library marshalling in `linit.c` and this removes the need to try to emulate this strategy on the diverse host toolchains that we support for compiling `luac.cross`.
+
+-  **Host / ESP interoperability**.  Our strategy is to build the firmware for the ESP target and `luac.cross` in the same make process.  This requires the host to use a little endian ANSI floating point host architecture such as x68, AMD64 or ARM, so that the LC binary formats are compatible.  This ain't a material constraint in practice.
+
+-  **Emergency GC.** The Lua VM takes a more aggressive stance than the standard Lua version on triggering a GC sweep on heap exhaustion.  This is because we run in a small RAM size environment. This means that any resource allocation within the Lua API can trigger a GC sweep which can call __GC metamethods which in turn can require to stack to be resized.

docs/modules/node.md

@@ -1,7 +1,7 @@
 # node Module
 | Since  | Origin / Contributor  | Maintainer  | Source  |
 | :----- | :-------------------- | :---------- | :------ |
-| 2014-12-22 | [Zeroday](https://github.com/funshine) | [Zeroday](https://github.com/funshine) | [node.c](../../app/modules/node.c)|
+| 2014-12-22 | [Zeroday](https://github.com/funshine) | [TerryE](https://github.com/TerryE) | [node.c](../../app/modules/node.c)|
 
 The node module provides access to system-level features such as sleep, restart and various info and IDs.
 
@@ -174,36 +174,13 @@ flash ID (number)
 
 ## node.flashindex()
 
-Returns the function reference for a function in the [LFS (Lua Flash Store)](../lfs.md).
+Deprecated synonym for [`node.LFS.get()`](#nodelfsget) to return an LFS function reference.
 
-#### Syntax
+Note that this returns `nil` if the function does not exist in LFS. 
-`node.flashindex(modulename)`
-
-#### Parameters
-`modulename`  The name of the module to be loaded.  If this is `nil` or invalid then an info list is returned
-
-#### Returns
--  In the case where the LFS in not loaded, `node.flashindex` evaluates to `nil`, followed by the flash mapped base addresss of the LFS, its flash offset, and the size of the LFS.
--  If the LFS is loaded and the function is called with the name of a valid module in the LFS, then the function is returned in the same way the `load()` and the other Lua load functions do.
--  Otherwise an extended info list is returned: the Unix time of the LFS build, the flash and mapped base addresses of the LFS and its current length, and an array of the valid module names in the LFS.
-
-#### Example
-
-The `node.flashindex()` is a low level API call that is normally wrapped using standard Lua code to present a simpler application API.  See the module `_init.lua` in the `lua_examples/lfs` directory for an example of how to do this.
 
 ## node.flashreload()
 
-Reload the [LFS (Lua Flash Store)](../lfs.md) with the flash image provided. Flash images are generated on the host machine using the `luac.cross`commnad.
+Deprecated synonym for [`node.LFS.reload()`](#nodelfsreload) to reload [LFS (Lua Flash Store)](../lfs.md) with the named flash image provided.
-
-#### Syntax
-`node.flashreload(imageName)`
-
-#### Parameters
-`imageName` The name of a image file in the filesystem to be loaded into the LFS.
-
-#### Returns
-`Error message`  LFS images are now gzip compressed.  In the case of the `imagename` being a valid LFS image, this is expanded and loaded into flash.  The ESP is then immediately rebooted, _so control is not returned to the calling Lua application_ in the case of a successful reload.  This reload process internally makes two passes through the LFS image file; and on the first it validates the file and header formats and detects any errors.  If any is detected then an error string is returned.
-
 
 ## node.flashsize()
 
@@ -278,7 +255,6 @@ system heap size left in bytes (number)
 
 Returns information about hardware, software version and build configuration.
 
-
 #### Syntax
 `node.info([group])`
 
@@ -294,6 +270,13 @@ If a `group` is given the return value will be a table containing the following
 	- `flash_size` (number)
 	- `flash_mode` (number) 0 = QIO, 1 = QOUT, 2 = DIO, 15 = DOUT.
 	- `flash_speed` (number)
+
+- for `group` = `"lfs"`
+	- `lfs_base` (number)	Flash offset of selected LFS region 
+	- `lfs_mapped` (number)	Mapped memory address of selected LFS region
+	- `lfs_size` (number)	size of selected LFS region
+	- `lfs_used` (number)	actual size used by current LFS image
+
 - for `group` = `"sw_version"`
 	- `git_branch` (string)
 	- `git_commit_id` (string)
@@ -302,6 +285,7 @@ If a `group` is given the return value will be a table containing the following
 	- `node_version_major` (number)
 	- `node_version_minor` (number)
 	- `node_version_revision` (number)
+
 - for `group` = `"build_config"`
 	- `ssl` (boolean)
 	- `lfs_size` (number) as defined at build time
@@ -361,6 +345,65 @@ See the `telnet/telnet.lua` in `lua_examples` for a more comprehensive example.
 #### See also
 [`node.output()`](#nodeoutput)
 
+
+## node.LFS
+
+Sub-table containing the API for [Lua Flash Store](../lfs.md)(**LFS**) access.  Programmers might prefer to map this to a global or local variable for convenience for example:
+```lua
+local LFS = node.LFS
+```
+This table contains the following methods and properties:
+
+Property/Method | Description
+-------|---------
+`config` | A synonym for [`node.info('lfs')`](#nodeinfo).  Returns the properties `lfs_base`, `lfs_mapped`, `lfs_size`, `lfs_used`.
+`get()` | See [node.LFS.get()](#nodelfsget).
+`list()` | See [node.LFS.list()](#nodelfslist).
+`reload()` |See [node.LFS.reload()](#nodelfsreload).
+`time` | Returns the Unix timestamp at time of image creation.
+
+
+## node.LFS.get() 
+
+Returns the function reference for a function in LFS.
+
+Note that unused `node.LFS` properties map onto the equialent `get()` call so for example: `node.LFS.mySub1` is a synonym for `node.LFS.get('mySub1')`.
+
+#### Syntax
+`node.LFS.get(modulename)`
+
+#### Parameters
+`modulename`  The name of the module to be loaded.
+
+#### Returns
+-  If the LFS is loaded and the `modulename` is a string that is the name of a valid module in the LFS, then the function is returned in the same way the `load()` and the other Lua load functions do
+-  Otherwise `nil` is returned.
+
+
+## node.LFS.list() 
+
+List the modules in LFS.
+
+
+#### Returns
+-  If no LFS image IS LOADED then `nil` is returned.
+-  Otherwise an sorted array of the name of modules in LFS is returned.
+
+## node.LFS.reload()
+
+Reload LFS with the flash image provided. Flash images can be generated on the host machine using the `luac.cross`command.
+
+#### Syntax
+`node.LFS.reload(imageName)`
+
+#### Parameters
+`imageName` The name of a image file in the filesystem to be loaded into the LFS.
+
+#### Returns
+-  In the case when the `imagename` is a valid LFS image, this is expanded and loaded into flash, and the ESP is then immediately rebooted, _so control is not returned to the calling Lua application_ in the case of a successful reload.
+-  The reload process internally makes multiple passes through the LFS image file. The first pass validates the file and header formats and detects many errors.  If any is detected then an error string is returned.
+
+
 ## node.output()
 
 Redirects the Lua interpreter to a `stdout` pipe when a CB function is specified (See  `pipe` module) and resets output to normal otherwise. Optionally also prints to the serial console.

docs/nodemcu-lrm.md

@@ -0,0 +1,330 @@
+# NodeMCU Reference Manual
+
+## Introduction
+
+NodeMCU firmware is an IoT project ("the Project") which implements a Lua-based runtime for SoC modules based on the Espressif ESP8266 and ESP32 architectures.  This NodeMCU Reference Manual (**NRM**) specifically addresses how the NodeMCU Lua implementation relates to standard Lua as described in the two versions of the Lua language that we currently support:
+
+-  The [Lua 5.1 Reference Manual](https://www.lua.org/manual/5.1/) and
+-  The [Lua 5.3 Reference Manual](https://www.lua.org/manual/5.3/) (**LRM**)
+
+Developers using the NodeMCU environment should familiarise themselves with the 5.3 LRM.
+
+The Project provides a wide range of standard library modules written in both C and Lua to support many ESP hardware modules and chips, and these are documented in separate sections in our [online documentation](index.md).
+
+The NRM supplements LRM content and module documentation by focusing on a complete description of the _differences_ between NodeMCU Lua and standard Lua 5.3 in use. It adopts the same structure and style as the LRM.  As NodeMCU provides a full implementation of the Lua language there is little content herein relating to Lua itself.  However, what NodeMCU does is to offer a number of enhancements that enable resources to be allocated in constant program memory — resources in standard Lua that are allocated in RAM; where this does impact is in the coding of C library modules and the APIs used to do this.  Hence the bulk of the differences relate to these APIs.
+
+One of our goals in introducing Lua 5.3 support was to maintain the continuity of our existing C modules by ensuring that they can be successfully compiled and executed in both the Lua 5.1 and 5.3 environments.  This goal was achieved by a combination of:
+-  enabling relevant compatibility options for standard Lua libraries;
+-  back porting some Lua 5.3 API enhancements back into our Lua 5.1 implementation, and 
+-  making some small changes to the module source to ensure that incompatible API use is avoided.
+
+Further details are given in the [Lua compatibility](#lua-compatibility) section below.  Notwithstanding this, the Project has now deprecated Lua 5.1 and will soon be moving this version into frozen support.
+
+As well as providing the ability to building runtime firmware environments for ESP chipsets, the Project also offers a `luac.cross` cross compiler that can be built for common platforms such as Windows 10 and Linux, and this enables developers to compile source modules into a binary file format for download to ESP targets and loading from those targets.
+ 
+## Basic Concepts
+
+The NodeNCU runtime offers a full implementation of the [LRM §2](https://www.lua.org/manual/5.3/manual.html#2) core concepts, with the following adjustments:
+
+### Values and Types
+
+-  The firmware is compiled to use 32-bit integers and single-precision (32-bit) floats.  Address pointers are also 32-bit, and this allows all Lua variables to be encoded in RAM as an 8-byte "`TValue`" (compared to the 12-byte `TValue` used in Lua 5.1).
+-  C modules can statically declare read-only variant of the `Table` structure type known as a "`ROTable`".  There are some limitations to the types for ROTable keys and value, in order to ensure that these are consistent with static declaration.  ROTables are stored in code space (in flash memory on the ESPs), and hence do not take up RAM resources.  However these are still represented by the Lua type _table_ and a Lua application can treat them during execution the same as any other read-only table.  Any attempt to write to a `ROTable` or to set its metatable will throw an error.
+-  NodeMCU also introduces a concept known as **Lua Flash Store (LFS)**.  This enables Lua code (and any string constants used in this code) to be compiled and stored in code space, and hence without using RAM resources.  Such LFS functions are still represented by the type _function_ and can be executed just like any other Lua function.
+
+### Environments and the Global Environment
+
+These are implemented as per the LRM.  Note that the Lua 5.1 and Lua 5.3 language implementations are different and can introduce breaking incompatibilities when moving between versions, but this is a Lua issue rather than a NodeMCU one.
+
+### Garbage Collection
+
+All LFS functions, any string constants used in these functions, and any ROTables are stored in static code space.  These are ignored by the Lua Garbage collector (LGC).  The LGC only scans RAM-based resources and recovers unused ones.  The NodeMCU LGC has slightly modified configuration settings that increase its aggressiveness as heap usage approaches RAM capacity.
+
+### Coroutines
+
+The firmware includes the full coroutine implementation, but note that there are some slight differences between the standard Lua 5.1 and Lua 5.3 C API implementations. (See [Feature breaks](#feature-breaks) below.)
+
+## Lua Language
+
+The NodeNCU runtime offers a full implementation of the Lua language as defined in [LRM §3](https://www.lua.org/manual/5.3/manual.html#3) and its subsections.
+
+## The Application Program Interface
+
+[LRM §4](https://www.lua.org/manual/5.3/manual.html#4) describes the C API for Lua.  This is used by NodeMCU modules written in the C source language to interface to the the Lua runtime.  The header file `lua.h` is used to define all API functions, together with their related types and constants.  This section 4 forms the primary reference, though the NodeMCU makes minor changes as detailed below to support LFS and ROTable resources.
+
+### Error Handling in C
+
+[LRM §4.6](https://www.lua.org/manual/5.3/manual.html#4.6) describes how errors are handled within the runtime. 
+
+The normal practice within the runtime and C modules is to throw any detected errors -- that is to unroll the call stack until the error is acquired by a routine that has declared an error handler.  Such an environment can be established by  [lua_pcall](https://www.lua.org/manual/5.3/manual.html#lua_pcall) and related API functions within C and by the Lua function [pcall](https://www.lua.org/manual/5.3/manual.html#pdf-pcall); this is known as a _protected environment_.  Errors which occur outside any protected environment are not caught by the Lua application and by default trigger a "panic". By default NodeMCU captures the error traceback and posts a new SDK task to print the error before restarting the processor.
+
+The NodeMCU runtime implements a non-blocking threaded model that is similar to that of `node.js`, and hence most Lua execution is initiated from event-triggered callback (CB) routines within C library modules.  NodeMCU enables full recovery of error diagnostics from otherwise unprotected Lua execution by adding an additional auxiliary library function [`luaL_pcallx`](#luaL_pcallx).  All event-driven Lua callbacks within our library modules use `luaL_pcallx` instead of `lua_call`.  This has the same behaviour if no uncaught error occurs.  However, in the case of an error that would have previously resulted in a panic, a new SDK task is posted with the error traceback as an upvalue to process this error information.
+
+The default action is to  print a full traceback and then trigger processor restart, that is a similar outcome as before but with recovery of the full error traceback.  However the `node.onerror()` library function is available to override this default action; for example developers might wish to print the error without restarting the processor, so that the circumstances which triggered the error can be investigated.
+
+### Additional API Functions and Types
+
+These are available for developers coding new library modules in C.  Note that if you compare the standard and NodeMCU versions of `lua.h` you will find a small number of entries not listed below.  This is because the Lua 5.1 and Lua 5.3 variants are incompatible owing to architectural differences.  However, `laubxlib.h` includes equivalent wrapper version-compatible functions that may be used safely for both versions.
+
+#### ROTable and ROTable_entry
+
+Extra structure types used in `LROT` macros to declare static RO tables.  See detailed section below.
+
+#### lua_createrotable
+
+` void (lua_createrotable) (lua_State *L, ROTable *t, const ROTable_entry *e, ROTable *mt);`         [-0, +1, -]
+
+Create a RAM based `ROTable` pointing to the `ROTable_entry` vector `e`, and metatable `mt`.
+
+#### lua_debugbreak and ASSERT
+
+`  void (lua_debugbreak)(void);`
+
+`lua_debugbreak()` and `ASSERT(condition)` are available for development debugging.  If `DEVELOPMENT_USE_GDB` is defined then these will respectively trigger a debugger break and evaluate a conditional assert prologue on the same.  If not, then these are effectively ignored and generate no executable code.
+
+#### lua_dump
+
+` int lua_dump (lua_State *L, lua_Writer writer, void *data, int strip);`         [-0, +0, –]
+
+Dumps function at the top of the stack function as a binary chunk as per LRM.  However the last argument `strip` is now an integer is in the range -1..2, rather a boolean as per standard Lua:
+
+-  -1, use the current default strip level (which can be set by [`lua_stripdebug`](#lua_stripdebug))
+-  0, keep all debug info
+-  1, discard Local and Upvalue debug info; keep line number info
+-  2, discard Local, Upvalue and line number debug info
+
+The internal NodeMCU `Proto` encoding of debug line number information is typically 15× more compact than in standard Lua; the intermediate `strip=1` argument allows the removal of most of the debug information whilst retaining the ability to produce a proper line number traceback on error. 
+
+#### lua_freeheap
+
+` int lua_freeheap (void);`         [-0, +0, –]
+
+returns the amount of free heap available to the Lua memory allocator.
+
+#### lua_gc
+
+`  int lua_gc (lua_State *L, int what, int data);`         [-0, +0, m]
+
+provides an option for 
+- `what`:
+    - `LUA_GCSETMEMLIMIT`  sets the available heap threshold (in bytes) at which aggressive sweeping starts.
+
+#### lua_getlfsconfig
+
+`  void lua_getlfsconfig (lua_State *L, int *conf);`         [-0, +0, -]
+
+if `conf` is not `NULL`, then this returns an int[5] summary of the LFS configuration.  The first 3 items are the mapped and flash address of the LFS region, and its allocated size.  If the LFS is loaded then the 4th is the current size used and the 5th (for Lua 5.3 only) the date-timestamp of loaded LFS image. 
+
+#### lua_getstate
+
+`  lua_State * lua_getstate();`         [-0, +0, -]
+
+returns the main thread `lua_State` record.  Used in CBs to initialise `L` for subsequent API call use.
+
+#### lua_pushrotable
+`  void lua_pushrotable (lua_State *L, ROTable *p);`         [-0, +1, -]
+
+Pushes a ROTable onto the stack.
+
+#### lua_stripdebug
+`  int lua_stripdebug (lua_State *L, int level);`         [-1, +0, e]
+
+This function has two modes.  A value is popped off the stack.
+-  If this value is `nil`, then the default strip level is set to given `level` if this is in the range 0 to 2. Returns the current default level.
+-  If this value is a Lua function (in RAM rather than in LFS), then the prototype hierarchy within the function is stripped of debug information to the specified level. Returns an _approximate_ estimate of the heap freed (this can be a lot higher if some strings can be garbage collected).
+
+#### lua_writestring
+
+`  void lua_writestring(const char *s, size_t l); /* macro */`
+
+Writes a string `s` of length `l` to `stdout`.  Note that any output redirection will be applied to the string.
+ 
+#### lua_writestringerror
+
+`  void lua_writestringerror(const char *s, void *p).  /* macro */`
+
+Writes an error with CString format specifier `s` and parameter `p` to `stderr`.  Note on the ESP devices this error will always be sent to `UART0`; output redirection will not be applied.
+
+### The Debug Interface
+
+#### lua_pushstringsarray
+`  int lua_pushstringsarray (lua_State *L, int opt);`         [-0, +1, m]
+
+Pushes an array onto the stack containing all strings in the specified strings table. If `opt` = 0 then the RAM table is used, else if `opt` = 1 and LFS is loaded then the LFS  table is used, else `nil` is pushed onto the stack.
+
+Returns a status boolean; true if a table has been pushed.
+
+### Auxiliary Library Functions and Types 
+
+Note that the LRM defines an auxiliary library which contains a set of functions that assist in coding convenience and economy.  These are strictly built on top of the basic API, and are defined in `lauxlib.h`.  By convention all auxiliary functions have the prefix `luaL_` so module code should only contain `lua_XXXX()` and `luaL_XXXX()` data declarations and functions.  And since the `lauxlib.h` itself incudes `lua.h`, all C modules should only need to `#include "lauxlib.h"` in their include preambles.
+
+NodeMCU adds some extra auxiliary functions above those defined in the LRM.
+
+#### luaL_lfsreload
+
+`  int  lua_lfsreload (lua_State *L);`         [-1, +1, -]
+
+This function pops the LFS image name from the stack, and if it exists and contains the correct image header then it reloads LFS with the specified image file, and immediately restarts with the new LFS image loaded, so control it not returned to the calling function. If the image is missing or the header is invalid then an error message is pushed onto the stack and control is returned.  Note that if the image has a valid header but its contents are invalid then the result is undetermined.
+
+#### luaL_pcallx
+
+`  int luaL_pcallx (lua_State *L, int narg, int nresults);`         [-(nargs + 1), +(nresults|1), –]
+
+Calls a function in protected mode and providing a full traceback on error.
+
+Both `nargs` and `nresults` have the same meaning as in [lua_call](https://www.lua.org/manual/5.3/manual.html#lua_call). If there are no errors during the call, then `luaL_pcallx` behaves exactly like `lua_call`. However, if there is any error, `lua_pcallx` has already established an traceback error handler for the call that catches the error. It cleans up the stack and returns the negative error code.
+
+Any caught error is posted to a separate NodeMCU task which which calls the error reporter as defined in the registry entry `onerror` with the traceback text as its argument.  The default action is to print the error and then set a 1 sec one-shot timer to restart the CPU.  (One second is enough time to allow the error to be sent over the network if redirection to a telnet session is in place.)  If the `onerror` entry is set to `print` for example, then the error is simply printed without restarting the CPU.
+
+Note that the Lua runtime does not call the error handler if the error is an out-of memory one, so in this case the out-of-memory error is posted to the error reporter without a traceback.
+
+#### luaL_posttask
+
+`  int luaL_posttask (lua_State* L, int prio);`         [-1, +0, e]
+
+Posts a task to execute the function popped from the stack at the specified user task priority
+-  `prio` one of:
+   -  `LUA_TASK_LOW`
+   -  `LUA_TASK_MEDIUM`
+   -  `LUA_TASK_HIGH`
+
+Note that the function is invoked with the priority as its parameter.
+
+#### luaL_pushlfsmodule
+
+`  int luaL_pushlfsmodule ((lua_State *L);`         [-1, +1, -]
+
+This function pops a module name from the stack. If this is a string, LFS is loaded and it contains the named module then its closure is pushed onto the stack as a function value, otherwise `nil` is pushed.
+
+Returns the type of the pushed value.
+
+#### luaL_pushlfsmodules
+
+`  int luaL_pushlfsmodules (lua_State *L);`         [-0, +1, m]
+
+If LFS is loaded then an array of the names of all of the modules in LFS is pushed onto the stack as a function value, otherwise `nil` is pushed.
+
+Returns the type of the pushed value.
+
+#### luaL_pushlfsdts
+
+`  int luaL_pushlfsdts (lua_State *L);`         [-0, +1, m-]
+
+If LFS is loaded then the Unix-style date-timestamp for the compile time of the image is pushed onto the stack as a integer value, otherwise `nil` is pushed.  Note that the primary use of this stamp is to act as a unique identifier for the image version.
+
+Returns the type of the pushed value.
+
+#### luaL_reref
+`  void (luaL_reref) (lua_State *L, int t, int *ref);`         [-1, +0, m]
+
+Variant of [luaL_ref](https://www.lua.org/manual/5.3/manual.html#luaL_ref).  If `*ref` is a valid reference in the table at index t, then this is replaced by the object at the top of the stack (and pops the object), otherwise it creates and returns a new reference using the `luaL_ref` algorithm.
+
+#### luaL_rometatable
+
+`  int (luaL_rometatable) (lua_State *L, const char* tname, const ROTable *p);`         [-0, +1, e]
+
+Equivalent to `luaL_newmetatable()` for ROTable metatables.  Adds key / ROTable entry to the registry  `[tname] = p`, rather than using a new RAM table.
+
+#### luaL_unref2
+
+`  luaL_unref2(l,t,r)`
+
+This macro executes `luaL_unref(L, t, r)` and then assigns `r = LUA_NOREF`.
+
+### Declaring modules and ROTables in NodeMCU
+
+All NodeMCU C library modules should include the standard header "`module.h`".  This internally includes `lnodemcu.h` and these together provide the macros to enable declaration of NodeMCU modules and ROTables within them.  All ROtable support macros are either prefixed by `LRO_` (Lua Read Only) or in the case of table entries `LROT_`. 
+
+#### NODEMCU_MODULE
+
+` NODEMCU_MODULE(sectionname, libraryname, map, initfunc)`
+
+This macro enables the module to be statically declared and linked in the `ROM` ROTable.  The global environment's metafield `__index` is set to the ROTable `ROM` hence any entries in the ROM table are resolved as read-only entries in the global environment.
+-  `sectionname`. This is the linker section for the module and by convention this is the uppercased library name (e.g. `FILE`).  Behind the scenes `_module_selected` is appended to this section name if corresponding "use module" macro (e.g. `LUA_USE_MODULES_FILE`) is defined in the configuration.  Only the modules sections `*_module_selected` are linked into the firmware image and those not selected are ignored.
+-  `libraryname`.  This is the name of the module (e.g. `file`) and is the key for the entry in the `ROM` ROTable.
+-  `map`.  This is the ROTable defining the functions and constants for the module, and this is the corresponding value for the entry in the `ROM` ROTable.
+-  `initfunc`.  If this is not NULL, it should be a valid C function and is call during Lua initialisation to carry out one-time initialisation of the module.
+
+#### LROT_BEGIN and LROT_END
+
+`  LROT_BEGIN(rt,mt,flags)`
+`  LROT_END(rt,mt,flags)`
+These macros start and end a ROTable definition.  The three parameters must be the same in both declarations.
+-  `rt`.  ROTable name.
+-  `mt`.  ROTable's metatable.  This should be of the form `LROT_TABLEREF(tablename)` if the metatable is used and `NULL` otherwise.
+-  `flags`. The Lua VM table access routines use a `flags` field to short-circuit where the access needs to honour metamethods during access.  In the case of a static ROTable this flag bit mask must be declared statically during compile rather than cached dynamically at runtime.  Hence if the table is a metatable and it includes the metamethods `__index`, `__newindex`, `__gc`, `__mode`, `__len` or `__eq`, then the mask field should or-in the corresponding mask for example if `__index` is used then the flags should include `LROT_MASK_INDEX`.  Note that index and GC are a very common combination, so `LROT_MASK_GC_INDEX` is also defined to be `(LROT_MASK_GC | LROT_MASK_INDEX)`.
+
+#### LROT_xxxx_ENTRY
+
+ROTables only support static declaration of string keys and value types: C function, Lightweight userdata, Numeric, ROTable.  These are entries are declared by means of the  `LROT_FUNCENTRY`, `LROT_LUDENTRY`, `LROT_NUMENTRY`, `LROT_INTENTRY`, `LROT_FLOATENTRY` and `LROT_TABENTRY` macros.  All take two parameters: the name of the key and the value. For Lua 5.1 builds `LROT_NUMENTRY` and `LROT_INTENTRY` both generate a numeric `TValue`, but in the case of Lua 5.3 these are separate numeric subtypes so these macros generate the appropriate subtype.
+
+Note that ROTable entries can be declared in any order except that keys starting with "`_`" must be declared at the head of the list.  This is a pragmatic constraint for runtime efficiency.  A lookaside cache is used to optimise key searches and results in a direct table probe in over 95% of ROTable accesses.  A table miss (that is the key doesn't exist) still requires a full scan of the list, and the main source of table misses are scans for metafield values.  Forcing these to be at the head of the ROTable allows the scan to abort on reading the first non-"`_`" key.
+
+ROTables can still support other key and value types by using an index metamethod to point at an C index access function.  For example this technique is used in the `utf8` library to return `utf8.charpattern`
+
+```C
+LROT_BEGIN(utf8_meta, NULL, LROT_MASK_INDEX)
+  LROT_FUNCENTRY( __index, utf8_lookup )
+LROT_END(utf8_meta, NULL, LROT_MASK_INDEX)
+
+LROT_BEGIN(utf8, LROT_TABLEREF(utf8_meta, 0)
+  LROT_FUNCENTRY( offset, byteoffset )
+  LROT_FUNCENTRY( codepoint, codepoint )
+  LROT_FUNCENTRY( char, utfchar )
+  LROT_FUNCENTRY( len, utflen )
+  LROT_FUNCENTRY( codes, iter_codes )
+LROT_END(utf8, LROT_TABLEREF(utf8_meta), 0)
+```
+
+Any reference to `utf8.charpattern` will call the `__index` method function (`utf8_lookup()`); this returns the UTF8 character pattern if the name equals `"charpattern"` and `nil` otherwise.  Hence `utf8` works as standard even though ROTables don't natively support a string value type.
+
+### Standard Libraries
+
+-  Basic Lua functions, coroutine support, Lua module support, string and table manipulation are as per the standard Lua implementation.  However, note that there are some breaking changes in the standard Lua string implementation as discussed in the LRM, e.g. the `\z` end-of-line separator; no string functions exhibit a CString behaviour (that is treat `"\0"` as a special character).
+-  The modulus operator is implemented for string data types so `str % var` is a synonym for `string.format(str, var)` and `str % tbl` is a synonym for `string.format(str, table.unpack(tbl))`.  This python-like formatting functionality is a very common extension to the string library, but is awkward to implement with `string` being a `ROTable`.
+-  The `string.dump()` `strip` parameter can take integer values 1,2,3 (the [`lua_stripdebug`](#lua_stripdebug) strip parameter + 1).  `false` is synonymous to `1`, `true` to `3` and omitted takes the default strip level.
+-  The `string` library does not offer locale support. 
+-  The 5.3 `math` library is expanded compared to the 5.1 one, and specifically:
+    - Included: ` abs`, ` acos`, ` asin`, ` atan`, ` ceil`, ` cos`, ` deg`, ` exp`, ` tointeger`, ` floor`, ` fmod`, ` ult`, ` log`, ` max`, ` min`, ` modf`, ` rad`, ` random`, ` randomseed`, ` sin`, ` sqrt`, ` tan` and ` type`
+    -  Not implemented: ` atan2`, ` cosh`, ` sinh`, ` tanh`, ` pow`, ` frexp`, ` ldexp` and ` log10`
+ -  Input, output OS Facilities (the `io` and `os` libraries) are not implement for firmware builds because of the minimal OS supported offered by the embedded run-time.  The separately documented `file` and `node` libraries provide functionally similar analogues.  The host execution environment implemented by `luac.cross` does support the `io` and `os` libraries.
+-  The full `debug` library is implemented less the `debug.debug()` function.
+    -  An extra function `debug.getstrings(type)` has been added; `type` is one of `'ROM'`, or `'RAM'` (the default). Returns a sorted array of the strings returned from the [`lua_getstrings`](#lua_getstrings) function.
+
+## Lua compatibility
+
+Standard Lua has a number of breaking incompatibilities that require conditional code to enable modules using these features to be compiled against both Lua 5.1 and Lua 5.3.  See [Lua 5.2 §8](https://www.lua.org/manual/5.2/manual.html#8) and [Lua 5.3 §8](https://www.lua.org/manual/5.3/manual.html#8) for incompatibilities with Lua 5.1.
+
+A key strategy in our NodeMCU migration to Lua 5.3 is that all NodeMCU application modules must be compilable and work under both Lua versions.  This has been achieved by three mechanisms
+-  The standard Lua build has conditionals to enable improved compatibility with earlier versions.  In general the Lua 5.1 compatibilities have been enabled in the Lua 5.3 builds.
+-  Regressing new Lua 5.3 features into the Lua 5.1 API.
+-  For a limited number of features the Project accepts that the two versions APIs are incompatible and hence modules should either avoid their use or use `#if LUA_VERSION_NUM == 501` conditional compilation.
+
+The following subsections detail how NodeMCU Lua versions deviate from standard Lua in order to achieve these objectives.
+
+### Enabling Compatibility modes
+
+The following Compatibility modes are enabled:
+-  `LUA_COMPAT_APIINTCASTS`.  These `ìnt`cast are still used within NodeMCU modules.
+-  `LUA_COMPAT_UNPACK`.  This retains `ROM.unpack` as a global synonym for `table.unpack` 
+-  `LUA_COMPAT_LOADERS`.  This keeps `package.loaders` as a synonym for `package.seachers`
+-  `LUA_COMPAT_LOADSTRING`.  This keeps `loadstring(s)` as a synonym for `load(s)`.
+
+### New Lua 5.3 features back-ported into the Lua 5.1 API
+
+-  Table access routines in Lua 5.3 are now type `int` rather than `void` and return the type of the value pushed onto the stack.  The 5.1 routines `lua_getfield`, `lua_getglobal`, `lua_geti`, `lua_gettable`, `lua_rawget`, `lua_rawgeti`, `lua_rawgetp` and `luaL_getmetatable` have been updated to mirror this behaviour and return the type of the value pushed onto the stack.
+-  There is a general numeric comparison API function `lua_compare()` with macros for `lua_equal()` and `lua_lessthan()` whereas 5.1 only support the `==` and `<` tests through separate API calls.  5.1 has been update to mirror the 5.3. implementation.
+-  Lua 5.3 includes a `lua_absindex(L, idx)` which converts ToS relative (e.g. `-1`) indices to stack base relative and hence independent of further push/pop operations.  This makes using down-stack indexes a lot simpler. 5.1 has been updated to mirror this 5.3 function.
+ 
+### Feature breaks
+
+-  `\0` is now a valid pattern character in search patterns, and the `%z` pattern is no longer supported. We suggest that modules either limit searching to non-null strings or accept that the source will require version variants.
+
+-  The stack pseudo-index `LUA_GLOBALSINDEX` has been removed. Modules must either get the global environment from the registry entry `LUA_RIDX_GLOBALS` or use the `lua_getglobal` API call.  All current global references in the NodeMCU library modules now use `lua_getglobal` where necessary.
+
+-  Shared userdata and table upvalues.  Lua 5.3 now supports the sharing of GCObjects such as userdata and tables as upvalue between C functions using `lua_getupvalue` and `lua_setupvalue`.  This has changed from 5.1 and we suggest that modules either avoid using these API calls or accept that the source will require version variants.
+
+-  The environment support has changed from Lua 5.1 to 5.3.  We suggest that modules either avoid using these API calls or accept that the source will require version variants.
+
+  -  The coroutine yield / resume API support has changed in Lua 5.3 to support yield and resume across C function calls.  We suggest that modules either avoid using these API calls or accept that the source will require version variants. 
+

docs/nodemcu-pil.md

@@ -0,0 +1,42 @@
+# Programming in NodeMCU
+
+The standard Lua runtime offers support for both Lua modules that can define multiple Lua functions and properties in an encapsulating table as described in the [Lua 5.3 Reference Manual](https://www.lua.org/manual/5.3/) \("**LRM**") and specifically in [LRM Section 6.3](https://www.lua.org/manual/5.3/manual.html#6.3).  Lua also provides a C API to allow developers to implement modules in compiled C.
+
+NodeMCU developers are also able to develop and incorporate their own C modules into their own firmware build using this standard API, although we encourage developers to download the standard [Lua Reference Manual](https://www.lua.org/manual/) and also buy of copy of Roberto Ierusalimschy's Programming in Lua edition 4 \("**PiL**").  The NodeMCU implementation extends standard Lua as documented in the [NodeMCU Reference Manual](nodemcu-lrm.md) \("**NRM**").
+
+Those developers who wish to develop or to modify existing C modules should have access to the LRM, PiL and NRM and familiarise themselves with these references.  These are the primary references; and this document does not repeat this content, but rather provide some NodeMCU-specific information to supplement it.
+
+From a perspective of developing C modules, there is very little difference from that of developing modules in standard Lua.  All of the standard Lua library modules (`bit`, `coroutine`, `debug`, `math`, `string`, `table`, `utf8`) use the C API for Lua and the NodeMCU versions have been updated to use NRM extensions. so their source code is available for browsing and using as style template (see the corresponding `lXXXlib.c` file in GitHub [NodeMCU lua53](../app/lua53) folder).
+
+The main functional change is that NodeMCU supports a read-only subclass of the `Table` type, known as a **`ROTable`**, which can be statically declared within the module source using static `const` declarations.  There are also limitations on the valid types for ROTable keys and value in order to ensure that these are consistent with static declaration; and hence ROTables are stored in code space (and therefore in flash memory on the IoT device). Hence unlike standard Lua tables, ROTables do not take up RAM resources. 
+
+Also unlike standard Lua, two global ROTables are used for the registration of C modules.  Again, static declaration macros plus linker "magic" (use of make filters plus linker _section_ directives) result in the marshalling of these ROTables during the make process, and because this is all ROTable based, the integration of modules into the firmware builds and their access from executing Lua applications depends on code space rather than RAM-based data structures. 
+
+Note that dynamic loading of C modules is not supported within the ESP SDK, so any library registration must be compiled into the source used in the firmware build.  Our approach is simple, flexible and avoids the RAM overheads of the standard Lua approach.  The special ROTable **`ROM`** is core to this approach.  The global environment table has an `__index` metamethod referencing this ROM table.  Hence, any non-raw lookups against the global table will also resolve against ROM.  All base Lua functions (such as `print`) and any C libraries (written to NodeMCU standards) have an entry in the ROM table and hence have global visibility.  This approach does not prevent developers use of standard Lua mechanisms, but rather it offers a simple low RAM use alternative.
+
+The `NODEMCU_MODULE` macro is used in each module to register it in an entry in the **`ROM`** ROTable.  It also adds a entry in the second (hidden) **`ROMentry`** ROTable.
+-  All `ROM` entries will resolve globally
+-  The Lua runtime scans the `ROMentry` ROTable during its start up, and it will execute any non-NULL `CFunction` values in this table.  This enables C modules to hook in any one-time start-up functions if they are needed.
+
+Note that the standard `make` will include any modules found in the `app/modules` folder within a firmware build _if_ the corresponding `LUA_USE_MODULES_modname` macro has been defined. These defines are conventionally set in a common include file `user_modules.h`, and this practice is mandated for any user-submitted modules that are added to to the NodeMCU distribution.  However, this does not prevent developers adding their own local modules to the `app/modules` folder and simply defining the corresponding  `LUA_USE_MODULES_modname` inline.
+
+This macro + linker approach renders the need for `luaL_reg` declarations and use of `luaL_openlib()` unnecessary, and these are not permitted in project-adopted `app/modules` files.
+
+Hence a NodeMCU C library module typically has a standard layout that parallels that of the standard Lua library modules and uses the same C API to access the Lua runtime:
+
+-  A `#ìnclude` block to resolve access to external resources.  All modules will include entries for `"module.h"`, and `"lauxlib.h"`.  They should _not_ reference any other `lXXX.h` includes from the Lua source directory as these are private to the Lua runtime. These may be followed by C standard runtime includes, external application libraries and any SDK resource headers needed.  Note that whilst we recommend using the C standard runtime API for `<stdlib.h>` etc., the SDK only implements a poorly documented subset of this API, so be aware that you might get linker errors and in which case you might need to recode some calls if you are using non-implemented functions.
+
+-  The only external interface to a C module should be via the Lua runtime and its `NODEMCU_MODULE` hooks.  Therefore all functions and resources should be declared `static` and be private to the module. These can be ordered as the developer wishes, subject of course to the need for appropriate forward declarations to comply with C scoping rules.
+
+-  Module methods will typically employ a Lua standard `static int somefunc (lua_State *L) { ... }` template.
+-  ROTables are typically declared at the end of the module to minimise the need for forward references and use the `LROT` macros described in the NRM.  Note that ROTables only support static declaration of string keys and the value types: C function, Lightweight userdata, Numeric, ROTable.  ROTables can also have ROTable metatables.
+   -  Whilst the ROTable search algorithm is a simply linear scan of the ROTable entries, the runtime also maintains a LRU cache of ROTable accesses, so typically over 95% of ROTable accesses bypass the linear scan and do a direct access to the appropriate entry.
+   -  ROTables are also reasonable lightweight and well integrated into the Lua runtime, so the normal metamethod processing works well.  This means that developers can use the `__index` method to implement other key and value typed entries through an index function.
+-  NodeMCU modules are intended to be compilable against both our Lua 5.1 and Lua 5.3 runtimes.  The NRM discusses the implications and constraints here.  However note that:
+   -  We have back-ported many new Lua 5.3 features into the NodeMCU Lua 5.1 API, so in general you can use the 5.3 API to code your modules.  Again the NRM notes the exceptions where you will either need variant code or to decide to limit yourself to the the 5.3 runtime. In this last case the simplest approach is to `#if LUA_VERSION_NUM != 503` to disable the 5.3 content so that 5.1 build can compile and link.   Note that all modules currently in the `app/modules` folder will compile against and execute within both the Lua 5.1 and the 5.3 environments.
+   -  Lua 5.3 uses a 32-bit representation for all numerics with separate subtypes for integer (stored as a 32 bit signed integer) and float (stored as 32bit single precision float).  This achieves the same RAM storage density as Lua 5.1 integer builds without the loss of use of floating point when convenient.  We have therefore decided that there is no benefit in having a separate Integer 5.3 build variant.
+ -  We recommend that developers make use of the full set of `luaL_` API calls to minimise code verbosity.  We have also added a couple of registry access optimisations that both simply and improve runtime performance when using the Lua registry for callback support.
+    - `luaL_reref()` replaces an existing registry reference in place (or creates a new one if needed).  Less code and faster execution than a `luaL_unref()` plus `luaL_ref()` construct.
+    - `luaL_unref2()` does the unref and set the static int hook to `LUA_NOREF`.
+
+Rather than include simple examples of module templates, we suggest that you review the modules in our GitHub repository, such as the [`utf8`](../app/lua53/lutf8lib.c) library.  Note that whilst all of the existing modules in `app/modules` folder compile and work, we plan to do a clean up of the core modules to ensure that they conform to best practice.

mkdocs.yml

@@ -32,15 +32,19 @@ pages:
     - Uploading code: 'upload.md'
     - Compiling code: 'compiling.md'
     - Support: 'support.md'
+- Reference:
+    - NodeMCU Language Reference Manual: 'nodemcu-lrm.md'
+    - Programming in NodeMCU: 'nodemcu-pil.md'
 - FAQs:
     - Lua Developer FAQ: 'lua-developer-faq.md'
     - Extension Developer FAQ: 'extn-developer-faq.md'
     - Hardware FAQ: 'hardware-faq.md'
 - Whitepapers:
+    - Lua 5.3 Support: 'lua53.md'
+    - Lua Flash Store (LFS): 'lfs.md'
+    - Lua Compact Debug (LCD): 'lcd.md'
     - Filesystem on SD card: 'sdcard.md'
     - Internal filesystem: 'spiffs.md'
-    - Lua Compact Debug (LCD): 'lcd.md'
-    - Lua Flash Store (LFS): 'lfs.md'
 - Lua Modules:
     - 'Lua modules directory': 'lua-modules/README.md'
     - 'bh1750': 'lua-modules/bh1750.md'

msvc/luac-cross/luac-cross.vcxproj

@@ -193,6 +193,7 @@
     <ClCompile Include="..\..\app\lua\llex.c" />
     <ClCompile Include="..\..\app\lua\lmathlib.c" />
     <ClCompile Include="..\..\app\lua\lmem.c" />
+    <ClCompile Include="..\..\app\lua\lnodemcu.c" />
     <ClCompile Include="..\..\app\lua\loadlib.c" />
     <ClCompile Include="..\..\app\lua\lobject.c" />
     <ClCompile Include="..\..\app\lua\lopcodes.c" />

msvc/luac-cross/luac-cross.vcxproj.filters

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
     <Filter Include="Source Files">
@@ -132,6 +132,9 @@
     <ClCompile Include="..\..\app\lua\lzio.c">
       <Filter>app\lua</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\app\lua\lnodemcu.c">
+      <Filter>app\lua</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\app\uzlib\uzlib.h">