Merge remote-tracking branch 'origin/dev-esp32' into ble

.github/workflows/build.yml

@@ -15,14 +15,14 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: recursive
       - name: Prepare cache key
         run: git rev-parse HEAD:sdk/esp32-esp-idf > idf.rev
         shell: bash
       - name: Cache Espressif tools
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.espressif
           key: ${{ runner.os }}-espressif-tools-${{ hashFiles('idf.rev') }}
@@ -58,7 +58,7 @@ jobs:
           echo lua_build_opts="$(expr "$(./build/luac_cross/luac.cross -v)" : '.*\[\(.*\)\]')" >> $GITHUB_ENV
         shell: bash
       - name: Upload luac.cross
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ success() }}
         with:
           name: luac.cross-${{ env.lua_build_opts }}-${{ matrix.target }}

.gitignore

@@ -8,7 +8,6 @@ build/
 app/
 components/*/.output/
 tools/toolchains
-extmods.ini
 .ccache
 bin
 

CMakeLists.txt

@@ -1,4 +1,11 @@
 cmake_minimum_required(VERSION 3.5)
+
+if(DEFINED ENV{EXTRA_COMPONENT_DIRS})
+  # Turn space-separated, quote-aware environment var into CMake list
+  separate_arguments(
+    EXTRA_COMPONENT_DIRS UNIX_COMMAND "$ENV{EXTRA_COMPONENT_DIRS}")
+endif()
+
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(nodemcu)
 

Makefile

@@ -21,11 +21,3 @@ all:
 	$(IDF_PATH)/tools/idf.py $(IDFPY_ARGS) "$@"
 
 endif
-
-# FIXME - needs updating to work in IDF4
-#
-#extmod-update:
-#	@tools/extmod/extmod.sh update
-#
-#extmod-clean:
-##	@tools/extmod/extmod.sh clean

components/base_nodemcu/include/module.h

@@ -52,25 +52,3 @@
     ROTable_entry MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(CONFIG_NODEMCU_CMODULE_,cfgname))) \
     = {luaname, LRO_ROVAL(map)}
 #endif
-
-
-// helper stringing macros
-#define xstr(s) str(s)
-#define str(s) #s
-
-// EXTMODNAME is injected by the generated component.mk
-#ifdef EXTMODNAME
-#define MODNAME xstr(EXTMODNAME)
-#else
-#define MODNAME "module"
-#endif
-
-// use NODEMCU_MODULE_METATABLE() to generate a unique metatable name for your objects:
-#define NODEMCU_MODULE_METATABLE() MODULE_EXPAND_(MODNAME xstr(__COUNTER__))
-
-// NODEMCU_MODULE_STD() defines the entry points for an external module:
-#define NODEMCU_MODULE_STD()                                             \
-    static const LOCK_IN_SECTION(libs)                                   \
-        ROTable_entry lua_lib_module = {MODNAME, LRO_FUNCVAL(module_init)}; \
-    const const LOCK_IN_SECTION(rotable)                                       \
-        ROTable_entry MODULE_EXPAND_PASTE_(EXTMODNAME, _entry) = {MODNAME, LRO_ROVAL(module_map)};

components/driver_i2c/i2c_sw_master.c

@@ -112,7 +112,7 @@ void i2c_sw_master_gpio_init(uint8_t sda, uint8_t scl)
 
   gpio_config_t cfg;
 
-  cfg.pin_bit_mask = 1 << sda | 1 << scl;
+  cfg.pin_bit_mask = 1ULL << sda | 1ULL << scl;
   cfg.mode = GPIO_MODE_INPUT_OUTPUT_OD;
   cfg.pull_up_en = GPIO_PULLUP_ENABLE;
   cfg.pull_down_en = GPIO_PULLDOWN_DISABLE;

components/lua/Kconfig

@@ -132,4 +132,35 @@ menu "Lua configuration"
       select NODEMCU_CMODULE_UART
       select LUA_BUILTIN_DEBUG
 
+    choice LUA_INIT_STRING
+        prompt "Boot command"
+        default LUA_INIT_STRING_INIT_LUA
+        help
+            Command to run on boot. This can be a .lua file, an LFS module, or
+            any valid Lua expression. By default init.lua is loaded and run
+            from the SPIFFS filesystem.
+        config LUA_INIT_STRING_INIT_LUA
+            bool "init.lua from SPIFFS"
+        config LUA_INIT_STRING_INIT_LFS
+            bool "init module from LFS"
+        config LUA_INIT_STRING_CUSTOM
+            bool "Custom"
+    endchoice
+
+    config LUA_INIT_STRING_CUSTOM_STRING
+        string "Custom boot command" if LUA_INIT_STRING_CUSTOM
+        default ""
+        help
+            Run a custom command on boot.
+
+            Specify @filename.lua to load "filename.lua" from SPIFFS.
+            Specify node.LFS.get('foo')() to load the module "foo" from LFS.
+            Or specify any other valid Lua expression to execute that on boot.
+
+    config LUA_INIT_STRING
+        string
+        default "@init.lua" if LUA_INIT_STRING_INIT_LUA
+        default "node.LFS.get('init')()" if LUA_INIT_STRING_INIT_LFS
+        default LUA_INIT_STRING_CUSTOM_STRING if LUA_INIT_STRING_CUSTOM
+
 endmenu

components/lua/common/lfs.c

@@ -31,8 +31,8 @@ bool lfs_get_location(lfs_location_info_t *out)
     return false; // Nothing to do if no LFS partition available
 
   out->size      = part->size; // in bytes
-  out->addr_mem  = spi_flash_phys2cache(out->addr_phys, SPI_FLASH_MMAP_DATA);
   out->addr_phys = part->address;
+  out->addr_mem  = spi_flash_phys2cache(out->addr_phys, SPI_FLASH_MMAP_DATA);
   if (!out->addr_mem) { // not already mmap'd, have to do it ourselves
     spi_flash_mmap_handle_t ignored;
     esp_err_t err = spi_flash_mmap(

components/lua/lua-5.1/lauxlib.h

@@ -179,6 +179,7 @@ LUALIB_API int  (luaL_posttask) ( lua_State* L, int prio );
 #define  LUA_TASK_LOW    0
 #define  LUA_TASK_MEDIUM 1
 #define  LUA_TASK_HIGH   2
+LUALIB_API int (luaL_totoggle) (lua_State *L, int idx);
 
 /* }====================================================== */
 

components/lua/lua-5.1/lnodemcu.c

@@ -257,3 +257,20 @@ LUA_API int lua_pushlfsindex (lua_State *L) {
   return p ? LUA_TFUNCTION : LUA_TNIL;
 }
 #endif
+
+
+/* luaL_totoggle provides lenient boolean interpretation for feature toggles
+ * true, 1 => true
+ * false, 0, nil => false
+ */
+LUALIB_API int luaL_totoggle(lua_State *L, int idx)
+{
+  if (lua_isboolean(L, idx))
+    return lua_toboolean(L, idx);
+  else if (lua_isnoneornil(L, idx))
+    return 0;
+  else if (lua_isnumber(L, idx))
+    return lua_tonumber(L, idx) != 0;
+  else
+    return luaL_error(L, "unexpected type");
+}

components/lua/lua-5.3/lauxlib.c

@@ -1177,3 +1177,20 @@ LUALIB_API int luaL_pcallx (lua_State *L, int narg, int nres) {
   return status;
 }
 #endif
+
+
+/* luaL_totoggle provides lenient boolean interpretation for feature toggles
+ * true, 1 => true
+ * false, 0, nil => false
+ */
+LUALIB_API bool luaL_totoggle(lua_State *L, int idx)
+{
+  if (lua_isboolean(L, idx))
+    return lua_toboolean(L, idx);
+  else if (lua_isnoneornil(L, idx))
+    return false;
+  else if (lua_isnumber(L, idx))
+    return lua_tonumber(L, idx) != 0;
+  else
+    return luaL_error(L, "unexpected type");
+}

components/lua/lua-5.3/lauxlib.h

@@ -295,6 +295,7 @@ LUALIB_API void (luaL_lfsreload) (lua_State *L);
 LUALIB_API int  (luaL_posttask) (lua_State* L, int prio);
 LUALIB_API int  (luaL_pcallx) (lua_State *L, int narg, int nres);
 #define luaL_pushlfsmodule(l) lua_pushlfsfunc(L)
+LUALIB_API bool (luaL_totoggle) (lua_State *L, int idx);
 
 /* }============================================================ */
 

components/lua/lua-5.3/lua.c

@@ -30,11 +30,7 @@
 #endif
 
 #ifndef LUA_INIT_STRING
-# if defined(CONFIG_NODEMCU_EMBED_LFS)
-#  define LUA_INIT_STRING "node.LFS.get('init')()"
-# else
-#  define LUA_INIT_STRING "@init.lua"
-# endif
+# define LUA_INIT_STRING CONFIG_LUA_INIT_STRING
 #endif
 
 #if !defined(STARTUP_COUNT)

components/modules/CMakeLists.txt

@@ -24,6 +24,7 @@ set(module_srcs
   "otaupgrade.c"
   "ow.c"
   "pipe.c"
+  "rtcmem.c"
   "qrcodegen.c"
   "sigma_delta.c"
   "sjson.c"
@@ -52,18 +53,24 @@ if(IDF_TARGET STREQUAL "esp32")
     "eth.c"
     "i2s.c"
     "pulsecnt.c"
+    "rmt.c"
     "sdmmc.c"
     "touch.c"
   )
 elseif(IDF_TARGET STREQUAL "esp32s2")
   list(APPEND module_srcs
     "dac.c"
+    "rmt.c"
+    "pulsecnt.c"
   )
 elseif(IDF_TARGET STREQUAL "esp32s3")
   list(APPEND module_srcs
+    "rmt.c"
+    "pulsecnt.c"
   )
 elseif(IDF_TARGET STREQUAL "esp32c3")
   list(APPEND module_srcs
+    "rmt.c"
   )
 endif()
 

components/modules/Kconfig

@@ -226,6 +226,20 @@ menu "NodeMCU modules"
             Includes the QR Code Generator from
             https://www.nayuki.io/page/qr-code-generator-library
 
+    config NODEMCU_CMODULE_RMT
+        bool "Remote Control pulse generator/receiver"
+        default "n"
+        help
+            Includes the rmt module to use the ESP32's built-in 
+            remote control hardware.
+
+    config NODEMCU_CMODULE_RTCMEM
+        bool "Access to a limited amount of battery backed memory (rtcmem)"
+        default "n"
+        help
+            Includes the rtcmem module to allow access to 
+            the battery backed memory.
+
     config NODEMCU_CMODULE_SDMMC
         depends on IDF_TARGET_ESP32
         bool "SD-MMC module"

components/modules/file.c

@@ -11,6 +11,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <dirent.h>
+#include <errno.h>
 
 static const char *default_fs_label =
   ((CONFIG_NODEMCU_DEFAULT_SPIFFS_LABEL &&
@@ -37,20 +38,33 @@ static int file_format( lua_State* L )
 // Lua: list()
 static int file_list( lua_State* L )
 {
-  const char *dirname = luaL_optstring(L, 1, ".");
-  
+  const char *dirname = luaL_optstring(L, 1, NULL);
+
   DIR *dir;
-  if ((dir = opendir(dirname))) {
+  if ((dir = opendir(dirname ? dirname : "/"))) {
     lua_newtable( L );
     struct dirent *e;
     while ((e = readdir(dir))) {
-      char *fname;
-      asprintf(&fname, "%s/%s", dirname, e->d_name);
-      if (!fname)
-        return luaL_error(L, "no memory");
+      char *fname = NULL;
+      if (dirname) {
+        asprintf(&fname, "%s/%s", dirname, e->d_name);
+        if (!fname) {
+          closedir(dir);
+          return luaL_error(L, "no memory");
+        }
+      } else {
+        fname = e->d_name;
+      }
       struct stat st = { 0, };
-      stat(fname, &st);
-      free(fname);
+      int err = stat(fname, &st);
+      if (err) {
+        // We historically ignored this error, so just warn (although it
+        // shouldn't really happen now).
+        NODE_ERR("Failed to stat %s err=%d\n", fname, err);
+      }
+      if (dirname) {
+        free(fname);
+      }
       lua_pushinteger(L, st.st_size);
       lua_setfield(L, -2, e->d_name);
     }
@@ -116,6 +130,29 @@ static int file_fsinfo( lua_State* L )
 }
 
 
+static int file_mkdir( lua_State *L )
+{
+  const char *name = luaL_checkstring(L, 1);
+  unsigned mode = luaL_optint(L, 2, 0777);
+  if (mkdir(name, mode) != 0) {
+    return
+      luaL_error(L, "failed to create directory '%s'; code %d", name, errno);
+  }
+  return 0;
+}
+
+
+static int file_rmdir( lua_State *L )
+{
+  const char *name = luaL_checkstring(L, 1);
+  if (rmdir(name) != 0) {
+    return
+      luaL_error(L, "failed to remove directory '%s'; code %d", name, errno);
+  }
+  return 0;
+}
+
+
 // Module function map
 LROT_BEGIN(file, NULL, 0)
   LROT_FUNCENTRY( list,      file_list )
@@ -124,6 +161,8 @@ LROT_BEGIN(file, NULL, 0)
   LROT_FUNCENTRY( rename,    file_rename )
   LROT_FUNCENTRY( exists,    file_exists )
   LROT_FUNCENTRY( fsinfo,    file_fsinfo )
+  LROT_FUNCENTRY( mkdir,     file_mkdir )
+  LROT_FUNCENTRY( rmdir,     file_rmdir )
 LROT_END(file, NULL, 0)
 
 

components/modules/ledc.c

@@ -30,6 +30,8 @@ static int lledc_new_channel( lua_State *L )
 
   ledc_timer.timer_num = opt_checkint_range(L, "timer", -1, 0, LEDC_TIMER_MAX-1);
 
+  ledc_timer.clk_cfg = LEDC_AUTO_CLK;
+
   /* Setup channel */
   ledc_channel_config_t channel_config = {
     .speed_mode = ledc_timer.speed_mode,
@@ -43,6 +45,8 @@ static int lledc_new_channel( lua_State *L )
 
   channel_config.gpio_num = opt_checkint_range(L, "gpio", -1, 0, GPIO_NUM_MAX-1);
 
+  channel_config.flags.output_invert = opt_checkbool(L, "invert", 0);
+
   lua_settop(L, top);
 
   esp_err_t timerErr = ledc_timer_config(&ledc_timer);

components/modules/node.c

@@ -111,9 +111,9 @@ static int node_bootreason( lua_State *L)
 #endif
 #if defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32S3) || defined(CONFIG_IDF_TARGET_ESP32C3)
     case GLITCH_RTC_RESET:
+    case EFUSE_RESET:
 #endif
 #if defined(CONFIG_IDF_TARGET_ESP32S3) || defined(CONFIG_IDF_TARGET_ESP32C3)
-    case EFUSE_RESET:
     case USB_UART_CHIP_RESET:
     case USB_JTAG_CHIP_RESET:
     case POWER_GLITCH_RESET:
@@ -603,7 +603,7 @@ static int writer(lua_State* L, const void* p, size_t size, void* u)
   if (!file)
     return 1;
 
-  if (size != 0 && (size != fwrite((const char *)p, size, 1, file)) )
+  if (size != 0 && (fwrite((const char *)p, size, 1, file) != 1) )
     return 1;
 
   return 0;

components/modules/project_include.cmake

@@ -0,0 +1,22 @@
+# Helper functions for external module registration.
+# - extmod_register_conditional() for modules with a Kconfig option
+# - extmod_register_unconditional() for always-enabled modules
+
+function(extmod_register_conditional confname)
+  if (${CONFIG_NODEMCU_CMODULE_${confname}})
+    # If the module is enabled in menuconfig, add the linker option
+    # "-u <confname>_module_selected1" to make the linker include this
+    # module. See components/core/include/module.h for further details
+    # on how this works.
+    message("Including external module ${confname}")
+    target_link_libraries(${COMPONENT_LIB} "-u ${confname}_module_selected1")
+  endif()
+endfunction()
+
+function(extmod_register_unconditional confname)
+  message("Including external module ${confname}")
+  # The module macros rely on the presence of a CONFIG_NODEMCU_CMODULE_XXX
+  # def, so we have to add it explicitly as it won't be coming from Kconfig
+  target_compile_options(${COMPONENT_LIB} PRIVATE "-DCONFIG_NODEMCU_CMODULE_${confname}")
+  target_link_libraries(${COMPONENT_LIB} "-u ${confname}_module_selected1")
+endfunction()

components/modules/rmt.c

@@ -0,0 +1,381 @@
+// Module for working with the rmt driver
+
+#include <string.h>
+#include "module.h"
+#include "lauxlib.h"
+#include "platform.h"
+#include "platform_rmt.h"
+#include "task/task.h"
+
+#include "driver/rmt.h"
+
+#include "common.h"
+
+typedef struct {
+  bool tx;
+  int channel;
+  int cb_ref;
+  struct _lrmt_cb_params *rx_params;
+} *lrmt_channel_t;
+
+typedef struct _lrmt_cb_params {
+  bool dont_call;
+  bool rx_shutting_down;
+  rmt_channel_t channel;
+  int cb_ref;
+  int data_ref;
+  int rc;
+  size_t len;
+  rmt_item32_t *data;
+} lrmt_cb_params;
+
+static task_handle_t cb_task_id;
+
+static int get_divisor(lua_State *L, int index) {
+  int bittime = luaL_checkinteger(L, index);
+
+  int divisor = bittime / 12500;   // 80MHz clock
+
+  luaL_argcheck(L, divisor >= 1 && divisor <= 255, index, "Bit time out of range");
+
+  return divisor;
+}
+
+static int configure_channel(lua_State *L, rmt_config_t *config, rmt_mode_t mode) {
+  lrmt_channel_t ud = (lrmt_channel_t)lua_newuserdata(L, sizeof(*ud));
+  if (!ud) return luaL_error(L, "not enough memory");
+  memset(ud, 0, sizeof(*ud));
+  luaL_getmetatable(L, "rmt.channel");
+  lua_setmetatable(L, -2);
+
+  // We have allocated the channel -- must free it if the rest of this method fails
+  int channel = platform_rmt_allocate(1, mode);
+
+  if (channel < 0) {
+    return luaL_error(L, "no spare RMT channel");
+  }
+
+  config->channel = channel;
+
+  ud->channel = channel;
+  ud->tx = mode == RMT_MODE_TX;
+
+  esp_err_t rc = rmt_config(config);
+  if (rc) {
+    platform_rmt_release(config->channel);
+    return luaL_error(L, "Failed to configure RMT");
+  }
+
+  rc = rmt_driver_install(config->channel, 1000, 0);
+  if (rc) {
+    platform_rmt_release(config->channel);
+    return luaL_error(L, "Failed to install RMT driver");
+  }
+
+  return 1;
+}
+
+static int lrmt_txsetup(lua_State *L) {
+  int gpio = luaL_checkinteger(L, 1);
+  int divisor = get_divisor(L, 2);
+
+  // We will set the channel later
+  rmt_config_t config = RMT_DEFAULT_CONFIG_TX(gpio, 0);
+  config.clk_div = divisor;
+
+  if (lua_type(L, 3) == LUA_TTABLE) {
+    lua_getfield(L, 3, "carrier_hz");
+    int hz = lua_tointeger(L, -1);
+    if (hz) {
+      config.tx_config.carrier_freq_hz = hz;
+      config.tx_config.carrier_en = true;
+    }
+    lua_pop(L, 1);
+
+    lua_getfield(L, 3, "carrier_duty");
+    int duty = lua_tointeger(L, -1);
+    if (duty) {
+      config.tx_config.carrier_duty_percent = duty;
+    }
+    lua_pop(L, 1);
+
+    lua_getfield(L, 3, "idle_level");
+    if (!lua_isnil(L, -1)) {
+      int level = lua_tointeger(L, -1);
+      config.tx_config.idle_level = level;
+      config.tx_config.idle_output_en = true;
+    }
+    lua_pop(L, 1);
+
+    lua_getfield(L, 3, "invert");
+    if (lua_toboolean(L, -1)) {
+      config.flags |= RMT_CHANNEL_FLAGS_INVERT_SIG;
+    }
+    lua_pop(L, 1);
+  }
+
+  configure_channel(L, &config, RMT_MODE_TX);
+  lua_pushinteger(L, divisor * 12500);
+  return 2;
+}
+
+static int lrmt_rxsetup(lua_State *L) {
+  int gpio = luaL_checkinteger(L, 1);
+  int divisor = get_divisor(L, 2);
+
+  // We will set the channel later
+  rmt_config_t config = RMT_DEFAULT_CONFIG_RX(gpio, 0);
+  config.clk_div = divisor;
+  config.rx_config.idle_threshold = 65535;
+
+  if (lua_type(L, 3) == LUA_TTABLE) {
+    lua_getfield(L, 3, "invert");
+    if (lua_toboolean(L, -1)) {
+      config.flags |= RMT_CHANNEL_FLAGS_INVERT_SIG;
+    }
+    lua_pop(L, 1);
+
+    lua_getfield(L, 3, "filter_ticks");
+    if (!lua_isnil(L, -1)) {
+      int ticks = lua_tointeger(L, -1);
+      if (ticks < 0 || ticks > 255) {
+        return luaL_error(L, "filter_ticks must be in the range 0 - 255");
+      }
+      config.rx_config.filter_ticks_thresh = ticks;
+      config.rx_config.filter_en = true;
+    }
+    lua_pop(L, 1);
+
+    lua_getfield(L, 3, "idle_threshold");
+    if (!lua_isnil(L, -1)) {
+      int threshold = lua_tointeger(L, -1);
+      if (threshold < 0 || threshold > 65535) {
+        return luaL_error(L, "idle_threshold must be in the range 0 - 65535");
+      }
+      config.rx_config.idle_threshold = threshold;
+    }
+    lua_pop(L, 1);
+  }
+
+  configure_channel(L, &config, RMT_MODE_RX);
+  lua_pushinteger(L, divisor * 12500);
+  return 2;
+}
+
+static void free_transmit_wait_params(lua_State *L, lrmt_cb_params *p) {
+  if (!p->data) {
+    luaL_unref(L, LUA_REGISTRYINDEX, p->cb_ref);
+    luaL_unref(L, LUA_REGISTRYINDEX, p->data_ref);
+  }
+  free(p);
+}
+
+static void handle_receive(void *param) {
+  lrmt_cb_params *p = (lrmt_cb_params *) param;
+
+  RingbufHandle_t rb = NULL;
+
+  //get RMT RX ringbuffer
+  rmt_get_ringbuf_handle(p->channel, &rb);
+  // Start receive
+  rmt_rx_start(p->channel, true);
+  while (!p->rx_shutting_down) {
+    size_t length = 0;
+    rmt_item32_t *items = NULL;
+
+    items = (rmt_item32_t *) xRingbufferReceive(rb, &length, 50 / portTICK_PERIOD_MS);
+    if (items && length) {
+      lrmt_cb_params *rx_params = malloc(sizeof(lrmt_cb_params) + length);
+      if (rx_params) {
+        memset(rx_params, 0, sizeof(*rx_params));
+	memcpy(rx_params + 1, items, length);
+        rx_params->cb_ref = p->cb_ref;
+	rx_params->data = (void *) (rx_params + 1);
+	rx_params->len = length;
+        rx_params->channel = p->channel;
+        task_post_high(cb_task_id, (task_param_t) rx_params);
+      } else {
+	printf("Unable allocate receive data memory\n");
+      }
+    }
+    if (items) {
+      vRingbufferReturnItem(rb, (void *) items);
+    }
+  }
+
+  p->dont_call = true;
+  task_post_high(cb_task_id, (task_param_t) p);
+
+  /* Destroy this task */
+  vTaskDelete(NULL);
+}
+
+static int lrmt_on(lua_State *L) {
+  lrmt_channel_t ud = (lrmt_channel_t)luaL_checkudata(L, 1, "rmt.channel");
+  if (ud->tx) {
+    return luaL_error(L, "Cannot receive on a TX channel");
+  }
+
+  luaL_argcheck(L, !strcmp(lua_tostring(L, 2), "data") , 2, "Must be 'data'");
+
+  luaL_argcheck(L, lua_type(L, 3) == LUA_TFUNCTION, 3, "Must be a function");
+
+  if (ud->rx_params) {
+    return luaL_error(L, "Can only call 'on' once");
+  }
+
+  // We have a callback
+  lrmt_cb_params *params = (lrmt_cb_params *) malloc(sizeof(*params));
+  if (!params) {
+    return luaL_error(L, "Cannot allocate memory");
+  }
+  memset(params, 0, sizeof(*params));
+
+  lua_pushvalue(L, 3);
+  params->cb_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+  ud->rx_params = params;
+  params->channel = ud->channel;
+
+  xTaskCreate(handle_receive, "rmt-rx-receiver", 3000, params, 2, NULL);
+
+  return 0;
+}
+
+static void wait_for_transmit(void *param) {
+  lrmt_cb_params *p = (lrmt_cb_params *) param;
+  esp_err_t rc = rmt_wait_tx_done(p->channel, 10000 / portTICK_PERIOD_MS);
+
+  p->rc = rc;
+  task_post_high(cb_task_id, (task_param_t) p);
+
+  /* Destroy this task */
+  vTaskDelete(NULL);
+}
+
+static int lrmt_send(lua_State *L) {
+  lrmt_channel_t ud = (lrmt_channel_t)luaL_checkudata(L, 1, "rmt.channel");
+  if (!ud->tx) {
+    return luaL_error(L, "Cannot send on an RX channel");
+  }
+  int string_index = 2;
+
+  if (lua_type(L, 2) == LUA_TTABLE) {
+    lua_getfield(L, 2, "concat");
+    lua_pushvalue(L, 2);
+    lua_pushstring(L, "");
+    lua_call(L, 2, 1);
+    string_index = -1;
+  }
+
+  size_t len;
+  const char *data = lua_tolstring(L, string_index, &len);
+  if (!data || !len) {
+    return 0;
+  }
+
+  if (len & 1) {
+    return luaL_error(L, "Length must be a multiple of 2");
+  }
+
+  if (len & 3) {
+    // Just tack on a "\0\0" -- this is needed as the hardware can
+    // only deal with multiple of 4 bytes.
+    luaL_Buffer b;
+    luaL_buffinit(L, &b);
+    luaL_addlstring(&b, data, len);
+    luaL_addlstring(&b, "\0\0", 2);
+    luaL_pushresult(&b);
+    data = lua_tolstring(L, -1, &len);
+    string_index = -1;
+  }
+
+  bool wait_for_done = true;
+
+  if (lua_type(L, 3) == LUA_TFUNCTION) {
+    // We have a callback
+    lrmt_cb_params *params = (lrmt_cb_params *) malloc(sizeof(*params));
+    if (!params) {
+      return luaL_error(L, "Cannot allocate memory");
+    }
+    memset(params, 0, sizeof(*params));
+
+    params->channel = ud->channel;
+
+    lua_pushvalue(L, 3);
+    params->cb_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+    lua_pushvalue(L, string_index);
+    params->data_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+    xTaskCreate(wait_for_transmit, "rmt-tx-waiter", 1024, params, 2, NULL);
+    wait_for_done = false;
+  }
+
+  // We want to transmit it
+  rmt_write_items(ud->channel, (rmt_item32_t *) data, len / sizeof(rmt_item32_t), wait_for_done);
+  return 0;
+}
+
+static int lrmt_close(lua_State *L) {
+  lrmt_channel_t ud = (lrmt_channel_t)luaL_checkudata(L, 1, "rmt.channel");
+
+  if (ud->channel >= 0) {
+    if (ud->rx_params) {
+      // We need to stop the listening task
+      ud->rx_params->rx_shutting_down = true;
+    } else {
+      rmt_driver_uninstall(ud->channel);
+      platform_rmt_release(ud->channel);
+    }
+    ud->channel = -1;
+  }
+
+  return 0;
+}
+
+static void cb_task(task_param_t param, task_prio_t prio) {
+  lrmt_cb_params *p = (lrmt_cb_params *) param;
+  lua_State *L = lua_getstate();
+
+  if (!p->dont_call) {
+    lua_rawgeti (L, LUA_REGISTRYINDEX, p->cb_ref);
+    if (p->data) {
+      lua_pushlstring(L, (char *) p->data, p->len);
+    } else {
+      lua_pushinteger(L, p->rc);
+    }
+
+    int res = luaL_pcallx(L, 1, 0);
+    if (res) {
+      printf("rmt callback threw an error\n");
+    }
+  }
+
+  if (p->rx_shutting_down) {
+    rmt_driver_uninstall(p->channel);
+    platform_rmt_release(p->channel);
+  }
+
+  free_transmit_wait_params(L, p);
+}
+
+// Module function map
+LROT_BEGIN(rmt_channel, NULL, LROT_MASK_GC_INDEX)
+  LROT_FUNCENTRY( __gc,            lrmt_close )
+  LROT_TABENTRY ( __index,         rmt_channel )
+  LROT_FUNCENTRY( on,              lrmt_on )
+  LROT_FUNCENTRY( close,           lrmt_close )
+  LROT_FUNCENTRY( send,            lrmt_send )
+LROT_END(rmt_channel, NULL, LROT_MASK_GC_INDEX)
+
+LROT_BEGIN(rmt, NULL, LROT_MASK_INDEX)
+  LROT_TABENTRY ( __index,         rmt )
+  LROT_FUNCENTRY( txsetup,         lrmt_txsetup )
+  LROT_FUNCENTRY( rxsetup,         lrmt_rxsetup )
+LROT_END(rmt, NULL, LROT_MASK_INDEX)
+
+int luaopen_rmt(lua_State *L) {
+  luaL_rometatable(L, "rmt.channel", LROT_TABLEREF(rmt_channel));  // create metatable
+  cb_task_id = task_get_id(cb_task);
+  return 0;
+}
+
+NODEMCU_MODULE(RMT, "rmt", rmt, luaopen_rmt);

components/modules/rtcmem.c

@@ -0,0 +1,52 @@
+// Module for RTC user memory access
+
+#include "module.h"
+#include "lauxlib.h"
+#include "esp_attr.h"
+
+#define RTC_USER_MEM_NUM_DWORDS 128
+
+RTC_NOINIT_ATTR uint32_t rtc_memory[RTC_USER_MEM_NUM_DWORDS];
+
+static int rtcmem_read32 (lua_State *L)
+{
+  int idx = luaL_checkinteger (L, 1);
+  int n = (lua_gettop(L) < 2) ? 1 : lua_tointeger (L, 2);
+  if (n == 0 || !lua_checkstack (L, n)) {
+        return 0;
+  }
+
+  int ret  = 0;
+  while (n > 0 && idx >= 0 && idx < RTC_USER_MEM_NUM_DWORDS)
+  {
+    lua_pushinteger (L, rtc_memory[idx++]);
+    --n;
+    ++ret;
+  }
+  return ret;
+}
+
+
+static int rtcmem_write32 (lua_State *L)
+{
+  int idx = luaL_checkinteger (L, 1);
+  int n = lua_gettop (L) - 1;
+  luaL_argcheck (
+    L, idx + n <= RTC_USER_MEM_NUM_DWORDS, 1, "RTC mem would overrun");
+  int src = 2;
+  while (n-- > 0)
+  {
+    rtc_memory[idx++] = (uint32_t) lua_tointeger(L, src++);
+  }
+  return 0;
+}
+
+
+// Module function map
+LROT_BEGIN(rtcmem, NULL, 0)
+  LROT_FUNCENTRY( read32, rtcmem_read32 )
+  LROT_FUNCENTRY( write32, rtcmem_write32 )
+LROT_END(rtcmem, NULL, 0)
+
+
+NODEMCU_MODULE(RTCMEM, "rtcmem", rtcmem, NULL);

components/modules/sodium.c

@@ -17,7 +17,7 @@ static int l_randombytes_random(lua_State *L)
 {
   check_init(L);
   uint32_t ret = randombytes_random();
-  lua_pushnumber(L, (lua_Number)ret);
+  lua_pushinteger(L, (int32_t)ret);
   return 1;
 }
 
@@ -26,7 +26,7 @@ static int l_randombytes_uniform(lua_State *L)
   check_init(L);
   uint32_t upper_bound = (uint32_t)luaL_checkinteger(L, 1);
   uint32_t ret = randombytes_uniform(upper_bound);
-  lua_pushnumber(L, (lua_Number)ret);
+  lua_pushinteger(L, (int32_t)ret);
   return 1;
 }
 

components/modules/uart.c

@@ -10,6 +10,10 @@
 
 static lua_State *gL = NULL;
 
+bool uart_has_on_data_cb(unsigned id){
+  return uart_status[id].receive_rf != LUA_NOREF;
+}
+
 bool uart_on_data_cb(unsigned id, const char *buf, size_t len){
   if(!buf || len==0)
     return false;
@@ -128,23 +132,24 @@ static int uart_on( lua_State* L )
 // Lua: actualbaud = setup( id, baud, databits, parity, stopbits, echo )
 static int uart_setup( lua_State* L )
 {
-  unsigned id, databits, parity, stopbits, echo = 1;
+  unsigned id, databits, parity, stopbits;
   uint32_t baud, res;
   uart_pins_t pins;
+  uart_pins_t* pins_to_use = NULL;
   
+  memset(&pins, 0, sizeof(pins));
   id = luaL_checkinteger( L, 1 );
   MOD_CHECK_ID( uart, id );
   baud = luaL_checkinteger( L, 2 );
   databits = luaL_checkinteger( L, 3 );
   parity = luaL_checkinteger( L, 4 );
   stopbits = luaL_checkinteger( L, 5 );
-  if(id == CONFIG_ESP_CONSOLE_UART_NUM && lua_isnumber(L,6)){
-    echo = lua_tointeger(L,6);
-    if(echo!=0)
-      input_echo = true;
-    else
-      input_echo = false;
-  } else if(id != CONFIG_ESP_CONSOLE_UART_NUM && lua_istable( L, 6 )) {
+  if (!lua_isnoneornil(L, 6)) {
+    if(id == CONFIG_ESP_CONSOLE_UART_NUM){
+      input_echo = luaL_checkinteger(L, 6) > 0;
+    } else {
+      luaL_checktable(L, 6);
+
       lua_getfield (L, 6, "tx");
       pins.tx_pin = luaL_checkint(L, -1);
       lua_getfield (L, 6, "rx");
@@ -165,9 +170,12 @@ static int uart_setup( lua_State* L )
       
       lua_getfield (L, 6, "flow_control");
       pins.flow_control = luaL_optint(L, -1, PLATFORM_UART_FLOW_NONE);
+
+      pins_to_use = &pins;
+    }
   }
 
-  res = platform_uart_setup( id, baud, databits, parity, stopbits, &pins );
+  res = platform_uart_setup( id, baud, databits, parity, stopbits, pins_to_use );
   lua_pushinteger( L, res );
   return 1;
 }

components/modules/wifi.c

@@ -137,6 +137,11 @@ LROT_BEGIN(wifi, NULL, 0)
   LROT_NUMENTRY ( AUTH_WPA2_WPA3_PSK,         WIFI_AUTH_WPA2_WPA3_PSK )
   LROT_NUMENTRY ( AUTH_WAPI_PSK,              WIFI_AUTH_WAPI_PSK )
 
+  LROT_NUMENTRY ( SAE_PWE_UNSPECIFIED,        WPA3_SAE_PWE_UNSPECIFIED )
+  LROT_NUMENTRY ( SAE_PWE_HUNT_AND_PECK,      WPA3_SAE_PWE_HUNT_AND_PECK )
+  LROT_NUMENTRY ( SAE_PWE_HASH_TO_ELEMENT,    WPA3_SAE_PWE_HASH_TO_ELEMENT )
+  LROT_NUMENTRY ( SAE_PWE_BOTH,               WPA3_SAE_PWE_BOTH )
+
   LROT_NUMENTRY ( STR_WIFI_SECOND_CHAN_NONE,  WIFI_SECOND_CHAN_NONE )
   LROT_NUMENTRY ( STR_WIFI_SECOND_CHAN_ABOVE, WIFI_SECOND_CHAN_ABOVE )
   LROT_NUMENTRY ( STR_WIFI_SECOND_CHAN_BELOW, WIFI_SECOND_CHAN_BELOW )

components/modules/wifi_sta.c

@@ -40,6 +40,10 @@
 #include "nodemcu_esp_event.h"
 #include <string.h>
 #include "esp_netif.h"
+#include <math.h>
+
+#define PMF_VAL_AVAILABLE 1
+#define PMF_VAL_REQUIRED  2
 
 static esp_netif_t *wifi_sta = NULL;
 static int scan_cb_ref = LUA_NOREF;
@@ -207,6 +211,20 @@ static int wifi_sta_setip(lua_State *L)
   return 0;
 }
 
+static int wifi_sta_settxpower(lua_State *L)
+{
+  lua_Number max_power = luaL_checknumber(L, 1);
+
+  esp_err_t err = esp_wifi_set_max_tx_power(floor(max_power * 4 + 0.5));
+
+  if (err != ESP_OK)
+    return luaL_error(L, "failed to set transmit power, code %d", err);
+
+  lua_pushboolean(L, err == ESP_OK);
+
+  return 1;
+}
+
 static int wifi_sta_sethostname(lua_State *L)
 {
   size_t l;
@@ -237,16 +255,18 @@ static int wifi_sta_config (lua_State *L)
   if (len > sizeof (cfg.sta.ssid))
     len = sizeof (cfg.sta.ssid);
   strncpy ((char *)cfg.sta.ssid, str, len);
+  lua_pop(L, 1);
 
   lua_getfield (L, 1, "pwd");
   str = luaL_optlstring (L, -1, "", &len);
   if (len > sizeof (cfg.sta.password))
     len = sizeof (cfg.sta.password);
   strncpy ((char *)cfg.sta.password, str, len);
+  lua_pop(L, 1);
 
   lua_getfield (L, 1, "bssid");
   cfg.sta.bssid_set = false;
-  if (lua_isstring (L, -1))
+  if (!lua_isnoneornil(L, -1))
   {
     const char *bssid = luaL_checklstring (L, -1, &len);
     const char *fmts[] = {
@@ -269,18 +289,76 @@ static int wifi_sta_config (lua_State *L)
     if (!cfg.sta.bssid_set)
       return luaL_error (L, "invalid BSSID: %s", bssid);
   }
+  lua_pop(L, 1);
 
   lua_getfield(L, 1, "pmf");
-  if (lua_isnumber(L, -1))
+  if (!lua_isnoneornil(L, -1))
   {
-    int pmf_mode = lua_tointeger(L, -1);
-    if (pmf_mode)
-      cfg.sta.pmf_cfg.capable = true;
-    if (pmf_mode == 2)
-      cfg.sta.pmf_cfg.required = true;
+    int pmf_mode = luaL_checkinteger(L, -1);
+    cfg.sta.pmf_cfg.required = (pmf_mode == PMF_VAL_REQUIRED);
   }
   else
-    cfg.sta.pmf_cfg.capable = true;
+    cfg.sta.pmf_cfg.required = false;
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "channel");
+  if (!lua_isnoneornil(L, -1))
+    cfg.sta.channel = luaL_checkinteger(L, -1);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "scan_method");
+  if (!lua_isnoneornil(L, -1))
+  {
+    static const wifi_scan_method_t vals[] = {
+      WIFI_FAST_SCAN, WIFI_ALL_CHANNEL_SCAN,
+    };
+    static const char *keys[] = { "fast", "all", };
+    cfg.sta.scan_method = vals[luaL_checkoption(L, -1, NULL, keys)];
+  }
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "listen_interval");
+  if (!lua_isnoneornil(L, -1))
+    cfg.sta.listen_interval = luaL_checkinteger(L, -1);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "sort_by");
+  if (!lua_isnoneornil(L, -1))
+  {
+    static const wifi_sort_method_t vals[] = {
+      WIFI_CONNECT_AP_BY_SIGNAL, WIFI_CONNECT_AP_BY_SECURITY,
+    };
+    static const char *keys[] = { "rssi", "authmode", };
+    cfg.sta.sort_method = vals[luaL_checkoption(L, -1, NULL, keys)];
+  }
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "threshold_rssi");
+  if (!lua_isnoneornil(L, -1))
+    cfg.sta.threshold.rssi = luaL_checkinteger(L, -1);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "threshold_authmode");
+  if (!lua_isnoneornil(L, -1))
+    cfg.sta.threshold.authmode = luaL_checkinteger(L, -1);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "rm");
+  cfg.sta.rm_enabled = luaL_totoggle(L, -1);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "btm");
+  cfg.sta.btm_enabled = luaL_totoggle(L, -1);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "mbo");
+  cfg.sta.mbo_enabled = luaL_totoggle(L, -1);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "sae_pwe");
+  if (!lua_isnoneornil(L, -1))
+    cfg.sta.sae_pwe_h2e = luaL_checkinteger(L, -1);
+  lua_pop(L, 1);
 
   SET_SAVE_MODE(save);
   esp_err_t err = esp_wifi_set_config (WIFI_IF_STA, &cfg);
@@ -308,6 +386,7 @@ static int wifi_sta_disconnect (lua_State *L)
 static int wifi_sta_getconfig (lua_State *L)
 {
   wifi_config_t cfg;
+  memset(&cfg, 0, sizeof(cfg));
   esp_err_t err = esp_wifi_get_config (WIFI_IF_STA, &cfg);
   if (err != ESP_OK)
     return luaL_error (L, "failed to get config, code %d", err);
@@ -329,6 +408,59 @@ static int wifi_sta_getconfig (lua_State *L)
     lua_setfield (L, -2, "bssid");
   }
 
+  lua_pushinteger(L,
+    cfg.sta.pmf_cfg.required ? PMF_VAL_REQUIRED : PMF_VAL_AVAILABLE);
+  lua_setfield(L, -2, "pmf");
+
+  const char *tmp;
+  switch(cfg.sta.scan_method)
+  {
+    case WIFI_FAST_SCAN: tmp = "fast"; break;
+    case WIFI_ALL_CHANNEL_SCAN: tmp = "all"; break;
+    default: tmp = NULL; break;
+  }
+  if (tmp)
+  {
+    lua_pushstring(L, tmp);
+    lua_setfield(L, -2, "scan_method");
+  }
+
+  lua_pushinteger(L, cfg.sta.channel);
+  lua_setfield(L, -2, "channel");
+
+  lua_pushinteger(L, cfg.sta.listen_interval);
+  lua_setfield(L, -2, "listen_interval");
+
+  switch(cfg.sta.sort_method)
+  {
+    case WIFI_CONNECT_AP_BY_SIGNAL: tmp = "rssi"; break;
+    case WIFI_CONNECT_AP_BY_SECURITY: tmp = "authmode"; break;
+    default: tmp = NULL; break;
+  }
+  if (tmp)
+  {
+    lua_pushstring(L, tmp);
+    lua_setfield(L, -2, "sort_by");
+  }
+
+  lua_pushinteger(L, cfg.sta.threshold.rssi);
+  lua_setfield(L, -2, "threshold_rssi");
+
+  lua_pushinteger(L, cfg.sta.threshold.authmode);
+  lua_setfield(L, -2, "threshold_authmode");
+
+  lua_pushboolean(L, cfg.sta.rm_enabled);
+  lua_setfield(L, -2, "rm");
+
+  lua_pushboolean(L, cfg.sta.btm_enabled);
+  lua_setfield(L, -2, "btm");
+
+  lua_pushboolean(L, cfg.sta.mbo_enabled);
+  lua_setfield(L, -2, "mbo");
+
+  lua_pushinteger(L, cfg.sta.sae_pwe_h2e);
+  lua_setfield(L, -2, "sae_pwe");
+
   return 1;
 }
 
@@ -438,20 +570,59 @@ static int wifi_sta_scan (lua_State *L)
 }
 
 
+static int wifi_sta_powersave(lua_State *L)
+{
+  static const wifi_ps_type_t vals[] = {
+    WIFI_PS_NONE, WIFI_PS_MIN_MODEM, WIFI_PS_MAX_MODEM,
+  };
+  static const char *keys[] = { "none", "min", "max" };
+
+  esp_err_t ret = esp_wifi_set_ps(vals[luaL_checkoption(L, 1, NULL, keys)]);
+  if (ret != ESP_OK)
+    return luaL_error(L, "set powersave failed, code %d", ret);
+
+  return 0;
+}
+
+
+static int wifi_sta_getpowersave(lua_State *L)
+{
+  wifi_ps_type_t ps;
+  esp_err_t ret = esp_wifi_get_ps(&ps);
+  if (ret != ESP_OK)
+    return luaL_error(L, "get powersave failed, code %d", ret);
+
+  const char *mode;
+  switch(ps)
+  {
+    case WIFI_PS_NONE: mode = "none"; break;
+    case WIFI_PS_MIN_MODEM: mode = "min"; break;
+    case WIFI_PS_MAX_MODEM: mode = "max"; break;
+    default:
+      return luaL_error(L, "unknown powersave mode??");
+  }
+  lua_pushstring(L, mode);
+
+  return 1;
+}
+
+
 LROT_BEGIN(wifi_sta, NULL, 0)
   LROT_FUNCENTRY( setip,       wifi_sta_setip )
-  LROT_FUNCENTRY( sethostname, wifi_sta_sethostname )
-  LROT_FUNCENTRY( config,      wifi_sta_config )
+  LROT_FUNCENTRY( sethostname, wifi_sta_sethostname)
+  LROT_FUNCENTRY( settxpower,  wifi_sta_settxpower)
+  LROT_FUNCENTRY( config,      wifi_sta_config)
   LROT_FUNCENTRY( connect,     wifi_sta_connect )
   LROT_FUNCENTRY( disconnect,  wifi_sta_disconnect )
   LROT_FUNCENTRY( getconfig,   wifi_sta_getconfig )
   LROT_FUNCENTRY( getmac,      wifi_sta_getmac )
   LROT_FUNCENTRY( on,          wifi_sta_on )
   LROT_FUNCENTRY( scan,        wifi_sta_scan )
+  LROT_FUNCENTRY( powersave,   wifi_sta_powersave )
+  LROT_FUNCENTRY( getpowersave,wifi_sta_getpowersave )
 
-  LROT_NUMENTRY(  PMF_OFF,       0 )
-  LROT_NUMENTRY(  PMF_AVAILABLE, 1 )
-  LROT_NUMENTRY(  PMF_REQUIRED,  2 )
+  LROT_NUMENTRY(  PMF_AVAILABLE, PMF_VAL_AVAILABLE )
+  LROT_NUMENTRY(  PMF_REQUIRED,  PMF_VAL_REQUIRED )
 LROT_END(wifi_sta, NULL, 0)
 
 NODEMCU_ESP_EVENT(WIFI_EVENT, WIFI_EVENT_SCAN_DONE, on_scan_done);

components/platform/dht.c

@@ -86,6 +86,7 @@ static int dht_init( uint8_t gpio_num )
     rmt_rx.channel = dht_rmt.channel;
     rmt_rx.gpio_num = gpio_num;
     rmt_rx.clk_div = 80;  // base period is 1us
+    rmt_rx.flags = 0;
     rmt_rx.mem_block_num = 1;
     rmt_rx.rmt_mode = RMT_MODE_RX;
     rmt_rx.rx_config.filter_en = true;

components/platform/onewire.c

@@ -126,6 +126,7 @@ static int onewire_rmt_init( uint8_t gpio_num )
       rmt_tx.gpio_num = gpio_num;
       rmt_tx.mem_block_num = 1;
       rmt_tx.clk_div = 80;
+      rmt_tx.flags = 0;
       rmt_tx.tx_config.loop_en = false;
       rmt_tx.tx_config.carrier_en = false;
       rmt_tx.tx_config.idle_level = 1;
@@ -138,6 +139,7 @@ static int onewire_rmt_init( uint8_t gpio_num )
           rmt_rx.channel = ow_rmt.rx;
           rmt_rx.gpio_num = gpio_num;
           rmt_rx.clk_div = 80;
+          rmt_rx.flags = 0;
           rmt_rx.mem_block_num = 1;
           rmt_rx.rmt_mode = RMT_MODE_RX;
           rmt_rx.rx_config.filter_en = true;

components/platform/platform.c

@@ -65,6 +65,7 @@ uart_status_t uart_status[NUM_UART];
 task_handle_t uart_event_task_id = 0;
 SemaphoreHandle_t sem = NULL;
 
+extern bool uart_has_on_data_cb(unsigned id);
 extern bool uart_on_data_cb(unsigned id, const char *buf, size_t len);
 extern bool uart_on_error_cb(unsigned id, const char *buf, size_t len);
 
@@ -74,14 +75,18 @@ void uart_event_task( task_param_t param, task_prio_t prio ) {
   uart_status_t *us = &uart_status[id];
   xSemaphoreGive(sem);
   if(post->type == PLATFORM_UART_EVENT_DATA) {
-    size_t i = 0;
-    while (i < post->size)
-    {
-      if (id == CONFIG_ESP_CONSOLE_UART_NUM && run_input) {
+    if (id == CONFIG_ESP_CONSOLE_UART_NUM && run_input) {
+      size_t i = 0;
+      while (i < post->size)
+      {
         unsigned used = feed_lua_input(post->data + i, post->size - i);
         i += used;
       }
-      else {
+    }
+    if (uart_has_on_data_cb(id)) {
+      size_t i = 0;
+      while (i < post->size)
+      {
         char ch = post->data[i];
         us->line_buffer[us->line_position] = ch;
         us->line_position++;
@@ -208,9 +213,11 @@ static void task_uart( void *pvParameters ){
 uint32_t platform_uart_setup( unsigned id, uint32_t baud, int databits, int parity, int stopbits, uart_pins_t* pins )
 {
   int flow_control = UART_HW_FLOWCTRL_DISABLE;
-  if(pins->flow_control & PLATFORM_UART_FLOW_CTS) flow_control |= UART_HW_FLOWCTRL_CTS;
-  if(pins->flow_control & PLATFORM_UART_FLOW_RTS) flow_control |= UART_HW_FLOWCTRL_RTS;
-  
+  if (pins != NULL) {
+	if(pins->flow_control & PLATFORM_UART_FLOW_CTS) flow_control |= UART_HW_FLOWCTRL_CTS;
+	if(pins->flow_control & PLATFORM_UART_FLOW_RTS) flow_control |= UART_HW_FLOWCTRL_RTS;
+  }
+
   uart_config_t cfg = {
      .baud_rate = baud,
      .flow_ctrl = flow_control,

components/platform/platform_rmt.c

@@ -24,19 +24,19 @@ static bool rmt_channel_check( uint8_t channel, uint8_t num_mem )
   return rmt_channel_check( channel-1, num_mem-1);
 }
 
-#if defined(CONFIG_IDF_TARGET_ESP32C3)
 int platform_rmt_allocate( uint8_t num_mem, rmt_mode_t mode )
-#else
-int platform_rmt_allocate( uint8_t num_mem, rmt_mode_t mode __attribute__((unused)))
-#endif
 {
   int channel;
   int alloc_min;
   int alloc_max;
   uint8_t tag = 1;
 
-#if defined(CONFIG_IDF_TARGET_ESP32C3)
-  /* The ESP32-C3 is limited to TX on channel 0-1 and RX on channel 2-3. */
+#if SOC_RMT_TX_CANDIDATES_PER_GROUP == SOC_RMT_CHANNELS_PER_GROUP
+  alloc_min = 0;
+  alloc_max = RMT_CHANNEL_MAX - 1;
+#else
+  /* On platforms where channels cannot do both TX and RX, the TX ones always
+     seem to start at index 0, and are then followed by the RX channels. */
   if( mode==RMT_MODE_TX ) {
     alloc_min = 0;
     alloc_max = SOC_RMT_TX_CANDIDATES_PER_GROUP - 1;
@@ -44,10 +44,6 @@ int platform_rmt_allocate( uint8_t num_mem, rmt_mode_t mode __attribute__((unuse
     alloc_min = RMT_CHANNEL_MAX - SOC_RMT_RX_CANDIDATES_PER_GROUP;
     alloc_max = RMT_CHANNEL_MAX - 1;
   }
-#else
-  /* The other ESP32 devices can do RX and TX on all channels. */
-  alloc_min = 0;
-  alloc_max = RMT_CHANNEL_MAX - 1;
 #endif
 
   for (channel = alloc_max; channel >= alloc_min; channel--) {

components/platform/ws2812.c

@@ -33,6 +33,11 @@
 
 #undef WS2812_DEBUG
 
+// If either of these fails, the reset logic in ws2812_sample_to_rmt will need revisiting.
+_Static_assert(SOC_RMT_MEM_WORDS_PER_CHANNEL % 8 == 0,
+  "SOC_RMT_MEM_WORDS_PER_CHANNEL is assumed to be a multiple of 8");
+_Static_assert(SOC_RMT_MEM_WORDS_PER_CHANNEL >= 16,
+  "SOC_RMT_MEM_WORDS_PER_CHANNEL is assumed to be >= 16");
 
 // divider to generate 100ns base period from 80MHz APB clock
 #define WS2812_CLKDIV (100 * 80 /1000)
@@ -58,82 +63,67 @@ const rmt_item32_t ws2812_rmt_bit1 = {
   .duration1 = WS2812_DURATION_T1L
 };
 
-#define ws2812_rmt_reset {.level0 = 0, .duration0 = 4, .level1 = 0, .duration1 = 4}
-// reset signal, spans one complete buffer block
-const rmt_item32_t ws2812_rmt_reset_block[64] = { [0 ... 63] = ws2812_rmt_reset };
-
+// This is one eighth of 512 * 100ns, ie in total a bit above the requisite 50us
+const rmt_item32_t ws2812_rmt_reset = { .level0 = 0, .duration0 = 32, .level1 = 0, .duration1 = 32 };
 
 // descriptor for a ws2812 chain
 typedef struct {
   bool valid;
+  bool needs_reset;
   uint8_t gpio;
   const uint8_t *data;
   size_t len;
-  size_t tx_offset;
 } ws2812_chain_t;
 
-
 // chain descriptor array
 static ws2812_chain_t ws2812_chains[RMT_CHANNEL_MAX];
 
-// interrupt handler for ws2812 ISR
-static intr_handle_t ws2812_intr_handle;
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
 
-
-static void ws2812_fill_memory_encoded( rmt_channel_t channel, const uint8_t *data, size_t len, size_t tx_offset )
+static void ws2812_sample_to_rmt(const void *src, rmt_item32_t *dest, size_t src_size, size_t wanted_num, size_t *translated_size, size_t *item_num)
 {
-  for (size_t idx = 0; idx < len; idx++) {
+  // Note: enabling these commented-out logs will ruin the timing so nothing
+  // will actually work when they're enabled. But I've kept them in as comments
+  // because they were useful in debugging the buffer management.
+  // ESP_DRAM_LOGW("ws2812", "ws2812_sample_to_rmt wanted=%u src_size=%u", wanted_num, src_size);
+
+  void *ctx;
+  rmt_translator_get_context(item_num, &ctx);
+  ws2812_chain_t *chain = (ws2812_chain_t *)ctx;
+
+  size_t reset_num = 0;
+  if (chain->needs_reset) {
+    // Haven't sent reset yet
+
+    // We split the reset into 8 even though it would fit in a single
+    // rmt_item32_t, simply so that dest stays 8-item aligned which means we
+    // don't have to worry about having to split a byte of src across multiple
+    // blocks (assuming the static asserts at the top of this file are true).
+    for (int i = 0; i < 8; i++) {
+      dest[i] = ws2812_rmt_reset;
+    }
+    dest += 8;
+    wanted_num -= 8;
+    reset_num = 8;
+    chain->needs_reset = false;
+  }
+
+  // Now write the actual data from src
+  const uint8_t *data = (const uint8_t *)src;
+  size_t data_num = MIN(wanted_num, src_size * 8) / 8;
+  for (size_t idx = 0; idx < data_num; idx++) {
     uint8_t byte = data[idx];
-
     for (uint8_t i = 0; i < 8; i++) {
-      RMTMEM.chan[channel].data32[tx_offset + idx*8 + i].val = byte & 0x80 ? ws2812_rmt_bit1.val : ws2812_rmt_bit0.val;
-
+      dest[idx * 8 + i] = (byte & 0x80) ? ws2812_rmt_bit1 : ws2812_rmt_bit0;
       byte <<= 1;
     }
   }
+
+  *translated_size = data_num;
+  *item_num = reset_num + data_num * 8;
+  // ESP_DRAM_LOGW("ws2812", "src bytes consumed: %u total rmt items: %u", *translated_size, *item_num);
 }
 
-
-static void ws2812_isr(void *arg)
-{
-  uint32_t intr_st = RMT.int_st.val;
-
-  for (rmt_channel_t channel = 0; channel < RMT_CHANNEL_MAX; channel++) {
-
-    if ((intr_st & (BIT(channel+24))) && (ws2812_chains[channel].valid)) {
-      RMT.int_clr.val = BIT(channel+24);
-
-      ws2812_chain_t *chain = &(ws2812_chains[channel]);
-#if defined(CONFIG_IDF_TARGET_ESP32) || defined(CONFIG_IDF_TARGET_ESP32S2)
-      uint32_t data_sub_len = RMT.tx_lim_ch[channel].limit/8;
-#elif defined(CONFIG_IDF_TARGET_ESP32S3)
-      uint32_t data_sub_len = RMT.chn_tx_lim[channel].tx_lim_chn/8;
-#else
-      uint32_t data_sub_len = RMT.tx_lim[channel].limit/8;
-#endif
-
-      if (chain->len >= data_sub_len) {
-        ws2812_fill_memory_encoded( channel, chain->data, data_sub_len, chain->tx_offset );
-        chain->data += data_sub_len;
-        chain->len -= data_sub_len;
-      } else if (chain->len == 0) {
-        RMTMEM.chan[channel].data32[chain->tx_offset].val = 0;
-      } else {
-        ws2812_fill_memory_encoded( channel, chain->data, chain->len, chain->tx_offset );
-        RMTMEM.chan[channel].data32[chain->tx_offset + chain->len*8].val = 0;
-        chain->data += chain->len;
-        chain->len = 0;
-      }
-      if (chain->tx_offset == 0) {
-        chain->tx_offset = data_sub_len * 8;
-      } else {
-        chain->tx_offset = 0;
-      }
-    }
-  }
-}
-
-
 int platform_ws2812_setup( uint8_t gpio_num, uint8_t num_mem, const uint8_t *data, size_t len )
 {
   int channel;
@@ -145,7 +135,7 @@ int platform_ws2812_setup( uint8_t gpio_num, uint8_t num_mem, const uint8_t *dat
     chain->gpio = gpio_num;
     chain->len = len;
     chain->data = data;
-    chain->tx_offset = 0;
+    chain->needs_reset = true;
 
 #ifdef WS2812_DEBUG
     ESP_LOGI("ws2812", "Setup done for gpio %d on RMT channel %d", gpio_num, channel);
@@ -187,6 +177,7 @@ int platform_ws2812_send( void )
   // common settings
   rmt_tx.mem_block_num = 1;
   rmt_tx.clk_div = WS2812_CLKDIV;
+  rmt_tx.flags = 0;
   rmt_tx.tx_config.loop_en = false;
   rmt_tx.tx_config.carrier_en = false;
   rmt_tx.tx_config.idle_level = 0;
@@ -206,23 +197,23 @@ int platform_ws2812_send( void )
         res = PLATFORM_ERR;
         break;
       }
+      if (rmt_translator_init(channel, ws2812_sample_to_rmt) != ESP_OK) {
+        res = PLATFORM_ERR;
+        break;
+      }
+      if (rmt_translator_set_context(channel, &ws2812_chains[channel]) != ESP_OK) {
+        res = PLATFORM_ERR;
+        break;
+      }
     }
   }
 
-
-  // hook-in our shared ISR
-  esp_intr_alloc( ETS_RMT_INTR_SOURCE, PLATFORM_RMT_INTR_FLAGS, ws2812_isr, NULL, &ws2812_intr_handle );
-
-
   // start selected channels one by one
   for (rmt_channel_t channel = 0; channel < RMT_CHANNEL_MAX && res == PLATFORM_OK; channel++) {
     if (ws2812_chains[channel].valid) {
-      // we just feed a single block for generating the reset to the rmt driver
-      // the actual payload data is handled by our private shared ISR
-      if (rmt_write_items( channel,
-                           (rmt_item32_t *)ws2812_rmt_reset_block,
-                           64,
-                           false ) != ESP_OK) {
+      // ws2812_sample_to_rmt takes care of translating the data to rmt_item32_t
+      // format, as well as prepending the reset sequence.
+      if (rmt_write_sample(channel, ws2812_chains[channel].data, ws2812_chains[channel].len, false) != ESP_OK) {
         res = PLATFORM_ERR;
         break;
       }
@@ -236,11 +227,6 @@ int platform_ws2812_send( void )
     }
   }
 
-
-  // un-hook our ISR
-  esp_intr_free( ws2812_intr_handle );
-
-
   return res;
 }
 

docs/build.md

@@ -121,3 +121,28 @@ Partition Table --->
   (components/platform/partitions-2MB.csv) Custom partition CSV file
   (0x10000) Factory app partition offset
 ```
+
+### Using external components
+
+It is possible, and relatively easy, to include external components and modules in NodeMCU. It is not uncommon to have one or more custom modules one wishes to include in the firmware. To enable this NodeMCU leverages the standard IDF `EXTRA_COMPONENT_DIRS` functionality. As such, it is possible to not only add extra Lua C modules, but also other components such as libraries.
+
+To include one (or more) additional IDF components, simply set the `EXTRA_COMPONENT_DIRS` environment variable to the space-separated list of directories of said components. E.g.
+
+```
+export EXTRA_COMPONENT_DIRS="/path/to/mymod /path/to/mylib"
+make menuconfig
+make
+```
+
+To get started, a template directory structure is provided in [extcomp-template/](../extcomp-template) which provides a skeleton for a simple Lua C module, including the build logic in `CMakeLists.txt`, the configuration option in `Kconfig` and the Lua C module code in `mymod.c`. A detailed discussion on the specifics is beyond this document, but the first two are described comprehensively in the [official IDF documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/build-system.html), and module development is covered in [Programming in NodeMCU](nodemcu-pil.md).
+
+In fact, to quickly try it out it's even possible to include the template itself, as-is:
+
+
+```
+export EXTRA_COMPONENT_DIRS="$PWD/extcomp-template"
+make menuconfig
+make
+```
+
+after which the command `mymod.hello()` is available in the Lua environment.

docs/modules/extmods.md

@@ -1,134 +0,0 @@
-
-# External modules. Plugging your own C modules. **BETA**
-
-Note: this feature is still in beta. Configuration files / API may change.
-
-In order to make the most of NodeMCU, you will undoubtedly have to connect your ESP device to many different hardware modules, which will require you to write driver code in C and expose a Lua interface.
-
-To make this easy, we have come up with the concept of "external modules". External modules allow you to refer to an external git repository containing the module, which will be downloaded/updated and built along the firmware. It is similar to git submodules, but without the complexity and having to alter the parent repository, while also adapted to the compilation requirements of NodeMCU and Lua.
-
-## How to use external modules:
-
-To use external modules, simply create an `extmods.ini` file in the repository root, with the following syntax:
-
-```ini
-[lua_mount_point]
-url=<git repo URL>
-ref=<branch name, tag or commit id>
-...
-```
-
-Where:
-* **lua_mount_point**: Name you want the referenced module to have in Lua.
-* **url**: Repository URL where to fetch the code from
-* **ref**: branch, tag or commit hash of the version of the module you want to pull in.
-* **disabled**: (optional) Whether or not to actually compile the module (see below)
-
-For example:
-
-```ini
-[helloworld]
-url=git@github.com:espore-ide/nodemcu-module-helloworld.git
-ref=master
-```
-
-You can add further sections to `extmods.ini`, one for each module you want to add. Once this file is ready, run the update command:
-
-```shell
-make extmod-update
-```
-
-This will download or update the modules to the external modules directory, `components/modules/external`.
-
-You can now compile the firmware with `make` as usual. The build system will find your external modules and compile them along the core modules.
-
-After this is flashed to the device, the module in the example will be available in lua as `helloworld`.
-
-### Updating to the latest code
-
-If your external module entry in `extmods.ini` points to a branch, e.g., `master`, you can update your local version to the latest code anytime by simply running `make extmod-update`.
-
-### Temporarily disabling an external module
-
-If you want to stop compiling and including a module in the build for any reason, without having to remove the entry from `extmods.ini`, simply add a `disabled=true` entry in the module section and run `make extmod-update`.
-
-Example:
-```ini
-[helloworld]
-url=https://github.com/espore-ide/nodemcu-module-helloworld.git
-ref=master
-disabled=true
-```
-
-### Mounting different module versions
-
-Provided the module is well written (no global variables, etc), it is even possible to easily mount different versions of the same module simultaneously for testing:
-
-```ini
-[helloworld]
-url=https://github.com/espore-ide/nodemcu-module-helloworld.git
-ref=master
-
-[helloworld_dev]
-url=https://github.com/espore-ide/nodemcu-module-helloworld.git
-ref=dev
-```
-
-Note that the second one points to a different branch, `dev`. Both modules will be visible in Lua under `helloworld` and `helloworld_dev` respectively.
-
-## How to write external modules:
-
-To write your own external module do the following:
-
-1. Create an empty repository in your favorite remote, GitHub, BitBucket, GitLab, etc, or fork the helloworld example.
-2. Create an entry in `extmods.ini` as explained above, with the `url=` key pointing to your repository. For modules that you author, it is recommended to use an updateable git URL in SSH format, such as `git@github.com:espore-ide/nodemcu-module-helloworld.git`.
-3. Run `make extmod-update`
-
-You can now change to `components/modules/external/your_module` and begin work. Since that is your own repository, you can work normally, commit, create branches, etc.
-
-### External module scaffolding
-
-External modules must follow a specific structure to declare the module in C. Please refer to the [helloworld.c](https://github.com/nodemcu/nodemcu-firmware/blob/dev-esp32/tools/example/helloworld.c) example, or use it as a template. In particular:
-
-1. Include `module.h` 
-2. Define a Module Function Map with name `module`
-3. Define a `module_init` function
-4. Include the module lua entries by adding a call to the `NODEMCU_MODULE_STD` macro
-
-Here is a bare minimum module:
-```c
-#include "module.h"
-
-// Module function map
-LROT_BEGIN(module)
-/* module-level functions go here*/
-LROT_END(module, NULL, 0)
-
-// module_init is invoked on device startup
-static int module_init(lua_State* L) {
-    // initialize your module, register metatables, etc
-    return 0;
-}
-
-NODEMCU_MODULE_STD();  // define Lua entries
-```
-For a full example module boilerplate, check the [helloworld.c](https://github.com/nodemcu/nodemcu-firmware/blob/dev-esp32/tools/example/helloworld.c) file.
-
-
-### Special makefile or compilation requirements
-
-If your module has special makefile requirements, you can create a `module.mk` file at the root of your module repository. This will be executed during compilation.
-
-### What is this "component.mk" file that appeared in my repo when running `make extmod-update` ?
-
-This file is ignored by your repository. Do not edit or check it in!. This file contains the necessary information to compile your module along with the others.
-
-Note that you don't even need to add this file to your `.gitignore`, since the `make extmod-update` operation configures your local copy to ignore it (via `.git/info/exclude`). 
-
-## Further work:
-
-* Support for per-module menuconfig (`Kconfig`). This is actually possible already, but need to work around potential config variable collisions in case two module authors happen to choose the same variable names.
-* Module registry: Create an official repository of known external modules.
-* Move all non-essential and specific hardware-related C modules currently in the NodeMCU repository to external modules, each in their own repository.
-* Create the necessary scaffolding to facilitate writing modules that will work both in ESP8266 and ESP32.
-* Port this work to the ESP8266 branch. Mostly, the scripts that make this possible could work in both branches directly.

docs/modules/file.md

@@ -115,6 +115,38 @@ for k,v in pairs(l) do
 end
 ```
 
+## file.mkdir()
+
+Creates a directory, provided the underlying file system supports directories. SPIFFS does not, but FAT (which you may have on an attached SD card) does.
+
+#### Syntax
+`file.mkdir(path [, mode])`
+
+#### Parameters
+- `path` the full path name of the directory to create. E.g. "/SD0/MYDIR".
+- `mode` optional, only used for file systems which use mode permissions. Defaults to 0777 (octal).
+
+#### Returns
+`nil`
+
+Throws an error if the directory could not be created. Error code 134 (at the
+time of writing) indicates that the filesystem at the given path does not
+support directories.
+
+## file.rmdir()
+
+Removes an empty directory, provided the underlying file system supports directories. SPIFFS does not, but FAT (which you may have on an attached SD card) does.
+
+#### Syntax
+`file.rmdir(path)`
+
+#### Parameters
+- `path` the path to the directory to remove. The directory must be empty.
+
+#### Returns
+`nil`
+
+Throws an error if the directory could not be removed.
 
 ## file.remove()
 

docs/modules/ledc.md

@@ -22,6 +22,7 @@ myChannel = ledc.newChannel({
   timer=ledc.TIMER_0 || ledc.TIMER_1 || ledc.TIMER_2 || ledc.TIMER_3,
   channel=ledc.CHANNEL_0 || ledc.CHANNEL_1 || ledc.CHANNEL_2 || ledc.CHANNEL_3 || ledc.CHANNEL_4 || ledc.CHANNEL_5 || ledc.CHANNEL_6 || ledc.CHANNEL_7,
   frequency=x,
+  invert=false,
   duty=x
 });
 ```
@@ -46,6 +47,7 @@ List of configuration tables:
     - ...
     - `ledc.CHANNEL_7`
 - `frequency` Timer frequency(Hz)
+- `invert` Inverts the output. False, with duty 0, is always low.
 - `duty` Channel duty, the duty range is [0, (2**bit_num) - 1]. Example: if ledc.TIMER_13_BIT is used maximum value is 4096 x 2 -1 = 8091
 
 #### Returns
@@ -264,7 +266,7 @@ channel:fadewithstep(1000, 10, 10);
 Set LEDC fade function.
 
 #### Syntax
-`channel:fadewithstep(duty, direction, scale, cycleNum, stepNum [, wait])`
+`channel:fade(duty, direction, scale, cycleNum, stepNum [, wait])`
 
 #### Parameters
 - `duty` Set the start of the gradient duty.

docs/modules/rmt.md

@@ -0,0 +1,139 @@
+# Remote Control Driver
+| Since  | Origin / Contributor  | Maintainer  | Source  |
+| :----- | :-------------------- | :---------- | :------ |
+| 2022-01-01 | [pjsg](https://github.com/pjsg) | [pjsg](https://github.com/pjsg) | [rmt.c](../../components/modules/rmt.c)|
+
+The RMT module provides a simple interface onto the ESP32 RMT peripheral. This allows the generation of
+arbitrary pulse trains with good timing accuracy. This can be used to generate IR remote control signals, or
+servo control pulses, or pretty much any high speed signalling system.  It isn't good for low speed stuff as the maximum
+pulse time is under 200ms -- though you can get longer by having multiple large values in a row. See the Data Encoding
+below for more details.
+
+## rmt.txsetup(gpio, bitrate, options)
+
+This sets up a transmit channel on the specified gpio pin at the specified rate. Various options described below
+can be specified in the `options`. The bit time is specified in picoseconds so that integer values can be used.
+
+An error will be thrown if the bit time cannot be approximated.
+
+#### Syntax
+`channel = rmt.txsetup(gpio, bittime[, options])`
+
+#### Parameters
+- `gpio` The GPIO pin number to use.
+- `bittime` The bit time to use in picoseconds. Only certain times can be handled exactly. The actual set time will be returned. The actual range is limited -- probably using 100,000 (0.1µS) or 1,000,000 (1µS). The actual constraint is that the interval is 1 - 255 cycles of an 80MHz clock.
+- `options` A table with the keys as defined below.
+
+##### Returns
+- The `rmt.channel` object that can be used for sending data
+- The actual bit time in picoseconds.
+
+#### Example
+```lua
+```
+
+#### Options table
+
+This optional table consists of a number of keys that control various aspects of the RMT transmission.
+
+- `invert` if true, then the output is inverted.
+- `carrier_hz` specifies that the signal is to modulate the carrier at the specified frequency. This is useful for IR transmissions.
+- `carrier_duty` specifies the duty cycle of the carrier. Defaults to 50%
+- `idle_level` specifies what value to send when the transmission completes.
+
+## rmt.rxsetup(gpio, bitrate, options)
+
+This sets up a receive channel on the specified gpio pin at the specified rate. Various options described below
+can be specified in the `options`. The bit time is specified in picoseconds so that integer values can be used.
+
+An error will be thrown if the bit time cannot be approximated.
+
+#### Syntax
+`channel = rmt.rxsetup(gpio, bittime[, options])`
+
+#### Parameters
+- `gpio` The GPIO pin number to use.
+- `bittime` The bit time to use in picoseconds. Only certain times can be handled exactly. The actual set time will be returned. The actual range is limited -- probably using 100,000 (0.1µS) or 1,000,000 (1µS). The actual constraint is that the interval is 1 - 255 cycles of an 80MHz clock.
+- `options` A table with the keys as defined below.
+
+##### Returns
+- The `rmt.channel` object that can be used for receiving data
+- The actual bit time in picoseconds.
+
+#### Example
+```lua
+```
+
+#### Options table
+
+This optional table consists of a number of keys that control various aspects of the RMT transmission.
+
+- `invert` if true, then the input is inverted.
+- `filter_ticks` If specified, then any pulse shorter than this will be ignored. This is in units of the bit time.
+- `idle_threshold` If specified, then any level longer than this will set the receiver as idle. The default is 65535 bit times.
+
+
+## channel:on(event, callback)
+
+This is establishes a callback to use when data is received and it also starts the data reception process. It can only be called once per receive
+channel.
+
+#### Syntax
+`channel:on(event, callback)`
+
+#### Parameters
+- `event` This must be the string 'data' and it sets the callback that gets invoked when data is received.
+- `callback` This is invoked with a single argument that is a string that contains the data received in the format described for `send` below. `struct.unpack` is your friend.
+
+#### Returns
+`nil`
+
+## channel:send(data, cb)
+
+This is a (default) blocking call that transmits the data using the parameters specified on the `txsetup` call.
+
+#### Syntax
+`channel:send(data[, cb])`
+
+#### Parameters
+- `data` This is either a string or a table of strings.
+- `cb` This is an optional callback when the transmission is actually complete. If specified, then the `send` call is non-blocking, and the callback invoked when the transmission is complete. Otherwise the `send` call is synchronous and does not return until transmission is complete.
+
+#### Data Encoding
+
+If the `data` supplied is a table (really an array), then the elements of the table are concatenated together and sent. The elements of the table must be strings.
+
+If the item being sent is a string, then it contains 16 bit packed integers. The top bit of the integer controls the output level.
+`struct.pack("H", value)` generates a suitable value to output a zero bit. `struct.pack("H", 32768 + value)` generates a one bit of the specified width.
+
+The widths are in units of the interval specified when the channel was setup.
+
+
+#### Returns
+`nil`
+
+#### Example
+
+This example sends a single R character at 19200 bps. You wouldn't actually want to do it this way.... In some applications this would be inverted.
+
+```
+channel = rmt.txsetup(25, 1000000000 / 19200, {idle_level=1})
+one = struct.pack("H", 32768 + 1000)
+zero = struct.pack("H", 1000)
+-- Send start bit, then R = 0x52 (reversed) then stop bit
+channel:send(zero .. zero .. one .. zero .. zero .. one .. zero .. one .. zero .. one)
+-- or using the table interface
+channel:send({zero, zero, one, zero, zero, one, zero, one, zero, one})
+```
+
+## channel:close()
+
+This shuts down the RMT channel and makes it available for other uses (e.g. ws2812). The channel cannot be used after this call returns. The channel
+is also released when the garbage collector frees it up. However you should always `close` the channel explicitly as otherwise you can run out of RMT channels
+before the garbage collector frees some up.
+
+#### Syntax
+`channel:close()`
+
+#### Returns
+`nil`

docs/modules/rtcmem.md

@@ -0,0 +1,66 @@
+# RTC User Memory Module
+| Since  | Origin / Contributor  | Maintainer  | Source  |
+| :----- | :-------------------- | :---------- | :------ |
+| 2015-06-25 | [DiUS](https://github.com/DiUS), [Johny Mattsson](https://github.com/jmattsson) | [PJSG](https://github.com/pjsg) | [rtcmem.c](../../app/modules/rtcmem.c)|
+
+The rtcmem module provides basic access to the RTC memory. 
+
+This memory is preserved while power is applied, making them highly useful for keeping state across sleep cycles. Some of this memory is reserved for system use, 
+and, for compatibility with NodeMCU on the ESP8266, 128 slots (each 32bit wide) of RTC memory are reserved by this module.
+This module then provides read and write access to these slots.
+
+This module is 100% compatible with the ESP8266 version, and this means that, there is no mechanism for arbitrating use of particular slots. It is up to the end user to be aware of which memory is used for what, and avoid conflicts. Unlike the ESP8266 version, no other NodeMCU module uses any of these slots.
+
+Note that this memory is not necessary preserved across reflashing the firmware. It is the responsibility of the
+developer to deal with getting inconsistent data.
+
+## rtcmem.read32()
+
+Reads one or more 32bit values from RTC user memory.
+
+#### Syntax
+`rtcmem.read32(idx [, num])`
+
+#### Parameters
+  - `idx` zero-based index to start reading from
+  - `num` number of slots to read (default 1)
+
+#### Returns
+The value(s) read from RTC user memory.
+
+If `idx` is outside the valid range [0,127] this function returns nothing.
+
+If `num` results in overstepping the end of available memory, the function only returns the data from the valid slots.
+
+#### Example
+```lua
+val = rtcmem.read32(0) -- Read the value in slot 0
+val1, val2 = rtcmem.read32(42, 2) -- Read the values in slots 42 and 43
+```
+#### See also
+[`rtcmem.write32()`](#rtcmemwrite32)
+
+## rtcmem.write32()
+
+Writes one or more values to RTC user memory, starting at index `idx`.
+
+Writing to indices outside the valid range [0,127] has no effect.
+
+#### Syntax
+`rtcmem.write32(idx, val [, val2, ...])`
+
+#### Parameters
+  - `idx` zero-based index to start writing to. Auto-increments if multiple values are given.
+  - `val` value to store (32bit)
+  - `val2...` additional values to store (optional)
+
+#### Returns
+`nil`
+
+#### Example
+```lua
+rtcmem.write32(0, 53) -- Store the value 53 in slot 0
+rtcmem.write32(42, 2, 5, 7) -- Store the values 2, 5 and 7 into slots 42, 43 and 44, respectively.
+```
+#### See also
+[`rtcmem.read32()`](#rtcmemread32)

docs/modules/sodium.md

@@ -9,13 +9,15 @@ In addition to the flag for enabling this module during ROM build, `Component co
 
 !!! note
 
-    Almost all functions in this module require a working random number generator. On the ESP32 this means that *WiFi must be started* otherwise ALL OF THE CRYPTOGRAPHY WILL SILENTLY BE COMPROMISED. Make sure to call `wifi.start()` before any of the functions in this module. See the [Espressif documentation](https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/system/system.html#random-number-generation) for more information. The only exception is `sodium.crypto_box.seal_open()` which does not require a random number source to operate.
+    Almost all functions in this module require a working random number generator. On the ESP32 this normally means that *WiFi must be started* otherwise ALL OF THE CRYPTOGRAPHY WILL SILENTLY BE COMPROMISED. Make sure to call `wifi.start()` before any of the functions in this module. See the [Espressif documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/random.html) for more information. The only exception is `sodium.crypto_box.seal_open()` which does not require a random number source to operate. The non-crypto functions in `sodium.random` can be used with WiFi off, providing that pseudo-randomness is acceptable for the purposes they are being used for.
 
 # Random number generation
-See also [https://download.libsodium.org/doc/generating_random_data](https://download.libsodium.org/doc/generating_random_data)
+See also [https://download.libsodium.org/doc/generating_random_data](https://download.libsodium.org/doc/generating_random_data). On the esp32, a custom implementation `esp_random()` is used which (usually) requires WiFi being enabled to be truly random.
 
 ## sodium.random.random()
-Returns a random integer between `0` and `0xFFFFFFFF` inclusive. Note that on a build using `LUA_NUMBER_INTEGRAL`, results may appear negative due to integer overflow. Wifi must be started, by calling `wifi.start()`, before calling this function.
+Returns a random signed 32-bit integer between `INT32_MIN` and `INT32_MAX`. On builds with 64-bit integer support (`5.3-int64-xxx`), you can convert this to an unsigned 32-bit integer with `result % (1 << 32)`, or on builds with only 64-bit double support (`5.1-doublefp`, `5.3-int32-doublefp`), with `result % (2 ^ 32)`.
+
+If WiFi is not started by calling `wifi.start()` before calling this function, then the result will only be pseudo-random, not truly random.
 
 #### Syntax
 `sodium.random.random()`
@@ -24,22 +26,24 @@ Returns a random integer between `0` and `0xFFFFFFFF` inclusive. Note that on a
 None
 
 #### Returns
-A uniformly-distributed random integer between `0` and `0xFFFFFFFF` inclusive.
+A uniformly-distributed random 32-bit integer.
 
 ## sodium.random.uniform()
-Returns a random integer  `0 <= result < upper_bound`. Unlike `sodium.random.random() % upper_bound`, it guarantees a uniform distribution of the possible output values even when `upper_bound` is not a power of 2. Note that on a build using `LUA_NUMBER_INTEGRAL`, if `upper_bound >= 0x80000000` the result may appear negative due to integer overflow. Wifi must be started, by calling `wifi.start()`, before calling this function.
+Returns a random integer `0 <= result < upper_bound`. Unlike `sodium.random.random() % upper_bound`, it guarantees a uniform distribution of the possible output values even when `upper_bound` is not a power of 2. Specifying an `upper_bound > 0x7FFFFFFF` is not recommended because the behavior will vary depending on the Lua build configuration.
+
+If WiFi is not started by calling `wifi.start()` before calling this function, then the result will only be pseudo-random, not truly random.
 
 #### Syntax
 `sodium.random.uniform(upper_bound)`
 
 #### Parameters
-- `upper_bound` must be an integer `<= 0xFFFFFFFF`.
+- `upper_bound` an integer.
 
 #### Returns
 An integer `>= 0` and `< upper_bound`
 
 ## sodium.random.buf()
-Generates `n` bytes of random data. Wifi must be started, by calling `wifi.start()`, before calling this function.
+Generates `n` bytes of random data. If WiFi is not started by calling `wifi.start()` before calling this function, then the result will only be pseudo-random, not truly random.
 
 #### Syntax
 `sodium.random.buf(n)`

docs/modules/wifi.md

@@ -149,11 +149,22 @@ being removed in the SDK/IDF. After start-up it is necessary to call
         - "AcDc0123c0DE"
     - `pmf` an optional setting to control whether Protected Management Frames
       are supported and/or required. One of:
-          - `wifi.sta.PMF_OFF`
           - `wifi.sta.PMF_AVAILABLE`
           - `wifi.sta.PMF_REQUIRED`.
       Defaults to `wifi.sta.PMF_AVAILABLE`. PMF is required when joining to
-      WPA3-Personal access points.
+      WPA3-Personal access points. The value `wifi.sta.PMF_OFF` is no longer
+      available as it is no longer supported by the wifi stack.
+
+    - `channel` optional integer value, the channel number to start scanning for the AP from, if known.
+    - `scan_method` optional string value, one of `"fast"` or `"all"` do either do a fast scan or all channel scan when looking for the AP to connect to. With fast scan, the first found matching AP is used even if it is not the best/closest one.
+    - `listen_interval` optional listen interval to receive beacon if max wifi power saving mode is enabled. Units is in AP beacon intervals. Defaults to 3.
+    - `sort_by` optional string value for preferential selection of AP. Must be one of `"rssi"` or `"authmode"` if present.
+    - `threshold_rssi` optional integer value to limit APs to only those which have a signal stronger than this value.
+    - `threshold_authmode` optional value to limit APs to those with an authentication mode of at least this settings. One of `wifi.AUTH_OPEN`, `wifi.AUTH_WEP`, `wifi.AUTH_WPA_PSK`, `wifi.AUTH_WPA2_PSK`, `wifi.AUTH_WPA_WPA2_PSK`, `wifi.AUTH_WPA2_ENTERPRISE`, `wifi.AUTH_WPA3_PSK`, `wifi.AUTH_WPA2_WPA3_PSK`, `wifi.AUTH_WAPI_PSK`.
+    - `rm` optional boolean, set to `true` to enable Radio Measurements
+    - `btm` optional boolean, set to `true` to enable BSS Transition Management
+    - `mbo` optional boolean, set to `true` to enable Multi-Band Operation
+    - `sae_pwe` optional, configures WPA3 SAE Password Element setting. One of `wifi.SAE_PWE_UNSPECIFIED`, `wifi.SAE_PWE_HUNT_AND_PECK`, `wifi.SAE_PWE_HASH_TO_ELEMENT` or `wifi.SAE_PWE_BOTH`.
 
 - `save` Save station configuration to flash. 
     - `true` configuration **will** be retained through power cycle. 
@@ -190,6 +201,21 @@ wifi.sta.config(station_cfg)
 - [`wifi.sta.connect()`](#wifistaconnect)
 - [`wifi.sta.disconnect()`](#wifistadisconnect)
 
+
+## wifi.sta.getconfig()
+
+Returns the current station configuration.
+
+#### Syntax
+`wifi.sta.getconfig()`
+
+#### Parameters
+`nil`
+
+#### Returns
+A table with the configuration settings. Refer to [`wifi.sta.config()`](#wifistaconfig) for field details.
+
+
 ## wifi.sta.connect()
 
 Connects to the configured AP in station mode. You will want to call this
@@ -233,6 +259,62 @@ Disconnects from AP in station mode.
 - [`wifi.sta.connect()`](#wifistaconnect)
 
 
+## wifi.sta.settxpower
+
+Allows adjusting the maximum TX power for the WiFi. This is (unfortunately) needed for some boards which 
+have a badly matched antenna.
+
+#### Syntax
+`wifi.sta.settxpower(power)`
+
+#### Parameters
+- `power` The maximum transmit power in dBm. This must have the range 2dBm - 20dBm. This value is a float.
+
+#### Returns
+A `boolean` where `true` is OK.
+
+#### Example
+
+```
+# Needed for the WEMOS C3 Mini
+wifi.sta.settxpower(8.5)
+```
+
+## wifi.sta.powersave
+
+Configures power-saving setting in station mode.
+
+#### Syntax
+`wifi.sta.powersave(setting)`
+
+#### Parameters
+- `setting` one of `"none"`, `"min"` or `"max"`. In `"min"` mode, the station wakes up every DTIM period to receive the beacon. In `"max"` mode, the station wakes up at the interval configured in `listen_interval` (see [`wifi.sta.config()`](#wifistaconfig). When set to `"none"` the station does not go to sleep and can receive frames immediately.
+
+#### Returns
+`nil`
+
+#### See also
+- [`wifi.sta.getpowersave()`](#wifistagetpowersave)
+
+
+## wifi.sta.getpowersave
+
+Returns the configured station power-saving mode.
+
+#### Syntax
+`wifi.sta.getpowersave()`
+
+#### Parameters
+`nil`
+
+#### Returns
+
+One of `"none"`, `"min"` or `"max"`. Refer to [`wifi.sta.powersave()`](#wifistapowersave) for details.
+
+#### See also
+- [`wifi.sta.powersave()`](#wifistapowersave)
+
+
 ## wifi.sta.on()
 
 Registers callbacks for WiFi station status events.
@@ -261,7 +343,7 @@ Event information provided for each event is as follows:
     - `ssid`: the SSID of the network
     - `bssid`: the BSSID of the AP
     - `channel`: the primary channel of the network
-    - `auth` authentication method, one of `wifi.AUTH_OPEN`, `wifi.AUTH_WPA_PSK`, `wifi.AUTH_WPA2_PSK`, `wifi.WPA_WPA2_PSK`, `wifi.AUTH_WPA3_PSK`, `wifi.AUTH_WAPI_PSK`
+    - `auth` authentication method, one of `wifi.AUTH_OPEN`, `wifi.AUTH_WEP`, `wifi.AUTH_WPA_PSK`, `wifi.AUTH_WPA2_PSK`, `wifi.AUTH_WPA_WPA2_PSK`, `wifi.AUTH_WPA2_ENTERPRISE`, `wifi.AUTH_WPA3_PSK`, `wifi.AUTH_WPA2_WPA3_PSK`, `wifi.AUTH_WAPI_PSK`
 - `disconnected`: information about the network/AP that was disconnected from:
     - `ssid`: the SSID of the network
     - `bssid`: the BSSID of the AP
@@ -353,7 +435,7 @@ The following fields are provided for each scanned AP:
 - `bssid`: the BSSID of the AP
 - `channel`: primary WiFi channel of the AP
 - `rssi`: Received Signal Strength Indicator value
-- `auth` authentication method, one of `wifi.AUTH_OPEN`, `wifi.AUTH_WPA_PSK`, `wifi.AUTH_WPA2_PSK`, `wifi.AUTH_WPA_WPA2_PSK`, `wifi.AUTH_WPA2_ENTERPRISE`, `wifi.AUTH_WPA2_WPA3_PSK`, `wifi.AUTH_WPA3_PSK`, `wifi.AUTH_WAPI_PSK`
+- `auth` authentication method, one of `wifi.AUTH_OPEN`, `wifi.AUTH_WEP`, `wifi.AUTH_WPA_PSK`, `wifi.AUTH_WPA2_PSK`, `wifi.AUTH_WPA_WPA2_PSK`, `wifi.AUTH_WPA2_ENTERPRISE`, `wifi.AUTH_WPA3_PSK`, `wifi.AUTH_WPA2_WPA3_PSK`, `wifi.AUTH_WAPI_PSK`
 - `bandwidth`: one of the following constants:
     - `wifi.HT20`
     - `wifi.HT40_ABOVE`

docs/nodemcu-lrm.md

@@ -231,6 +231,17 @@ Equivalent to `luaL_newmetatable()` for ROTable metatables.  Adds key / ROTable
 
 This macro executes `luaL_unref(L, t, r)` and then assigns `r = LUA_NOREF`.
 
+#### luaL_totoggle
+
+`  bool luaL_totoggle(lua_State *L, int idx)`
+
+There are several ways of indicating a configuration toggle value:
+  - The modern Lua way, with a boolean (`true`/`false`)
+  - The "classic" Lua way, with `1`/`nil`
+  - The "C" way, with `1`/`0`
+
+When implementing C modules for NodeMCU and needing to indicate an on/off setting, the preference is to do it as a boolean. In the interest of ease of use on the other hand, it is however nice to also support the other styles. The `luaL_totoggle` function provides just that.
+
 ### Declaring modules and ROTables in NodeMCU
 
 All NodeMCU C library modules should include the standard header "`module.h`".  This internally includes `lnodemcu.h` and these together provide the macros to enable declaration of NodeMCU modules and ROTables within them.  All ROtable support macros are either prefixed by `LRO_` (Lua Read Only) or in the case of table entries `LROT_`. 

docs/nodemcu-pil.md

@@ -6,7 +6,7 @@ NodeMCU developers are also able to develop and incorporate their own C modules
 
 Those developers who wish to develop or to modify existing C modules should have access to the LRM, PiL and NRM and familiarise themselves with these references.  These are the primary references; and this document does not repeat this content, but rather provide some NodeMCU-specific information to supplement it.
 
-From a perspective of developing C modules, there is very little difference from that of developing modules in standard Lua.  All of the standard Lua library modules (`bit`, `coroutine`, `debug`, `math`, `string`, `table`, `utf8`) use the C API for Lua and the NodeMCU versions have been updated to use NRM extensions. so their source code is available for browsing and using as style template (see the corresponding `lXXXlib.c` file in GitHub [NodeMCU lua53](../app/lua53) folder).
+From a perspective of developing C modules, there is very little difference from that of developing modules in standard Lua.  All of the standard Lua library modules (`bit`, `coroutine`, `debug`, `math`, `string`, `table`, `utf8`) use the C API for Lua and the NodeMCU versions have been updated to use NRM extensions. so their source code is available for browsing and using as style template (see the corresponding `lXXXlib.c` file in GitHub [NodeMCU lua53](../components/lua/lua-5.3) folder).
 
 The main functional change is that NodeMCU supports a read-only subclass of the `Table` type, known as a **`ROTable`**, which can be statically declared within the module source using static `const` declarations.  There are also limitations on the valid types for ROTable keys and value in order to ensure that these are consistent with static declaration; and hence ROTables are stored in code space (and therefore in flash memory on the IoT device). Hence unlike standard Lua tables, ROTables do not take up RAM resources. 
 
@@ -18,9 +18,9 @@ The `NODEMCU_MODULE` macro is used in each module to register it in an entry in
 -  All `ROM` entries will resolve globally
 -  The Lua runtime scans the `ROMentry` ROTable during its start up, and it will execute any non-NULL `CFunction` values in this table.  This enables C modules to hook in any one-time start-up functions if they are needed.
 
-Note that the standard `make` will include any modules found in the `app/modules` folder within a firmware build _if_ the corresponding `LUA_USE_MODULES_modname` macro has been defined. These defines are conventionally set in a common include file `user_modules.h`, and this practice is mandated for any user-submitted modules that are added to to the NodeMCU distribution.  However, this does not prevent developers adding their own local modules to the `app/modules` folder and simply defining the corresponding  `LUA_USE_MODULES_modname` inline.
+For a module to be included in the build, it has to be enabled in the sdkconfig file (e.g. via running `make menuconfig`). Some modules are enabled by default. Between compile time macros based on the sdkconfig and linker processing only the enabled modules are actually included into the firmware.
 
-This macro + linker approach renders the need for `luaL_reg` declarations and use of `luaL_openlib()` unnecessary, and these are not permitted in project-adopted `app/modules` files.
+This macro + linker approach renders the need for `luaL_reg` declarations and use of `luaL_openlib()` unnecessary, and these are not permitted in project-adopted `components/modules` files.
 
 Hence a NodeMCU C library module typically has a standard layout that parallels that of the standard Lua library modules and uses the same C API to access the Lua runtime:
 
@@ -33,10 +33,10 @@ Hence a NodeMCU C library module typically has a standard layout that parallels
    -  Whilst the ROTable search algorithm is a simply linear scan of the ROTable entries, the runtime also maintains a LRU cache of ROTable accesses, so typically over 95% of ROTable accesses bypass the linear scan and do a direct access to the appropriate entry.
    -  ROTables are also reasonable lightweight and well integrated into the Lua runtime, so the normal metamethod processing works well.  This means that developers can use the `__index` method to implement other key and value typed entries through an index function.
 -  NodeMCU modules are intended to be compilable against both our Lua 5.1 and Lua 5.3 runtimes.  The NRM discusses the implications and constraints here.  However note that:
-   -  We have back-ported many new Lua 5.3 features into the NodeMCU Lua 5.1 API, so in general you can use the 5.3 API to code your modules.  Again the NRM notes the exceptions where you will either need variant code or to decide to limit yourself to the the 5.3 runtime. In this last case the simplest approach is to `#if LUA_VERSION_NUM != 503` to disable the 5.3 content so that 5.1 build can compile and link.   Note that all modules currently in the `app/modules` folder will compile against and execute within both the Lua 5.1 and the 5.3 environments.
+   -  We have back-ported many new Lua 5.3 features into the NodeMCU Lua 5.1 API, so in general you can use the 5.3 API to code your modules.  Again the NRM notes the exceptions where you will either need variant code or to decide to limit yourself to the the 5.3 runtime. In this last case the simplest approach is to `#if LUA_VERSION_NUM != 503` to disable the 5.3 content so that 5.1 build can compile and link.   Note that all modules currently in the `components/modules` folder will compile against and execute within both the Lua 5.1 and the 5.3 environments.
    -  Lua 5.3 uses a 32-bit representation for all numerics with separate subtypes for integer (stored as a 32 bit signed integer) and float (stored as 32bit single precision float).  This achieves the same RAM storage density as Lua 5.1 integer builds without the loss of use of floating point when convenient.  We have therefore decided that there is no benefit in having a separate Integer 5.3 build variant.
  -  We recommend that developers make use of the full set of `luaL_` API calls to minimise code verbosity.  We have also added a couple of registry access optimisations that both simply and improve runtime performance when using the Lua registry for callback support.
     - `luaL_reref()` replaces an existing registry reference in place (or creates a new one if needed).  Less code and faster execution than a `luaL_unref()` plus `luaL_ref()` construct.
     - `luaL_unref2()` does the unref and set the static int hook to `LUA_NOREF`.
 
-Rather than include simple examples of module templates, we suggest that you review the modules in our GitHub repository, such as the [`utf8`](../app/lua53/lutf8lib.c) library.  Note that whilst all of the existing modules in `app/modules` folder compile and work, we plan to do a clean up of the core modules to ensure that they conform to best practice.
+Rather than include simple examples of module templates, we suggest that you review the modules in our GitHub repository, such as the [`utf8`](../components/lua/lua-5.3/lutf8lib.c) library.  Note that whilst all of the existing modules in `components/modules` folder compile and work, we plan to do a clean up of the core modules to ensure that they conform to best practice.

docs/requirements.txt

@@ -1 +1,2 @@
 mkdocs==1.2.2
+jinja2<3.1

extcomp-template/CMakeLists.txt

@@ -0,0 +1,20 @@
+# Update source files and includes as necessary
+idf_component_register(
+  SRCS
+    "mymod.c"
+  PRIV_INCLUDE_DIRS
+    "${CMAKE_CURRENT_BINARY_DIR}"
+  PRIV_REQUIRES
+    "base_nodemcu"
+    "lua"
+    "platform"
+)
+
+# To register the module with NodeMCU, use one of the below functions.
+# The name given MUST match the first argument to the NODEMCU_MODULE() in
+# the module's C file.
+#
+# For modules with a Kconfig option, use:
+extmod_register_conditional(MYMOD)
+# ...and for modules without a Kconfig option, instead use:
+#extmod_register_unconditional(MYMOD)

extcomp-template/Kconfig

@@ -0,0 +1,8 @@
+config NODEMCU_CMODULE_MYMOD
+    bool "External NodeMCU module: mymod"
+    default "y"
+    help
+        Includes the mymod module. This module is only an example for
+        showing how to use external modules. Note that the config option
+        name has to be prefixed with NODEMCU_CMODULE_ and the module
+        registered with extmod_register_conditional(MYMOD) in CMakeLists.txt

extcomp-template/mymod.c

@@ -0,0 +1,21 @@
+#include "module.h"
+
+static int lmymod_hello(lua_State *L)
+{
+  if (lua_isnoneornil(L, 1))
+    lua_pushliteral(L, "world");
+
+  lua_getglobal(L, "print");
+  lua_pushliteral(L, "Hello,");
+  lua_pushvalue(L, 1);
+  lua_call(L, 2, 0);
+
+  return 0;
+}
+
+
+LROT_BEGIN(mymod, NULL, 0)
+  LROT_FUNCENTRY(hello,   lmymod_hello)
+LROT_END(mymod, NULL, 0)
+
+NODEMCU_MODULE(MYMOD, "mymod", mymod, NULL);

mkdocs.yml

@@ -38,7 +38,6 @@ pages:
       - Lua 5.3 Support: 'lua53.md'
       - Lua Flash Store (LFS): 'lfs.md'
       - Filesystem on SD card: 'sdcard.md'
-      - Writing external C modules: 'modules/extmods.md'
   - C Modules:
       - 'adc': 'modules/adc.md'
       - 'bit': 'modules/bit.md'

sdk/esp32-esp-idf

@@ -1 +1 @@
-Subproject commit d83021a6e8550b4d462e11d61aaab0214dc03f5a
+Subproject commit 6407ecb3f8d2cc07c4c230e7e64f2046af5c86f7

tools/embed_lfs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-LUA_APP_SRC="$@"
+LUA_APP_SRC=("$@")
 
 MAP_FILE=build/nodemcu.map
 LUAC_OUTPUT=build/luac.out
@@ -15,43 +15,37 @@ if [ ! -f "${LUAC_CROSS}" ]; then
 	exit 1
 fi
 
-LFS_ADDR_SIZE=$(grep -E "0x[0-9a-f]+[ ]+0x[0-9a-f]+[ ]+esp-idf/embedded_lfs/libembedded_lfs.a\(lua.flash.store.reserved.S.obj\)" "${MAP_FILE}" | grep -v -w 0x0 | tr -s ' ')
-if [ -z "${LFS_ADDR_SIZE}" ]; then
-	echo "Error: LFS segment not found. Use 'make clean; make' perhaps?"
-	exit 1
+# Extract the start/end symbols of the LFS object, then calculate the
+# available size from that.
+LFS_ADDR=$(grep -E '0x[0-9a-f]+ +_binary_lua_flash_store_reserved_start' "${MAP_FILE}" | awk '{print $1}')
+LFS_ADDR_END=$(grep -E '0x[0-9a-f]+ +_binary_lua_flash_store_reserved_end' "${MAP_FILE}" | awk '{print $1}')
+if [ "${LFS_ADDR}" = "" ]
+then
+  echo "Error: LFS segment address not found"
+  exit 1
 fi
+LFS_SIZE=$((LFS_ADDR_END - LFS_ADDR))
 
-LFS_ADDR=$(echo "${LFS_ADDR_SIZE}" | cut -d ' ' -f 2)
-if [ -z "${LFS_ADDR}" ]; then
-	echo "Error: LFS segment address not found"
-	exit 1
-fi
-# The reported size is +4 due to the length field added by the IDF
-LFS_SIZE=$(( $(echo "${LFS_ADDR_SIZE}" | cut -d ' ' -f 3) - 4 ))
-if [ -z "${LFS_SIZE}" ]; then
-	echo "Error: LFS segment size not found"
-	exit 1
-fi
-
-echo "LFS segment address ${LFS_ADDR}, length ${LFS_SIZE}"
+printf "LFS segment address %s, length %s (0x%x)\n" "${LFS_ADDR}" "${LFS_SIZE}" "${LFS_SIZE}"
 
 if ${LUAC_CROSS} -v | grep -q 'Lua 5.1'
 then
   echo "Generating Lua 5.1 LFS image..."
-  ${LUAC_CROSS} -a ${LFS_ADDR} -m ${LFS_SIZE} -o ${LUAC_OUTPUT} ${LUA_APP_SRC}
+  ${LUAC_CROSS} -a "${LFS_ADDR}" -m ${LFS_SIZE} -o ${LUAC_OUTPUT} "${LUA_APP_SRC[@]}"
 else
   set -e
   echo "Generating intermediate Lua 5.3 LFS image..."
-  ${LUAC_CROSS} -f -m ${LFS_SIZE} -o ${LUAC_OUTPUT}.tmp ${LUA_APP_SRC}
+  ${LUAC_CROSS} -f -m ${LFS_SIZE} -o ${LUAC_OUTPUT}.tmp "${LUA_APP_SRC[@]}"
   echo "Converting to absolute LFS image..."
-  ${LUAC_CROSS} -F ${LUAC_OUTPUT}.tmp -a ${LFS_ADDR} -o ${LUAC_OUTPUT}
+  ${LUAC_CROSS} -F ${LUAC_OUTPUT}.tmp -a "${LFS_ADDR}" -o ${LUAC_OUTPUT}
   rm ${LUAC_OUTPUT}.tmp
 fi
+# shellcheck disable=SC2181
 if [ $? != 0 ]; then
 	echo "Error: luac.cross failed"
 	exit 1
 else
-  echo "Generated $(ls -l ${LUAC_OUTPUT} | cut -f5 -d' ') bytes of LFS data"
+  echo "Generated $(stat -c "%s" ${LUAC_OUTPUT}) bytes of LFS data"
 fi
 # cmake depencies don't seem to pick up the change to luac.out?
 rm -f build/lua.flash.store.reserved

tools/extmod/component.mk.template

@@ -1,16 +0,0 @@
-# This file is autogenerated! DO NOT EDIT!!
-# To add special makefile directives, create a module.mk file in your repo
-# Make sure you have this file in your .gitignore to avoid checking it in your module repo
-
-MODNAME="%%MODNAME%%"
-MODULE_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
-
-COMPONENT_ADD_LDFLAGS=-u $(MODNAME)_entry -l$(MODNAME)
-
-CFLAGS += \
-	-DEXTMODNAME=$(MODNAME) \
-	-Werror=unused-function \
-	-Werror=unused-but-set-variable \
-	-Werror=unused-variable 
-
--include $(MODULE_DIR)module.mk

tools/extmod/example/helloworld.c

@@ -1,61 +0,0 @@
-// Helloworld sample module
-
-#include "esp_log.h"
-#include "lauxlib.h"
-#include "lnodeaux.h"
-#include "module.h"
-
-static const char* HELLOWORLD_METATABLE = NODEMCU_MODULE_METATABLE();
-
-// hello_context_t struct contains information to wrap a "hello world object"
-typedef struct {
-    char* my_name;  // pointer to the greeter's name
-} hello_context_t;
-
-// Lua: helloworldobj:hello(text)
-static int helloworld_hello(lua_State* L) {
-    hello_context_t* context = (hello_context_t*)luaL_checkudata(L, 1, HELLOWORLD_METATABLE);
-    printf("Hello, %s: %s\n", context->my_name, luaL_optstring(L, 2, "How are you?"));
-    return 0;
-}
-
-// helloworld_delete is called on garbage collection
-static int helloworld_delete(lua_State* L) {
-    hello_context_t* context = (hello_context_t*)luaL_checkudata(L, 1, HELLOWORLD_METATABLE);
-    printf("Helloworld object with name '%s' garbage collected\n", context->my_name);
-    luaX_free_string(L, context->my_name);
-    return 0;
-}
-
-// Lua: modulename.new(string)
-static int helloworld_new(lua_State* L) {
-    //create a new lua userdata object and initialize to 0.
-    hello_context_t* context = (hello_context_t*)lua_newuserdata(L, sizeof(hello_context_t));
-
-    context->my_name = luaX_alloc_string(L, 1, 100);
-
-    luaL_getmetatable(L, HELLOWORLD_METATABLE);
-    lua_setmetatable(L, -2);
-
-    return 1;  //one object returned, the helloworld context wrapped in a lua userdata object
-}
-
-// object function map:
-LROT_BEGIN(helloworld_metatable)
-LROT_FUNCENTRY(hello, helloworld_hello)
-LROT_FUNCENTRY(__gc, helloworld_delete)
-LROT_TABENTRY(__index, helloworld_metatable)
-LROT_END(helloworld_metatable, NULL, 0)
-
-// Module function map
-LROT_BEGIN(module)
-LROT_FUNCENTRY(new, helloworld_new)
-LROT_END(module, NULL, 0)
-
-// module_init is invoked on device startup
-static int module_init(lua_State* L) {
-    luaL_rometatable(L, HELLOWORLD_METATABLE, (void*)helloworld_metatable_map);  // create metatable for helloworld
-    return 0;
-}
-
-NODEMCU_MODULE_STD();  // define Lua entries

tools/extmod/extmod.sh

@@ -1,166 +0,0 @@
-#!/bin/bash
-
-# External modules update script.
-
-# This script parses the repository root extmods.ini file to locate
-# external modules to download
-
-export EXTMOD_DIR="components/modules/external"      # Location where to store external modules:
-EXTMOD_BIN_PATH="./tools/extmod"                     # Location of this script
-TEMPLATE_MK="$EXTMOD_BIN_PATH/component.mk.template" # Location of the template component.mk
-
-# Include the ini file reader script
-. $EXTMOD_BIN_PATH/read_ini.sh
-
-# Returns the given value in the INI file, passing section and value
-function sectionVar() {
-    local varname="INI__$1__$2"
-    echo "${!varname}"
-}
-
-# helper pushd to make it silent
-function pushd() {
-    command pushd "$@" >/dev/null
-}
-
-# helper popd to make it silent
-function popd() {
-    command popd >/dev/null
-}
-
-function usage() {
-
-    echo ""
-    echo "extmod.sh - Manages external modules"
-    echo "Usage:"
-    echo "extmod.sh <commands>"
-    echo "update : Parses extmods.ini and updates all modules"
-    echo "clean  : Effectively cleans the contents of the external modules directory ($EXTMOD_DIR)"
-    echo ""
-
-}
-
-# Generic command line parser
-function readCommandLine() {
-    while test ${#} -gt 0; do
-        case "$1" in
-        "clean")
-            CLEAN=1
-            ;;
-        "update")
-            UPDATE=1
-            ;;
-        *)
-            echo -e "Error: Unrecognized parameter\n"
-            usage
-            exit 1
-            ;;
-        esac
-        shift
-    done
-}
-
-function updateMod() {
-    local modname="$1"
-    local url="$(sectionVar "$modname" "url")"
-    local ref="$(sectionVar "$modname" "ref")"
-    local disabled="$(sectionVar "$modname" "disabled")"
-    local path="$EXTMOD_DIR/$modname"
-    local component_mk="$path/component.mk"
-
-    if [[ ! -d "$path" ]]; then
-        echo "$modname not present. Downloading from $url ..."
-        if ! git clone --quiet "$url" -b "$ref" "$path"; then
-            echo "Error cloning $modname in $url"
-            return 1
-        fi
-        # Add "component.mk" to local repo gitignore
-        echo "component.mk" >>"$path/.git/info/exclude"
-    fi
-    echo "Updating $modname ..."
-
-    if ! pushd "$path"; then
-        echo "Cannot change to $path".
-        return 1
-    fi
-    if ! git status >/dev/null; then
-        echo "Error processing $modname. Error reading git repo status."
-        popd
-        return 1
-    fi
-    if [ "$(git status --short)" == "" ]; then
-        if ! git fetch --quiet; then
-            echo "Error fetching $modname"
-            popd
-            return 1
-        fi
-        if ! git checkout "$ref" --quiet; then
-            echo "Error setting ref $ref in $modname. Does $ref exist?"
-            popd
-            return 1
-        fi
-        if ! git clean -d -f --quiet; then
-            echo "Error repo $modname after checkout."
-            popd
-            return 1
-        fi
-        # check if HEAD was detached (like when checking out a tag or commit)
-        if git symbolic-ref HEAD 2>/dev/null; then
-            # This is a branch. Update it.
-            if ! git pull --quiet; then
-                echo "Error pulling $ref from $url."
-                popd
-                return 1
-            fi
-        fi
-
-    else
-        echo "$modname working directory in $path is not clean. Skipping..."
-        popd
-        return 0
-    fi
-    popd
-
-    if [ "$disabled" != "1" ]; then
-        if ! sed 's/%%MODNAME%%/'"$modname"'/g' "$TEMPLATE_MK" >"$component_mk"; then
-            echo "Error generating $component_mk"
-            return 1
-        fi
-    else
-        echo "Warning: Module $modname is disabled and won't be included in build"
-        [ -f "$component_mk" ] && rm "$component_mk"
-    fi
-    echo "Successfully updated $modname."
-    return 0
-}
-
-function update() {
-
-    if ! read_ini "extmods.ini"; then
-        echo "Error reading extmods.ini"
-    fi
-
-    mkdir -p "$EXTMOD_DIR"
-
-    for modname in $INI__ALL_SECTIONS; do
-        if ! updateMod "$modname"; then
-            echo "Error updating $modname"
-            return 1
-        fi
-    done
-    echo "Successfully updated all modules"
-}
-
-function main() {
-    if [ "$CLEAN" == "1" ]; then
-        echo "Cleaning ${EXTMOD_DIR:?} ..."
-        rm -rf "${EXTMOD_DIR:?}/"*
-    fi
-
-    if [ "$UPDATE" == "1" ]; then
-        update
-    fi
-}
-
-readCommandLine "$@"
-main

tools/extmod/read_ini.sh

@@ -1,256 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2009    Kevin Porter / Advanced Web Construction Ltd
-#                       (http://coding.tinternet.info, http://webutils.co.uk)
-# Copyright (c) 2010-2014     Ruediger Meier <sweet_f_a@gmx.de>
-#                             (https://github.com/rudimeier/)
-#
-# License: BSD-3-Clause, see LICENSE file
-#
-# Simple INI file parser.
-#
-# See README for usage.
-#
-#
-
-function read_ini() {
-    # Be strict with the prefix, since it's going to be run through eval
-    function check_prefix() {
-        if ! [[ "${VARNAME_PREFIX}" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
-            echo "read_ini: invalid prefix '${VARNAME_PREFIX}'" >&2
-            return 1
-        fi
-    }
-
-    function check_ini_file() {
-        if [ ! -r "$INI_FILE" ]; then
-            echo "read_ini: '${INI_FILE}' doesn't exist or not" \
-                "readable" >&2
-            return 1
-        fi
-    }
-
-    # enable some optional shell behavior (shopt)
-    function pollute_bash() {
-        if ! shopt -q extglob; then
-            SWITCH_SHOPT="${SWITCH_SHOPT} extglob"
-        fi
-        if ! shopt -q nocasematch; then
-            SWITCH_SHOPT="${SWITCH_SHOPT} nocasematch"
-        fi
-        shopt -q -s ${SWITCH_SHOPT}
-    }
-
-    # unset all local functions and restore shopt settings before returning
-    # from read_ini()
-    function cleanup_bash() {
-        shopt -q -u ${SWITCH_SHOPT}
-        unset -f check_prefix check_ini_file pollute_bash cleanup_bash
-    }
-
-    local INI_FILE=""
-    local INI_SECTION=""
-
-    # {{{ START Deal with command line args
-
-    # Set defaults
-    local BOOLEANS=1
-    local VARNAME_PREFIX=INI
-    local CLEAN_ENV=0
-
-    # {{{ START Options
-
-    # Available options:
-    #	--boolean		Whether to recognise special boolean values: ie for 'yes', 'true'
-    #					and 'on' return 1; for 'no', 'false' and 'off' return 0. Quoted
-    #					values will be left as strings
-    #					Default: on
-    #
-    #	--prefix=STRING	String to begin all returned variables with (followed by '__').
-    #					Default: INI
-    #
-    #	First non-option arg is filename, second is section name
-
-    while [ $# -gt 0 ]; do
-
-        case $1 in
-
-        --clean | -c)
-            CLEAN_ENV=1
-            ;;
-
-        --booleans | -b)
-            shift
-            BOOLEANS=$1
-            ;;
-
-        --prefix | -p)
-            shift
-            VARNAME_PREFIX=$1
-            ;;
-
-        *)
-            if [ -z "$INI_FILE" ]; then
-                INI_FILE=$1
-            else
-                if [ -z "$INI_SECTION" ]; then
-                    INI_SECTION=$1
-                fi
-            fi
-            ;;
-
-        esac
-
-        shift
-    done
-
-    if [ -z "$INI_FILE" ] && [ "${CLEAN_ENV}" = 0 ]; then
-        echo -e "Usage: read_ini [-c] [-b 0| -b 1]] [-p PREFIX] FILE" \
-            "[SECTION]\n  or   read_ini -c [-p PREFIX]" >&2
-        cleanup_bash
-        return 1
-    fi
-
-    if ! check_prefix; then
-        cleanup_bash
-        return 1
-    fi
-
-    local INI_ALL_VARNAME="${VARNAME_PREFIX}__ALL_VARS"
-    local INI_ALL_SECTION="${VARNAME_PREFIX}__ALL_SECTIONS"
-    local INI_NUMSECTIONS_VARNAME="${VARNAME_PREFIX}__NUMSECTIONS"
-    if [ "${CLEAN_ENV}" = 1 ]; then
-        eval unset "\$${INI_ALL_VARNAME}"
-    fi
-    unset ${INI_ALL_VARNAME}
-    unset ${INI_ALL_SECTION}
-    unset ${INI_NUMSECTIONS_VARNAME}
-
-    if [ -z "$INI_FILE" ]; then
-        cleanup_bash
-        return 0
-    fi
-
-    if ! check_ini_file; then
-        cleanup_bash
-        return 1
-    fi
-
-    # Sanitise BOOLEANS - interpret "0" as 0, anything else as 1
-    if [ "$BOOLEANS" != "0" ]; then
-        BOOLEANS=1
-    fi
-
-    # }}} END Options
-
-    # }}} END Deal with command line args
-
-    local LINE_NUM=0
-    local SECTIONS_NUM=0
-    local SECTION=""
-
-    # IFS is used in "read" and we want to switch it within the loop
-    local IFS=$' \t\n'
-    local IFS_OLD="${IFS}"
-
-    # we need some optional shell behavior (shopt) but want to restore
-    # current settings before returning
-    local SWITCH_SHOPT=""
-    pollute_bash
-
-    while read -r line || [ -n "$line" ]; do
-        #echo line = "$line"
-
-        ((LINE_NUM++))
-
-        # Skip blank lines and comments
-        if [ -z "$line" -o "${line:0:1}" = ";" -o "${line:0:1}" = "#" ]; then
-            continue
-        fi
-
-        # Section marker?
-        if [[ "${line}" =~ ^\[[a-zA-Z0-9_]{1,}\]$ ]]; then
-
-            # Set SECTION var to name of section (strip [ and ] from section marker)
-            SECTION="${line#[}"
-            SECTION="${SECTION%]}"
-            eval "${INI_ALL_SECTION}=\"\${${INI_ALL_SECTION}# } $SECTION\""
-            ((SECTIONS_NUM++))
-
-            continue
-        fi
-
-        # Are we getting only a specific section? And are we currently in it?
-        if [ ! -z "$INI_SECTION" ]; then
-            if [ "$SECTION" != "$INI_SECTION" ]; then
-                continue
-            fi
-        fi
-
-        # Valid var/value line? (check for variable name and then '=')
-        if ! [[ "${line}" =~ ^[a-zA-Z0-9._]{1,}[[:space:]]*= ]]; then
-            echo "Error: Invalid line:" >&2
-            echo " ${LINE_NUM}: $line" >&2
-            cleanup_bash
-            return 1
-        fi
-
-        # split line at "=" sign
-        IFS="="
-        read -r VAR VAL <<<"${line}"
-        IFS="${IFS_OLD}"
-
-        # delete spaces around the equal sign (using extglob)
-        VAR="${VAR%%+([[:space:]])}"
-        VAL="${VAL##+([[:space:]])}"
-        VAR=$(echo $VAR)
-
-        # Construct variable name:
-        # ${VARNAME_PREFIX}__$SECTION__$VAR
-        # Or if not in a section:
-        # ${VARNAME_PREFIX}__$VAR
-        # In both cases, full stops ('.') are replaced with underscores ('_')
-        if [ -z "$SECTION" ]; then
-            VARNAME=${VARNAME_PREFIX}__${VAR//./_}
-        else
-            VARNAME=${VARNAME_PREFIX}__${SECTION}__${VAR//./_}
-        fi
-        eval "${INI_ALL_VARNAME}=\"\${${INI_ALL_VARNAME}# } ${VARNAME}\""
-
-        if [[ "${VAL}" =~ ^\".*\"$ ]]; then
-            # remove existing double quotes
-            VAL="${VAL##\"}"
-            VAL="${VAL%%\"}"
-        elif [[ "${VAL}" =~ ^\'.*\'$ ]]; then
-            # remove existing single quotes
-            VAL="${VAL##\'}"
-            VAL="${VAL%%\'}"
-        elif [ "$BOOLEANS" = 1 ]; then
-            # Value is not enclosed in quotes
-            # Booleans processing is switched on, check for special boolean
-            # values and convert
-
-            # here we compare case insensitive because
-            # "shopt nocasematch"
-            case "$VAL" in
-            yes | true | on)
-                VAL=1
-                ;;
-            no | false | off)
-                VAL=0
-                ;;
-            esac
-        fi
-
-        # enclose the value in single quotes and escape any
-        # single quotes and backslashes that may be in the value
-        VAL="${VAL//\\/\\\\}"
-        VAL="\$'${VAL//\'/\'}'"
-
-        eval "$VARNAME=$VAL"
-    done <"${INI_FILE}"
-
-    # return also the number of parsed sections
-    eval "$INI_NUMSECTIONS_VARNAME=$SECTIONS_NUM"
-
-    cleanup_bash
-}