Refactor to clean up and fix console handling (#3666)

* Refactor into new 'console' module.

A breaking change, but should finally see us move away from the chronic edge
cases and inconsistent behaviour we have while trying to shoe-horn the
usb-serial-jtag and cdc-acm consoles into uart behaviour and assumptions.

* Fix and document console.write()

Added example on using framed data transmission over the console.

* fixup uart examples

* Add workaround for silently dropped console output.

* Add file upload helper script for console module.

Plus, it can serve as a reference for any IDEs which may need/want
updating.

* Fixup really silly copy/paste error.

* Make upload-file.py work better on CDC-ACM console.

* Updated console module doc with CDC-ACM info.

* Load file in binary mode in upload-file.py.

components/base_nodemcu/user_main.c

@@ -11,47 +11,18 @@
 #include "lua.h"
 #include "linput.h"
 #include "platform.h"
-#include <string.h>
-#include <stdlib.h>
-#include <fcntl.h>
 #include "sdkconfig.h"
 #include "esp_system.h"
 #include "esp_event.h"
 #include "esp_spiffs.h"
 #include "esp_netif.h"
-#include "esp_vfs_dev.h"
-#include "esp_vfs_cdcacm.h"
-#include "esp_vfs_usb_serial_jtag.h"
-#include "driver/usb_serial_jtag.h"
 #include "nvs_flash.h"
 
 #include "task/task.h"
-#include "sections.h"
 #include "nodemcu_esp_event.h"
 
 #include "freertos/FreeRTOS.h"
-#include "freertos/task.h"
+#include "freertos/semphr.h"
-#include "freertos/queue.h"
-
-#define SIG_LUA 0
-#define SIG_UARTINPUT 1
-
-// Line ending config from Kconfig
-#if CONFIG_NEWLIB_STDIN_LINE_ENDING_CRLF
-# define RX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CRLF
-#elif CONFIG_NEWLIB_STDIN_LINE_ENDING_CR
-# define RX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CR
-#else
-# define RX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_LF
-#endif
-
-#if CONFIG_NEWLIB_STDOUT_LINE_ENDING_CRLF
-# define TX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CRLF
-#elif CONFIG_NEWLIB_STDOUT_LINE_ENDING_CR
-# define TX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CR
-#else
-# define TX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_LF
-#endif
 
 
 // We don't get argument size data from the esp_event dispatch, so it's
@@ -71,8 +42,6 @@ typedef struct {
 static task_handle_t     relayed_event_task;
 static SemaphoreHandle_t relayed_event_handled;
 
-static task_handle_t lua_feed_task;
-
 
 // This function runs in the context of the system default event loop RTOS task
 static void relay_default_loop_events(
@@ -166,129 +135,10 @@ static void nodemcu_init(void)
 }
 
 
-static bool have_console_on_data_cb(void)
-{
-#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
-  return uart_has_on_data_cb(CONFIG_ESP_CONSOLE_UART_NUM);
-#else
-  return false;
-#endif
-}
-
-
-static void console_nodemcu_task(task_param_t param, task_prio_t prio)
-{
-  (void)prio;
-  char c = (char)param;
-
-  if (run_input)
-    feed_lua_input(&c, 1);
-
-#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
-  if (have_console_on_data_cb())
-    uart_feed_data(CONFIG_ESP_CONSOLE_UART_NUM, &c, 1);
-#endif
-
-  // The IDF doesn't seem to honor setvbuf(stdout, NULL, _IONBF, 0) :(
-  fsync(fileno(stdout));
-}
-
-
-static void console_task(void *)
-{
-  for (;;)
-  {
-    /* We can't use a large read buffer here as some console choices
-     * (e.g. usb-serial-jtag) don't support read timeouts/partial reads,
-     * which breaks the echo support and makes for a bad user experience.
-     */
-    char c;
-    ssize_t n = read(fileno(stdin), &c, 1);
-    if (n > 0 && (run_input || have_console_on_data_cb()))
-    {
-      if (!task_post_block_high(lua_feed_task, (task_param_t)c))
-      {
-        NODE_ERR("Lost console input data?!\n");
-      }
-    }
-  }
-}
-
-
-static void console_init(void)
-{
-  fflush(stdout);
-  fsync(fileno(stdout));
-
-  /* Disable buffering */
-  setvbuf(stdin, NULL, _IONBF, 0);
-  setvbuf(stdout, NULL, _IONBF, 0);
-
-  /* Disable non-blocking mode */
-  fcntl(fileno(stdin), F_SETFL, 0);
-  fcntl(fileno(stdout), F_SETFL, 0);
-
-#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
-  /* Based on console/advanced example */
-
-  esp_vfs_dev_uart_port_set_rx_line_endings(
-    CONFIG_ESP_CONSOLE_UART_NUM, RX_LINE_ENDINGS_CFG);
-  esp_vfs_dev_uart_port_set_tx_line_endings(
-    CONFIG_ESP_CONSOLE_UART_NUM, TX_LINE_ENDINGS_CFG);
-
-  /* Configure UART. Note that REF_TICK is used so that the baud rate remains
-   * correct while APB frequency is changing in light sleep mode.
-   */
-  const uart_config_t uart_config = {
-    .baud_rate = CONFIG_ESP_CONSOLE_UART_BAUDRATE,
-    .data_bits = UART_DATA_8_BITS,
-    .parity = UART_PARITY_DISABLE,
-    .stop_bits = UART_STOP_BITS_1,
-#if SOC_UART_SUPPORT_REF_TICK
-    .source_clk = UART_SCLK_REF_TICK,
-#elif SOC_UART_SUPPORT_XTAL_CLK
-    .source_clk = UART_SCLK_XTAL,
-#endif
-  };
-  /* Install UART driver for interrupt-driven reads and writes */
-  uart_driver_install(CONFIG_ESP_CONSOLE_UART_NUM, 256, 0, 0, NULL, 0);
-  uart_param_config(CONFIG_ESP_CONSOLE_UART_NUM, &uart_config);
-
-  /* Tell VFS to use UART driver */
-  esp_vfs_dev_uart_use_driver(CONFIG_ESP_CONSOLE_UART_NUM);
-
-#elif CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG
-  /* Based on @pjsg's work */
-
-  esp_vfs_dev_usb_serial_jtag_set_rx_line_endings(RX_LINE_ENDINGS_CFG);
-  esp_vfs_dev_usb_serial_jtag_set_tx_line_endings(TX_LINE_ENDINGS_CFG);
-
-  usb_serial_jtag_driver_config_t usb_serial_jtag_config =
-    USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT();
-  /* Install USB-SERIAL-JTAG driver for interrupt-driven reads and write */
-  usb_serial_jtag_driver_install(&usb_serial_jtag_config);
-
-  esp_vfs_usb_serial_jtag_use_driver();
-#elif CONFIG_ESP_CONSOLE_USB_CDC
-  /* Based on console/advanced_usb_cdc */
-
-  esp_vfs_dev_cdcacm_set_rx_line_endings(RX_LINE_ENDINGS_CFG);
-  esp_vfs_dev_cdcacm_set_tx_line_endings(TX_LINE_ENDINGS_CFG);
-#else
-# error "Unsupported console type"
-#endif
-
-  xTaskCreate(
-    console_task, "console", 1024, NULL, ESP_TASK_MAIN_PRIO+1, NULL);
-}
-
-
 void __attribute__((noreturn)) app_main(void)
 {
   task_init();
 
-  lua_feed_task = task_get_id(console_nodemcu_task);
-
   relayed_event_handled = xSemaphoreCreateBinary();
   relayed_event_task = task_get_id(handle_default_loop_event);
 
@@ -304,8 +154,6 @@ void __attribute__((noreturn)) app_main(void)
   nvs_flash_init ();
   esp_netif_init ();
 
-  console_init();
-
   start_lua ();
   task_pump_messages ();
   __builtin_unreachable ();

components/lua/Kconfig

@@ -129,7 +129,7 @@ menu "Lua configuration"
       bool
       default y
       select NODEMCU_CMODULE_PIPE
-      select NODEMCU_CMODULE_UART
+      select NODEMCU_CMODULE_CONSOLE
       select LUA_BUILTIN_DEBUG
 
     choice LUA_INIT_STRING

components/modules/CMakeLists.txt

@@ -11,6 +11,7 @@ set(module_srcs
   "bit.c"
   "bthci.c"
   "common.c"
+  "console.c"
   "crypto.c"
   "dht.c"
   "encoder.c"
@@ -35,6 +36,7 @@ set(module_srcs
   "rmt.c"
   "rtcmem.c"
   "qrcodegen.c"
+  "serial_common.c"
   "sigma_delta.c"
   "sjson.c"
   "sodium.c"

components/modules/Kconfig

@@ -28,6 +28,12 @@ menu "NodeMCU modules"
         help
             Includes the can module.
 
+    config NODEMCU_CMODULE_CONSOLE
+        bool "Console module"
+        default y
+        help
+            Includes the console module (required by our Lua VM).
+
     config NODEMCU_CMODULE_CRYPTO
         bool "Crypto module"
         default "n"
@@ -339,6 +345,6 @@ menu "NodeMCU modules"
         bool "UART module"
         default y
         help
-            Includes the UART module (required by our Lua VM).
+            Includes the UART module.
 
 endmenu

components/modules/console.c

@@ -0,0 +1,263 @@
+#include "module.h"
+#include "platform.h"
+#include "lauxlib.h"
+#include "linput.h"
+#include "serial_common.h"
+#include "task/task.h"
+
+#include "esp_vfs_dev.h"
+#include "esp_vfs_cdcacm.h"
+#include "esp_vfs_usb_serial_jtag.h"
+#include "driver/usb_serial_jtag.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+
+// Line ending config from Kconfig
+#if CONFIG_NEWLIB_STDIN_LINE_ENDING_CRLF
+# define RX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CRLF
+#elif CONFIG_NEWLIB_STDIN_LINE_ENDING_CR
+# define RX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CR
+#else
+# define RX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_LF
+#endif
+
+#if CONFIG_NEWLIB_STDOUT_LINE_ENDING_CRLF
+# define TX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CRLF
+#elif CONFIG_NEWLIB_STDOUT_LINE_ENDING_CR
+# define TX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_CR
+#else
+# define TX_LINE_ENDINGS_CFG ESP_LINE_ENDINGS_LF
+#endif
+
+typedef enum { NONINTERACTIVE, INTERACTIVE } console_mode_t;
+
+static serial_input_cfg_t *cb_cfg;
+static task_handle_t feed_lua_task;
+
+
+// --- Console input task related -----------------------------------
+
+static void console_feed_lua(task_param_t param, task_prio_t prio)
+{
+  (void)prio;
+  char c = (char)param;
+
+  if (run_input)
+    feed_lua_input(&c, 1);
+
+  if (serial_input_has_data_cb(cb_cfg))
+    serial_input_feed_data(cb_cfg, &c, 1);
+
+  // The IDF doesn't seem to honor setvbuf(stdout, NULL, _IONBF, 0) :(
+  fflush(stdout);
+  fsync(fileno(stdout));
+}
+
+
+static void console_task(void *)
+{
+  for (;;)
+  {
+    // TODO: Support linenoise editing here as an option?
+    // The run_input switch would need to also control whether we do
+    // linenoise or raw byte input, to allow for binary xfers.
+    // But, we would have a big race condition here as the execution
+    // of the last line happens after we've already started reading the
+    // next one. We'd have to use a newer version of linenoise than what
+    // the IDF has, so we get the async interface. Plus switch everything to
+    // using select() before picking which input method we're using.
+    // For the race condition, would it be sufficient to wait for the next
+    // next prompt display to be reasonably certain it's switched?
+    // But even the prompt handling would be problematic with linenoise as
+    // that's fixed on linenoiseEditStart(). To solve that we'd need to
+    // be running the console within the LVM task, synchronously, but we
+    // can't do that because we need the LVM accessible to handle events.
+    // These are incompatible design constraints, sigh.
+
+    /* We can't use a large read buffer here as some console choices
+     * (e.g. usb-serial-jtag) don't support read timeouts/partial reads,
+     * which breaks the echo support and makes for a bad user experience.
+     */
+    char c;
+    ssize_t n = read(fileno(stdin), &c, 1);
+    if (n > 0 && (run_input || serial_input_has_data_cb(cb_cfg)))
+    {
+      if (!task_post_block_high(feed_lua_task, (task_param_t)c))
+      {
+        NODE_ERR("Lost console input data?!\n");
+      }
+    }
+  }
+}
+
+
+static void console_init(void)
+{
+  fflush(stdout);
+  fsync(fileno(stdout));
+
+  /* Disable buffering */
+  setvbuf(stdin, NULL, _IONBF, 0);
+  setvbuf(stdout, NULL, _IONBF, 0);
+
+  /* Disable non-blocking mode */
+  fcntl(fileno(stdin), F_SETFL, 0);
+  fcntl(fileno(stdout), F_SETFL, 0);
+
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
+  /* Based on console/advanced example */
+
+  esp_vfs_dev_uart_port_set_rx_line_endings(
+    CONFIG_ESP_CONSOLE_UART_NUM, RX_LINE_ENDINGS_CFG);
+  esp_vfs_dev_uart_port_set_tx_line_endings(
+    CONFIG_ESP_CONSOLE_UART_NUM, TX_LINE_ENDINGS_CFG);
+
+  /* Configure UART. Note that REF_TICK is used so that the baud rate remains
+   * correct while APB frequency is changing in light sleep mode.
+   */
+  const uart_config_t uart_config = {
+    .baud_rate = CONFIG_ESP_CONSOLE_UART_BAUDRATE,
+    .data_bits = UART_DATA_8_BITS,
+    .parity = UART_PARITY_DISABLE,
+    .stop_bits = UART_STOP_BITS_1,
+#if SOC_UART_SUPPORT_REF_TICK
+    .source_clk = UART_SCLK_REF_TICK,
+#elif SOC_UART_SUPPORT_XTAL_CLK
+    .source_clk = UART_SCLK_XTAL,
+#endif
+  };
+  /* Install UART driver for interrupt-driven reads and writes */
+  uart_driver_install(CONFIG_ESP_CONSOLE_UART_NUM, 256, 0, 0, NULL, 0);
+  uart_param_config(CONFIG_ESP_CONSOLE_UART_NUM, &uart_config);
+
+  /* Tell VFS to use UART driver */
+  esp_vfs_dev_uart_use_driver(CONFIG_ESP_CONSOLE_UART_NUM);
+
+#elif CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG
+  /* Based on @pjsg's work */
+
+  esp_vfs_dev_usb_serial_jtag_set_rx_line_endings(RX_LINE_ENDINGS_CFG);
+  esp_vfs_dev_usb_serial_jtag_set_tx_line_endings(TX_LINE_ENDINGS_CFG);
+
+  usb_serial_jtag_driver_config_t usb_serial_jtag_config =
+    USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT();
+  /* Install USB-SERIAL-JTAG driver for interrupt-driven reads and write */
+  usb_serial_jtag_driver_install(&usb_serial_jtag_config);
+
+  esp_vfs_usb_serial_jtag_use_driver();
+#elif CONFIG_ESP_CONSOLE_USB_CDC
+  /* Based on console/advanced_usb_cdc */
+
+  esp_vfs_dev_cdcacm_set_rx_line_endings(RX_LINE_ENDINGS_CFG);
+  esp_vfs_dev_cdcacm_set_tx_line_endings(TX_LINE_ENDINGS_CFG);
+#else
+# error "Unsupported console type"
+#endif
+
+  xTaskCreate(
+    console_task, "console", configMINIMAL_STACK_SIZE,
+    NULL, ESP_TASK_MAIN_PRIO+1, NULL);
+}
+
+
+// --- Lua interface related ----------------------------------------
+
+static int retrying_write(const char *buf, size_t len)
+{
+  size_t written = 0;
+  while (written < len)
+  {
+    // At least the USB-Serial-JTAG appears to silently drop characters
+    // sometimes when writing more than 255 bytes, so we break such strings
+    // up into multiple calls as a workaround.
+    const size_t MAX_LEN = 255;
+    size_t left = len - written;
+    size_t to_write = left > MAX_LEN ? MAX_LEN : left;
+    size_t n = fwrite(buf + written, 1, to_write, stdout);
+    // Additionally, we have to explicitly flush after each chunk we've written.
+    fflush(stdout);
+    fsync(fileno(stdout));
+
+    if (n > 0)
+      written += n;
+    else if (ferror(stdout))
+      break;
+    else
+      vTaskDelay(1);
+  }
+  return written;
+}
+
+
+// Lua: console.on("method", [number/char], function)
+static int console_on(lua_State *L)
+{
+  return serial_input_register(L, cb_cfg);
+}
+
+
+// Lua: console.mode(onoff)
+static int console_mode(lua_State *L)
+{
+  switch (luaL_checkint(L, 1))
+  {
+    case NONINTERACTIVE: run_input = false; break;
+    case INTERACTIVE: run_input = true; break;
+    default: luaL_error(L, "invalid mode");
+  }
+  return 0;
+}
+
+
+// Lua: console.write(str_or_num [, str_or_num2 ... ])
+static int console_write(lua_State *L)
+{
+  int total = lua_gettop(L);
+  for (int s = 1; s <= total; ++s)
+  {
+    if (lua_type(L, s) == LUA_TSTRING)
+    {
+      size_t len = 0;
+      const char *buf = lua_tolstring(L, s, &len);
+      retrying_write(buf, len);
+    }
+    else if (lua_isnumber(L, s))
+    {
+      int n = lua_tointeger(L, s);
+      if (n < 0 || n > 255)
+        return luaL_error(L, "invalid number");
+      char ch = n;
+      retrying_write(&ch, 1);
+    }
+  }
+  return 0;
+}
+
+
+// Module function map
+LROT_BEGIN(console, NULL, 0)
+  LROT_FUNCENTRY( mode,               console_mode      )
+  LROT_FUNCENTRY( on,                 console_on        )
+  LROT_FUNCENTRY( write,              console_write     )
+  LROT_NUMENTRY( INTERACTIVE,         INTERACTIVE       )
+  LROT_NUMENTRY( NONINTERACTIVE,      NONINTERACTIVE    )
+LROT_END(console, NULL, 0)
+
+
+int luaopen_console( lua_State *L ) {
+  cb_cfg = serial_input_new();
+  if (!cb_cfg)
+    return luaL_error(L, "out of mem");
+
+  feed_lua_task = task_get_id(console_feed_lua);
+
+  console_init();
+
+  return 0;
+}
+
+NODEMCU_MODULE(CONSOLE, "console", console, luaopen_console);

components/modules/serial_common.c

@@ -0,0 +1,197 @@
+// Common routines for handling serial input data
+
+#include "serial_common.h"
+#include "lauxlib.h"
+#include <string.h>
+
+// This is the historical max value
+#define MAX_SERIAL_INPUT 255
+
+struct serial_input_cfg {
+  int receive_ref;
+  int error_ref;
+  char *line_buffer;
+  size_t line_buffer_size;
+  size_t line_position;
+  uint16_t need_len;
+  int16_t end_char;
+};
+
+
+static const char nostack[] = "out of stack";
+
+static bool serial_input_invoke(int ref, const char *buf, size_t len)
+{
+  if(ref == LUA_NOREF || !buf || len == 0)
+    return false;
+
+  lua_State *L = lua_getstate();
+
+  int top = lua_gettop(L);
+  luaL_checkstack(L, 2, nostack);
+  lua_rawgeti(L, LUA_REGISTRYINDEX, ref);
+  lua_pushlstring(L, buf, len);
+  luaL_pcallx(L, 1, 0);
+  lua_settop(L, top);
+  return true;
+}
+
+
+serial_input_cfg_t *serial_input_new(void)
+{
+  serial_input_cfg_t *cfg = calloc(1, sizeof(serial_input_cfg_t));
+  if (!cfg)
+    return NULL;
+  cfg->receive_ref = cfg->error_ref = LUA_NOREF;
+  cfg->end_char = -1;
+  return cfg;
+}
+
+
+void serial_input_free(lua_State *L, serial_input_cfg_t *cfg)
+{
+  if (cfg->receive_ref != LUA_NOREF)
+    luaL_unref(L, LUA_REGISTRYINDEX, cfg->receive_ref);
+  if (cfg->error_ref != LUA_NOREF)
+    luaL_unref(L, LUA_REGISTRYINDEX, cfg->error_ref);
+  free(cfg->line_buffer);
+  free(cfg);
+}
+
+
+bool serial_input_dispatch_data(serial_input_cfg_t *cfg, const char *buf, size_t len)
+{
+  if (!cfg)
+    return false;
+  else
+    return serial_input_invoke(cfg->receive_ref, buf, len);
+}
+
+
+bool serial_input_report_error(serial_input_cfg_t *cfg, const char *buf, size_t len)
+{
+  if (!cfg)
+    return false;
+  else
+    return serial_input_invoke(cfg->error_ref, buf, len);
+}
+
+
+void serial_input_feed_data(serial_input_cfg_t *cfg, const char *buf, size_t len)
+{
+  if (!cfg || !cfg->line_buffer || !buf || !len)
+    return;
+
+  const uint16_t need_len = cfg->need_len;
+  const int16_t end_char = cfg->end_char;
+  const size_t max_wanted =
+    (end_char >= 0 && need_len == 0) ? cfg->line_buffer_size : need_len;
+
+  for (unsigned i = 0; i < len; ++i)
+  {
+    char ch = buf[i];
+    cfg->line_buffer[cfg->line_position] = ch;
+    cfg->line_position++;
+
+    bool at_end = (cfg->line_position >= max_wanted);
+    bool end_char_found =
+      (end_char >= 0 && (uint8_t)ch == (uint8_t)end_char);
+    if (at_end || end_char_found) {
+      // Reset line position early so callback can resize line_buffer if desired
+      int n = cfg->line_position;
+      cfg->line_position = 0;
+      serial_input_dispatch_data(cfg, cfg->line_buffer, n);
+    }
+  }
+}
+
+
+bool serial_input_has_data_cb(serial_input_cfg_t *cfg)
+{
+  return cfg && cfg->receive_ref != LUA_NOREF;
+}
+
+
+// on("method", [number/char], function)
+int serial_input_register(lua_State *L, serial_input_cfg_t *cfg)
+{
+  const char *method = luaL_checkstring(L, 1);
+  const bool is_data = (strcmp(method, "data") == 0);
+  const bool is_error = (strcmp(method, "error") == 0);
+  if (!is_data && !is_error)
+    return luaL_error(L, "method not supported");
+
+  int fn_idx = -1;
+
+  if (lua_isnumber(L, 2))
+  {
+    cfg->need_len = luaL_checkinteger(L, 2);
+    cfg->end_char = -1;
+  }
+  else if (lua_isstring(L, 2))
+  {
+    size_t len;
+    const char *end = luaL_checklstring(L, 2, &len);
+    if (len != 1)
+      return luaL_error(L, "only single byte end marker supported");
+    cfg->need_len = 0;
+    cfg->end_char = end[0];
+  }
+  else if (lua_isfunction(L, 2))
+  {
+    fn_idx = 2;
+  }
+
+  if (fn_idx == -1 && lua_isfunction(L, 3))
+  {
+    fn_idx = 3;
+  }
+
+
+  if (is_data)
+  {
+    if (cfg->receive_ref != LUA_NOREF)
+      luaL_unref2(L, LUA_REGISTRYINDEX, cfg->receive_ref); // unref & clear
+
+    if (fn_idx != -1) // Register and (re)alloc resources
+    {
+      luaL_checkstack(L, 1, nostack);
+      lua_pushvalue(L, fn_idx);
+      cfg->receive_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+      size_t min_size = (cfg->need_len > 0) ? cfg->need_len : MAX_SERIAL_INPUT;
+      // Prevent dropping input; this should be an exceedingly rare condition
+      if (cfg->line_position >= min_size)
+        min_size = cfg->line_position + 1;
+
+      if (cfg->line_buffer_size < min_size)
+      {
+        cfg->line_buffer = realloc(cfg->line_buffer, min_size);
+        cfg->line_buffer_size = (cfg->line_buffer) ? min_size : 0;
+        if (!cfg->line_buffer)
+          return luaL_error(L, "out of mem");
+      }
+    }
+    else // Free resources
+    {
+      free(cfg->line_buffer);
+      cfg->line_buffer = NULL;
+      cfg->line_buffer_size = 0;
+      cfg->line_position = 0;
+    }
+  }
+  else if (is_error)
+  {
+    if (cfg->error_ref != LUA_NOREF)
+      luaL_unref2(L, LUA_REGISTRYINDEX, cfg->error_ref); // unref & clear
+
+    if (fn_idx != -1)
+    {
+      luaL_checkstack(L, 1, nostack);
+      lua_pushvalue(L, fn_idx);
+      cfg->error_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+    }
+  }
+
+  return 0;
+}

components/modules/serial_common.h

@@ -0,0 +1,91 @@
+#ifndef SERIAL_COMMON_H
+#define SERIAL_COMMON_H
+
+#include "lua.h"
+
+#include <stdint.h>
+#include <stdbool.h>
+
+struct serial_input_cfg;
+typedef struct serial_input_cfg serial_input_cfg_t;
+
+/**
+ * Instantiate a new serial_input object.
+ * @returns a freshly allocated serial input object, with no further resources
+ *   associated with it.
+ */
+serial_input_cfg_t *serial_input_new(void);
+
+/**
+ * Free a serial_input_cfg_t object.
+ * Releases all associated resources. The object may not be passed to any
+ * serial_input_xxx functions after this.
+ *
+ * Must only be called from the Lua VM task context.
+ */
+void serial_input_free(lua_State *L, serial_input_cfg_t *cfg);
+
+
+/**
+ * Helper function to hand registration of "data" and "error" callbacks.
+ * Expects the following calling signature:
+ *   on("method", [number/char], function)
+ *
+ * Must only be called from the Lua VM task context.
+ *
+ * @param L The current Lua VM.
+ * @param cfg Instance to un/register with. Must've been initialised with
+ *   @c serial_input_init() originally.
+ * @return Zero. Will luaL_error() on invalid args.
+ */
+int serial_input_register(lua_State *L, serial_input_cfg_t *cfg);
+
+/**
+ * Feed data into a serial_input stream for processing.
+ *
+ * Must only be called from the Lua VM task context, as it will invoke
+ * Lua callbacks as necessary.
+ * Uses lua_getstate() to obtain the LVM instance.
+ *
+ * @param cfg The serial_input instance.
+ * @param buf The data buffer from which to feed bytes.
+ * @param len The number of bytes available in the buffer.
+ */
+void serial_input_feed_data(serial_input_cfg_t *cfg, const char *buf, size_t len);
+
+/**
+ * Checks whether a "data" callback is registered.
+ *
+ * @param cfg The serial_input instance.
+ * @return Whether a "data" callback is currently registered.
+ */
+bool serial_input_has_data_cb(serial_input_cfg_t *cfg);
+
+/**
+ * Direct access to invoking a configured "data" callback.
+ *
+ * Must only be called from the Lua VM task context.
+ * Uses lua_getstate() to obtain the LVM instance.
+ *
+ * @param cfg The serial_input instance.
+ * @param buf The data which to pass to the callback.
+ * @param len The number of bytes available in the buffer.
+ * @return True if the callback was successfully invoked (registered, and valid
+ *   non-empty data passed).
+ */
+bool serial_input_dispatch_data(serial_input_cfg_t *cfg, const char *buf, size_t len);
+
+/**
+ * Direct access to invoking a configured "error" callback.
+ * Must only be called from the Lua VM task context.
+ * Uses lua_getstate() to obtain the LVM instance.
+ *
+ * @param cfg The serial_input instance.
+ * @param msg The message to pass to the error callback.
+ * @param len The length of the message, in bytes.
+ * @return True if the callback was successfully invoked (registered, and valid
+ *   non-empty data passed).
+ */
+bool serial_input_report_error(serial_input_cfg_t *cfg, const char *msg, size_t len);
+
+#endif

components/modules/uart.c

@@ -3,61 +3,23 @@
 #include "module.h"
 #include "lauxlib.h"
 #include "platform.h"
+#include "serial_common.h"
 #include "linput.h"
 #include "lmem.h"
 
 #include <stdint.h>
 #include <string.h>
 
-typedef struct {
+static serial_input_cfg_t *uart_cb_cfg[NUM_UART];
-  int receive_rf;
-  int error_rf;
-  char *line_buffer;
-  size_t line_position;
-  uint16_t need_len;
-  int16_t end_char;
-} uart_cb_cfg_t;
-
-static lua_State *gL = NULL;
-static uart_cb_cfg_t uart_cb_cfg[NUM_UART];
-
-
-static bool uart_on_data_cb(unsigned id, const char *buf, size_t len){
-  if(!buf || len==0)
-    return false;
-  if(uart_cb_cfg[id].receive_rf == LUA_NOREF)
-    return false;
-  if(!gL)
-    return false;
-
-  int top = lua_gettop(gL);
-  lua_rawgeti(gL, LUA_REGISTRYINDEX, uart_cb_cfg[id].receive_rf);
-  lua_pushlstring(gL, buf, len);
-  luaL_pcallx(gL, 1, 0);
-  lua_settop(gL, top);
-  return !run_input;
-}
 
 
 bool uart_on_error_cb(unsigned id, const char *buf, size_t len){
-  if(!buf || len==0)
+  return serial_input_report_error(uart_cb_cfg[id], buf, len);
-    return false;
-  if(uart_cb_cfg[id].error_rf == LUA_NOREF)
-    return false;
-  if(!gL)
-    return false;
-
-  int top = lua_gettop(gL);
-  lua_rawgeti(gL, LUA_REGISTRYINDEX, uart_cb_cfg[id].error_rf);
-  lua_pushlstring(gL, buf, len);
-  luaL_pcallx(gL, 1, 0);
-  lua_settop(gL, top);
-  return true;
 }
 
 
 bool uart_has_on_data_cb(unsigned id){
-  return uart_cb_cfg[id].receive_rf != LUA_NOREF;
+  return serial_input_has_data_cb(uart_cb_cfg[id]);
 }
 
 
@@ -66,116 +28,43 @@ void uart_feed_data(unsigned id, const char *buf, size_t len)
   if (id >= NUM_UART)
     return;
 
-  uart_cb_cfg_t *cfg = &uart_cb_cfg[id];
+  serial_input_feed_data(uart_cb_cfg[id], buf, len);
-  if (!cfg->line_buffer)
+}
-    return;
 
-  for (unsigned i = 0; i < len; ++i)
+
+static int ensure_valid_id(lua_State *L, int id)
 {
-    char ch = buf[i];
+  MOD_CHECK_ID(uart, id);
-    cfg->line_buffer[cfg->line_position] = ch;
-    cfg->line_position++;
 
-    uint16_t need_len = cfg->need_len;
+  int console = -1;
-    int16_t end_char = cfg->end_char;
+#if defined(CONFIG_ESP_CONSOLE_UART_DEFAULT) || defined(CONFIG_ESP_CONSOLE_UART_CUSTOM)
-    size_t max_wanted =
+  console = CONFIG_ESP_CONSOLE_UART_NUM;
-      (end_char >= 0 && need_len == 0) ? LUA_MAXINPUT : need_len;
+#endif
-    bool at_end = (cfg->line_position >= max_wanted);
+
-    bool end_char_found =
+  if (id == console)
-      (end_char >= 0 && (uint8_t)ch == (uint8_t)end_char);
+    return luaL_error(L,
-    if (at_end || end_char_found) {
+      "uart in use by system console; use the 'console' module instead");
-      uart_on_data_cb(id, cfg->line_buffer, cfg->line_position);
+
-      cfg->line_position = 0;
+  return 0;
-    }
-  }
 }
 
 
 // Lua: uart.on([id], "method", [number/char], function, [run_input])
 static int uart_on( lua_State* L )
 {
-  unsigned id = CONFIG_ESP_CONSOLE_UART_NUM;
+  int id = 0;
-  size_t sl, el;
+  if (lua_isnumber(L, 1))
-  int32_t run = 1;
-  uint8_t stack = 1;
-  const char *method;
-  
-  if( lua_isnumber( L, stack ) ) {
-    id = ( unsigned )luaL_checkinteger( L, stack );
-    MOD_CHECK_ID( uart, id );
-    stack++;
-  }
-
-  uart_cb_cfg_t *cfg = &uart_cb_cfg[id];
-
-  method = luaL_checklstring( L, stack, &sl );
-  stack++;
-  if (method == NULL)
-    return luaL_error( L, "wrong arg type" );
-
-  if( lua_type( L, stack ) == LUA_TNUMBER )
   {
-    cfg->need_len = (uint16_t)luaL_checkinteger(L, stack);
+    id = luaL_checkinteger(L, 1);
-    stack++;
+    lua_remove(L, 1);
-    cfg->end_char = -1;
-    if(cfg->need_len > 255)
-    {
-      cfg->need_len = 255;
-      return luaL_error( L, "wrong arg range" );
-    }
-  }
-  else if(lua_isstring(L, stack))
-  {
-    const char *end = luaL_checklstring( L, stack, &el );
-    stack++;
-    if(el!=1){
-      return luaL_error( L, "wrong arg range" );
-    }
-    cfg->end_char = (int16_t)end[0];
-    cfg->need_len = 0;
   }
 
-  if (lua_isfunction(L, stack)) {
+  ensure_valid_id(L, id);
-    if ( lua_isnumber(L, stack+1) ){
+
-      run = lua_tointeger(L, stack+1);
+  return serial_input_register(L, uart_cb_cfg[id]);
-    }
-    lua_pushvalue(L, stack);  // copy argument (func) to the top of stack
-  } else {
-    lua_pushnil(L);
-  }
-  if(sl == 4 && strcmp(method, "data") == 0){
-    if(id == CONFIG_ESP_CONSOLE_UART_NUM)
-      run_input = true;
-    if(cfg->receive_rf != LUA_NOREF){
-      luaL_unref(L, LUA_REGISTRYINDEX, cfg->receive_rf);
-      cfg->receive_rf = LUA_NOREF;
-    }
-    if(!lua_isnil(L, -1)){
-      cfg->receive_rf = luaL_ref(L, LUA_REGISTRYINDEX);
-      gL = L;
-      if(id == CONFIG_ESP_CONSOLE_UART_NUM && run==0)
-        run_input = false;
-    } else {
-      lua_pop(L, 1);
-    }
-  } else if(sl == 5 && strcmp(method, "error") == 0){
-    if(cfg->error_rf != LUA_NOREF){
-      luaL_unref(L, LUA_REGISTRYINDEX, cfg->error_rf);
-      cfg->error_rf = LUA_NOREF;
-    }
-    if(!lua_isnil(L, -1)){
-      cfg->error_rf = luaL_ref(L, LUA_REGISTRYINDEX);
-      gL = L;
-    } else {
-      lua_pop(L, 1);
-    }
-  } else {
-    lua_pop(L, 1);
-    return luaL_error( L, "method not supported" );
-  }
-  return 0; 
 }
 
+
 // Lua: actualbaud = setup( id, baud, databits, parity, stopbits, echo )
 static int uart_setup( lua_State* L )
 {
@@ -186,15 +75,12 @@ static int uart_setup( lua_State* L )
   
   memset(&pins, 0, sizeof(pins));
   id = luaL_checkinteger( L, 1 );
-  MOD_CHECK_ID( uart, id );
+  ensure_valid_id(L, id);
   baud = luaL_checkinteger( L, 2 );
   databits = luaL_checkinteger( L, 3 );
   parity = luaL_checkinteger( L, 4 );
   stopbits = luaL_checkinteger( L, 5 );
   if (!lua_isnoneornil(L, 6)) {
-    if(id == CONFIG_ESP_CONSOLE_UART_NUM){
-      input_echo = luaL_checkinteger(L, 6) > 0;
-    } else {
     luaL_checktable(L, 6);
 
     lua_getfield (L, 6, "tx");
@@ -220,7 +106,6 @@ static int uart_setup( lua_State* L )
 
     pins_to_use = &pins;
   }
-  }
 
   res = platform_uart_setup( id, baud, databits, parity, stopbits, pins_to_use );
   lua_pushinteger( L, res );
@@ -232,7 +117,7 @@ static int uart_setmode(lua_State* L)
   unsigned id, mode;
 
   id = luaL_checkinteger( L, 1 );
-  MOD_CHECK_ID( uart, id );
+  ensure_valid_id(L, id);
   mode = luaL_checkinteger( L, 2 );
 
   platform_uart_setmode(id, mode);
@@ -249,7 +134,7 @@ static int uart_write( lua_State* L )
   int total = lua_gettop( L ), s;
   
   id = luaL_checkinteger( L, 1 );
-  MOD_CHECK_ID( uart, id );
+  ensure_valid_id(L, id);
   for( s = 2; s <= total; s ++ )
   {
     if( lua_type( L, s ) == LUA_TNUMBER )
@@ -275,13 +160,8 @@ static int uart_stop( lua_State* L )
 {
   unsigned id;
   id = luaL_checkinteger( L, 1 );
-  MOD_CHECK_ID( uart, id );
+  ensure_valid_id(L, id);
   platform_uart_stop( id );
-  if (uart_cb_cfg[id].line_buffer)
-  {
-    luaM_freemem(L, uart_cb_cfg[id].line_buffer, LUA_MAXINPUT);
-    uart_cb_cfg[id].line_buffer = NULL;
-  }
   return 0;
 }
 
@@ -291,9 +171,7 @@ static int uart_start( lua_State* L )
   unsigned id;
   int err;
   id = luaL_checkinteger( L, 1 );
-  MOD_CHECK_ID( uart, id );
+  ensure_valid_id(L, id);
-  if (!uart_cb_cfg[id].line_buffer)
-    uart_cb_cfg[id].line_buffer = luaM_malloc(L, LUA_MAXINPUT);
   err = platform_uart_start( id );
   lua_pushboolean( L, err == 0 );
   return 1;
@@ -303,7 +181,7 @@ static int uart_getconfig(lua_State* L) {
     uint32_t id, baud, databits, parity, stopbits;
 
     id = luaL_checkinteger(L, 1);
-    MOD_CHECK_ID(uart, id);
+    ensure_valid_id(L, id);
 
     int err = platform_uart_get_config(id, &baud, &databits, &parity, &stopbits);
     if (err) {
@@ -320,7 +198,7 @@ static int uart_getconfig(lua_State* L) {
 static int uart_wakeup (lua_State *L)
 {
   uint32_t id = luaL_checkinteger(L, 1);
-  MOD_CHECK_ID(uart, id);
+  ensure_valid_id(L, id);
   int threshold = luaL_checkinteger(L, 2);
   int err = platform_uart_set_wakeup_threshold(id, threshold);
   if (err) {
@@ -332,7 +210,7 @@ static int uart_wakeup (lua_State *L)
 static int luart_tx_flush (lua_State *L)
 {
   uint32_t id = luaL_checkinteger(L, 1);
-  MOD_CHECK_ID(uart, id);
+  ensure_valid_id(L, id);
   platform_uart_flush(id);
   return 0;
 }
@@ -367,13 +245,9 @@ LROT_END(uart, NULL, 0)
 int luaopen_uart( lua_State *L ) {
   for(int id = 0; id < sizeof(uart_cb_cfg)/sizeof(uart_cb_cfg[0]); id++)
   {
-    uart_cb_cfg_t *cfg = &uart_cb_cfg[id];
+    uart_cb_cfg[id] = serial_input_new();
-    cfg->receive_rf = LUA_NOREF;
+    if (!uart_cb_cfg[id])
-    cfg->error_rf = LUA_NOREF;
+      return luaL_error(L, "out of mem");
-    cfg->line_buffer = NULL;
-    cfg->line_position = 0;
-    cfg->need_len = 0;
-    cfg->end_char = -1;
   }
   return 0;
 }

components/platform/platform.c

@@ -88,14 +88,6 @@ void uart_event_task( task_param_t param, task_prio_t prio ) {
   unsigned id = post->id;
   xSemaphoreGive(sem);
   if(post->type == PLATFORM_UART_EVENT_DATA) {
-    if (id == CONFIG_ESP_CONSOLE_UART_NUM && run_input) {
-      size_t i = 0;
-      while (i < post->size)
-      {
-        unsigned used = feed_lua_input(post->data + i, post->size - i);
-        i += used;
-      }
-    }
     if (uart_has_on_data_cb(id))
       uart_feed_data(id, post->data, post->size);
 
@@ -206,6 +198,10 @@ static void task_uart( void *pvParameters ){
 // pins must not be null for non-console uart
 uint32_t platform_uart_setup( unsigned id, uint32_t baud, int databits, int parity, int stopbits, uart_pins_t* pins )
 {
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
+  if (id == CONFIG_ESP_CONSOLE_UART_NUM)
+    return 0;
+#endif
   int flow_control = UART_HW_FLOWCTRL_DISABLE;
   if (pins != NULL) {
 	if(pins->flow_control & PLATFORM_UART_FLOW_CTS) flow_control |= UART_HW_FLOWCTRL_CTS;
@@ -286,29 +282,28 @@ void platform_uart_setmode(unsigned id, unsigned mode)
 
 void platform_uart_send_multi( unsigned id, const char *data, size_t len )
 {
-  size_t i;
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
-  if (id == CONFIG_ESP_CONSOLE_UART_NUM) {
+  if (id == CONFIG_ESP_CONSOLE_UART_NUM)
-      for( i = 0; i < len; i ++ ) {
+    return;
-        putchar (data[ i ]);
+#endif
-    }
-  } else {
   uart_write_bytes(id, data, len);
 }
-}
 
 void platform_uart_send( unsigned id, uint8_t data )
 {
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
   if (id == CONFIG_ESP_CONSOLE_UART_NUM)
-    putchar (data);
+    return;
-  else
+#endif
   uart_write_bytes(id, (const char *)&data, 1);
 }
 
 void platform_uart_flush( unsigned id )
 {
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
   if (id == CONFIG_ESP_CONSOLE_UART_NUM)
-    fflush (stdout);
+    return;
-  else
+#endif
   uart_tx_flush(id);
 }
 
@@ -354,9 +349,12 @@ void platform_uart_stop( unsigned id )
 }
 
 int platform_uart_get_config(unsigned id, uint32_t *baudp, uint32_t *databitsp, uint32_t *parityp, uint32_t *stopbitsp) {
-    int err;
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
+  if (id == CONFIG_ESP_CONSOLE_UART_NUM)
+    return -1;
+#endif
 
-    err = uart_get_baudrate(id, baudp);
+    int err = uart_get_baudrate(id, baudp);
     if (err != ESP_OK) return -1;
     *baudp &= 0xFFFFFFFE; // round down
 
@@ -405,6 +403,10 @@ int platform_uart_get_config(unsigned id, uint32_t *baudp, uint32_t *databitsp,
 
 int platform_uart_set_wakeup_threshold(unsigned id, unsigned threshold)
 {
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
+  if (id == CONFIG_ESP_CONSOLE_UART_NUM)
+    return -1;
+#endif
   esp_err_t err = uart_set_wakeup_threshold(id, threshold);
   return (err == ESP_OK) ? 0 : -1;
 }

docs/modules/console.md

@@ -0,0 +1,232 @@
+# Console Module
+| Since  | Origin / Contributor  | Maintainer  | Source  |
+| :----- | :-------------------- | :---------- | :------ |
+| 2024-10-22 | [jmattsson](https://github.com/jmattsson) | [jmattsson](https://github.com/jmattsson) | [console.c](../../components/modules/console.c)|
+
+The `console` modules allows direct access to the system console. The system
+console has typically been a UART, but by now several options are available
+across the ESP32 range of SoCs, including UART, USB-Serial-JTAG and USB CDC-ACM.
+
+By default the system console is linked up to provide an interactive Lua
+shell (REPL — Read-Execute-Print Loop). It also provides a hook for listening
+in on the data received on the console programatically, and the interactivity
+may also be disabled (and re-enabled) programatically if so desired.
+
+There is a helper script (`scripts/upload-file.py`) which can be used to
+easily upload files to NodeMCU via this module. The script may also be used
+as inspiration for integrating such functionality into IDEs.
+
+If using a SoC with USB CDC-ACM as the console, consider increasing the receive
+buffer from the default. This type of console is quite prone to overflows, and
+increasing the receive buffer helps mitigate (but not completely resolve) that.
+Look for `Component config -> ESP system settings ->  Size of USB CDC RX buffer`
+in the menuconfig (`ESP_CONSOLE_USB_CDC_RX_BUF_SIZE` in sdkconfig). Increasing
+this value from the default 64 to 512 makes it match what is typically used
+for a UART console. Some utilities and IDEs may have their own minimum
+requirements for the receive buffer.
+
+## console.on()
+
+Used to register or deregister a callback function to handle console events.
+
+#### Syntax
+`console.on(method, [number/end_char], [function])`
+
+#### Parameters
+- `method`. One of
+    - "data" for bytes received on the console
+    - "error" if an error condition is encountered on the console
+- `number/end_char`. Only for event `data`.
+    - if pass in a number n, the callback will called when n chars are received.
+    - if n=0, will receive every char in buffer.
+    - if pass in a one char string "c", the callback will called when "c" is encounterd, or max n=255 received.
+- `function` callback function. 
+    - event "data" has a callback like this: `function(data) end`
+    - event "error" has a callback like this: `function(err) end`
+
+To unregister the callback, specify `nil` as the function.
+
+#### Returns
+`nil`
+
+#### Example
+```lua
+-- when 4 chars is received.
+console.on("data", 4, function(data)
+  print("received from console:", data)
+  if data=="quit" then
+    console.on("data", 0, nil)
+  end
+end)
+-- when '\r' is received.
+console.on("data", "\r", function(data)
+  print("received from console:", data)
+  if data=="quit\r" then
+    console.on("data", 0, nil)
+  end
+end)
+
+-- error handler
+console.on("error", function(err)
+  print("error on console:", err)
+end)
+```
+
+## console.mode()
+
+Controls the interactivity of the console.
+
+#### Syntax
+`console.mode(mode)`
+
+#### Parameters
+- `mode` One of
+    - `console.INTERACTIVE` automatically pass console data to the Lua VM
+      for execution. This is the default mode.
+    - `console.NONINTERACTIVE` disables the automatic passing of console data
+      to the Lua VM. The data only goes to the registered "data" callback,
+      if any.
+
+#### Returns
+`nil`
+
+#### Example
+Implement a REL instead of the usual REPL
+```lua
+console.on("data", "\r", function(line) node.input(line.."\r\n") end)
+console.mode(console.NONINTERACTIVE)
+```
+
+Receive potentially binary data on the console, and process it without letting
+it reach the Lua interpreter.
+```lua
+-- Instantiate a stream handling function that processes a classic framing
+-- protocol: <STX><escaped-data><ETX> where <escaped-data> has the
+-- STX, ETX and DLE symbols escaped as <DLE><STX>, <DLE><ETX>, <DLE><DLE>.
+-- The chunk_cb gets called incrementally with partial stream data which is
+-- effectively unescaped. When the end of frame is encountered, the done_cb
+-- gets invoked.
+-- To avoid overruns on slower consoles (e.g. CDC-ACM) each block gets
+-- acknowledged by printing another prompt. This allows the sender to easily
+-- throttle the upload to a maintainable pace.
+function transmission_receiver(chunk_cb, done_cb, blocksize)
+  local inframe = false
+  local escaped = false
+  local done = false
+  local len = 0
+  local STX = 2
+  local ETX = 3
+  local DLE = 16
+  local function dispatch(data, i, j)
+    if (j - i) < 0 then return end
+    chunk_cb(data:sub(i, j))
+  end
+  return function(data)
+    if done then return end
+    len = len + #data
+    while len >= blocksize do
+      len = len - blocksize
+      console.write("> ")
+    end
+    local from
+    local to
+    for i = 1, #data
+    do
+      local b = data:byte(i)
+      if inframe
+      then
+        if not from then from = i end -- first valid byte
+        if escaped
+        then
+          escaped = false
+        else
+          if b == DLE
+          then
+            escaped = true
+            dispatch(data, from, i-1)
+            from = nil
+          elseif b == ETX
+          then
+            done = true
+            to = i-1
+            break
+          end
+        end
+      else -- look for an (unescaped) STX to sync frame start
+        if b == DLE then escaped = true
+        elseif b == STX and not escaped then inframe = true
+        else escaped = false
+        end
+      end
+      -- else ignore byte outside of framing
+    end
+    if from then dispatch(data, from, to or #data) end
+    if done then done_cb() end
+  end
+end
+
+function print_hex(chunk)
+  for i = 1, #chunk
+  do
+    print(string.format("%02x ", chunk:sub(i,i)))
+  end
+end
+
+function resume_interactive()
+  console.on("data", 0, nil)
+  console.mode(console.INTERACTIVE)
+end
+
+-- The 0 may be adjusted upwards for improved efficiency, but be mindful to
+-- always send enough data to reach the ETX marker if so.
+console.on("data", 0, transmission_receiver(print_hex, resume_interactive, 64))
+console.mode(console.NONINTERACTIVE)
+```
+
+Example C program for encoding data suitable for the above.
+```C
+#include <stdio.h>
+
+#define STX 0x02
+#define ETX 0x03
+#define DLE 0x10
+
+int main(int argc, char *argv[])
+{
+  putchar(STX);
+  int ch;
+  while ((ch = getchar()) != EOF)
+  {
+    switch(ch)
+    {
+      case STX: case ETX: case DLE: putchar(DLE); break;
+      default: break;
+    }
+    putchar(ch);
+  }
+  putchar(ETX);
+  return 0;
+}
+```
+
+## console.write()
+
+Provides ability to write raw, unformatted data to the console.
+
+#### Syntax
+`console.write(str_or_num [, str2_or_num ...])`
+
+#### Parameters
+- `str_or_num` Either
+    - A string to write to the console. May contain binary data.
+    - A number representing the character code to write the console.
+  Multiple parameters may be given, and they will be written in sequence.
+
+#### Returns
+`nil`
+
+#### Example
+Write "Hello world!\n" to the console, in a roundabout manner
+```lua
+console.write("Hello", 0x20, "world", 0x21, "\n")
+```

docs/modules/uart.md

@@ -5,31 +5,35 @@
 
 The [UART](https://en.wikipedia.org/wiki/Universal_asynchronous_receiver/transmitter) (Universal asynchronous receiver/transmitter) module allows configuration of and communication over the UART serial port.
 
-The default setup for the console uart is controlled by build-time settings. The default uart for console is `UART0`. The default rate is 115,200 bps. In addition, auto-baudrate detection is enabled for the first two minutes
+If the UART is in use as the system console, it is unavailable for use by this
-after platform boot. This will cause a switch to the correct baud rate once a few characters are received. Auto-baudrate detection is disabled when `uart.setup` is called.
+module. Instead, refer to the `console` module.
 
-For other uarts, you should call `uart.setup` and `uart.start` to get them working.
+If your IDE does not yet support uploading files via the `console` module,
+consider using the utility script `scripts/upload-file.py`, e.g.
+`scripts/upload-file.py init.lua` (use `scripts/upload-file.py -h` for help).
+
+Before using a UART, you must call `uart.setup` and `uart.start` to set it up.
 
 ## uart.on()
 
-Sets the callback function to handle UART events.
+Sets the callback function to handle UART events. For a UART used by the
+console, refer to the `console` module instead.
 
 #### Syntax
-`uart.on([id], method, [number/end_char], [function], [run_input])`
+`uart.on([id], method, [number/end_char], [function])`
 
 #### Parameters
-- `id` uart id, default value is uart num of the console.
+- `id` uart id, except console uart. Default value is uart 0.
 - `method` "data", data has been received on the UART. "error", error occurred on the UART.
 - `number/end_char`. Only for event `data`.
-	- if pass in a number n<255, the callback will called when n chars are received.
+	- if pass in a number n, the callback will called when n chars are received.
 	- if n=0, will receive every char in buffer.
 	- if pass in a one char string "c", the callback will called when "c" is encounterd, or max n=255 received.
 - `function` callback function. 
   - event "data" has a callback like this: `function(data) end`
   - event "error" has a callback like this: `function(err) end`. `err` could be one of "out_of_memory", "break", "rx_error".
-- `run_input` 0 or 1. Only for "data" event on console uart. If 0, input from UART will not go into Lua interpreter, can accept binary data. If 1, input from UART will go into Lua interpreter, and run.
 
-To unregister the callback, provide only the "data" parameter.
+To unregister the callback, provide only the "method" parameter.
 
 #### Returns
 `nil`
@@ -43,7 +47,7 @@ uart.on("data", 4,
 	if data=="quit" then
 	  uart.on("data") -- unregister callback function
 	end
-end, 0)
+end)
 -- when '\r' is received.
 uart.on("data", "\r",
   function(data)
@@ -51,7 +55,7 @@ uart.on("data", "\r",
 	if data=="quit\r" then
 	  uart.on("data") -- unregister callback function
 	end
-end, 0)
+end)
 
 -- uart 2
 uart.on(2, "data", "\r",
@@ -71,17 +75,16 @@ uart.on(2, "error",
 (Re-)configures the communication parameters of the UART.
 
 #### Syntax
-`uart.setup(id, baud, databits, parity, stopbits, echo_or_pins)`
+`uart.setup(id, baud, databits, parity, stopbits, pins)`
 
 #### Parameters
-- `id` uart id
+- `id` uart id, except console uart
 - `baud` one of 300, 600, 1200, 2400, 4800, 9600, 19200, 38400, 57600, 74880, 115200, 230400, 256000, 460800, 921600, 1843200, 3686400
 - `databits` one of 5, 6, 7, 8
 - `parity` `uart.PARITY_NONE`, `uart.PARITY_ODD`, or `uart.PARITY_EVEN`
 - `stopbits` `uart.STOPBITS_1`, `uart.STOPBITS_1_5`, or `uart.STOPBITS_2`
-- `echo_or_pins`
+- `pins`
-  - for console uart, this should be a int. if 0, disable echo, otherwise enable echo
+  - table with the following entries:
-  - for others, this is a table:
     - `tx` int. TX pin. Required
 	- `rx` int. RX pin. Required
 	- `cts` in. CTS pin. Optional
@@ -113,7 +116,7 @@ Returns the current configuration parameters of the UART.
 `uart.getconfig(id)`
 
 #### Parameters
-- `id` UART id (0 or 1).
+- `id` uart id, except console uart
 
 #### Returns
 Four values as follows:
@@ -130,7 +133,7 @@ print (uart.getconfig(0))
 ```
 
 ## uart.start()
-Start the UART. You do not need to call `start()` on the console uart.
+Start the UART.
 
 #### Syntax
 `uart.start(id)`
@@ -143,7 +146,7 @@ Boolean. `true` if uart is started.
 
 
 ## uart.stop()
-Stop the UART. You should not call `stop()` on the console uart.
+Stop the UART.
 
 #### Syntax
 `uart.stop(id)`
@@ -164,7 +167,7 @@ Set UART controllers communication mode
 `uart.setmode(id, mode)`
 
 #### Parameters
-- `id` uart id
+- `id` uart id, except console uart
 - `mode` value should be one of
     - `uart.MODE_UART` default UART mode, is set after uart.setup() call
     - `uart.MODE_RS485_COLLISION_DETECT` receiver must be always enabled, transmitter is automatically switched using RTS pin, collision is detected by UART hardware (note: no event is generated on collision, limitation of esp-idf)
@@ -184,15 +187,15 @@ Wait for any data currently in the UART transmit buffers to be written out. It c
 `uart.txflush(id)`
 
 #### Parameters
-- `id` uart id
+- `id` uart id, except console uart
 
 #### Returns
 `nil`
 
 #### Example
 ```lua
-print("I want this to show up now not in 5 seconds")
+uart.write(0, "I want this to show up now not in 5 seconds")
-uart.txflush(0) -- assuming 0 is the console uart
+uart.txflush(0)
 node.sleep({secs=5})
 ```
 
@@ -208,7 +211,7 @@ Configure the light sleep wakeup threshold. This is the number of positive edges
 `uart.wakeup(id, val)`
 
 #### Parameters
-- `id` uart id
+- `id` uart id, except console uart
 - `val` the new value 
 
 #### Returns
@@ -231,7 +234,7 @@ Write string or byte to the UART.
 `uart.write(id, data1 [, data2, ...])`
 
 #### Parameters
-- `id` uart id
+- `id` uart id, except console uart
 - `data1`... string or byte to send via UART
 
 #### Returns

mkdocs.yml

@@ -43,6 +43,7 @@ nav:
       - 'bit': 'modules/bit.md'
       - 'bthci': 'modules/bthci.md'
       - 'can': 'modules/can.md'
+      - 'console': 'modules/console.md'
       - 'crypto': 'modules/crypto.md'
       - 'dac': 'modules/dac.md'
       - 'dht': 'modules/dht.md'

scripts/upload-file.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+
+# A helper utility to allow uploading of files to NodeMCU versions which use
+# the 'console' module, rather than having the console multiplexed via the
+# 'uart' module.
+
+import argparse
+import serial
+import sys
+import atexit
+
+STX = 0x02
+ETX = 0x03
+DLE = 0x10
+
+# The loader we send to NodeMCU so that we may upload a (binary) file safely.
+# Uses STX/ETX/DLE framing and escaping.
+# The CDC-ACM console gets overwhelmed unless we throttle the send by using
+# an ack scheme. We use a fake prompt for simplicity's sake for that.
+loader = b'''
+(function()
+  local function transmission_receiver(chunk_cb)
+    local inframe = false
+    local escaped = false
+    local done = false
+    local len = 0
+    local STX = 2
+    local ETX = 3
+    local DLE = 16
+    local function dispatch(data, i, j)
+      if (j - i) < 0 then return end
+      chunk_cb(data:sub(i, j))
+    end
+    return function(data)
+      if done then return end
+      len = len + #data
+      while len >= @BLOCKSIZE@ do
+        len = len - @BLOCKSIZE@
+        console.write("> ")
+      end
+      local from
+      local to
+      for i = 1, #data
+      do
+        local b = data:byte(i)
+        if inframe
+        then
+          if not from then from = i end -- first valid byte
+          if escaped then escaped = false else
+            if b == DLE
+            then
+              escaped = true
+              dispatch(data, from, i-1)
+              from = nil
+            elseif b == ETX
+            then
+              done = true
+              to = i-1
+              break
+            end
+          end
+        else -- look for an (unescaped) STX to sync frame start
+          if b == DLE then escaped = true
+          elseif b == STX and not escaped then inframe = true
+          else escaped = false end
+        end
+        -- else ignore byte outside of framing
+      end
+      if from then dispatch(data, from, to or #data) end
+      if done then chunk_cb(nil) end
+    end
+  end
+
+  local function file_saver(name)
+    local f = io.open(name, "w")
+    return function(chunk)
+      if chunk then f:write(chunk) else
+        f:close()
+        console.on("data", 0, nil)
+        console.mode(console.INTERACTIVE)
+        console.write("done")
+      end
+    end
+  end
+
+  console.on("data", 0, transmission_receiver(file_saver(
+    "@FILENAME@")))
+  console.mode(console.NONINTERACTIVE)
+  console.write("ready")
+end)()
+'''
+
+def parse_args():
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(description="NodeMCU file uploader.")
+    parser.add_argument("file", help="File to read data from.")
+    parser.add_argument("name", nargs="?", help="Name to upload file as.")
+    parser.add_argument("-p", "--port", default="/dev/ttyUSB0", help="Serial port (default: /dev/ttyUSB0).")
+    parser.add_argument("-b", "--bitrate", type=int, default=115200, help="Bitrate (default: 115200).")
+    parser.add_argument("-s", "--blocksize", type=int, default=80, help="Block size of file data, tweak for speed/reliability of upload (default: 80)")
+    return parser.parse_args()
+
+def load_file(filename):
+    """Open a file and read its contents into memory."""
+    try:
+        with open(filename, "rb") as f:
+            data = f.read()
+        return data
+    except IOError as e:
+        print(f"Error reading file {filename}: {e}")
+        sys.exit(1)
+
+def xprint(msg):
+    print(msg, end='', flush=True)
+
+def wait_prompt(ser, ignore):
+    """Wait until we see the '> ' prompt, or the serial times out"""
+    buf = bytearray()
+    b = ser.read()
+    timeout = 5
+    while timeout > 0:
+        if b == b'':
+            timeout -= 1
+            xprint('!')
+        else:
+            buf.extend(b)
+        if not ignore and buf.find(b'Lua error:') != -1:
+            xprint(buf.decode())
+            line = ser.readline()
+            while line != b'':
+                xprint(line.decode())
+                line = ser.readline()
+            sys.exit(1)
+        if buf.find(b'> ') != -1:
+            return True
+        b = ser.read()
+    xprint(buf.decode())
+    return False
+
+def wait_line_match(ser, match, timeout):
+    """Wait until the 'match' string is found within a line, or times out"""
+    line = ser.readline()
+    while timeout > 0:
+        if line.find(match) != -1:
+            return True
+        elif line == b'':
+            timeout -= 1
+            xprint('!')
+    return False
+
+def sync(ser):
+    """Get ourselves to a clean prompt so we can understand the output"""
+    ser.write(b'\x03\x03\n')
+    if not wait_prompt(ser, True):
+        return False
+    ser.write(b"print('sync')\n")
+    return wait_line_match(ser, b'sync', 5) and wait_prompt(ser, True)
+
+def cleanup():
+    """Cleanup function to send final data and close the serial port."""
+    if ser:
+        # Ensure we don't leave the console in a weird state if we get
+        # interrupted.
+        ser.write(ETX)
+        ser.write(ETX)
+        ser.write(b"\n")
+        ser.readline()
+        ser.close()
+
+def line_interactive_send(ser, data):
+    """Send one line at a time, waiting for the prompt before sending next"""
+    for line in data.split(b'\n'):
+        ser.write(line)
+        ser.write(b'\n')
+        if not wait_prompt(ser, False):
+            return False
+        xprint('.')
+    return True
+
+def chunk_data(data, size):
+    """Split a data block into chunks"""
+    return (data[0+i:size+i] for i in range(0, len(data), size))
+
+def chunk_interactive_send(ser, data, size):
+    """Send the data chunked into blocks, waiting for an ack in between"""
+    n=0
+    for chunk in chunk_data(data, size):
+        ser.write(chunk)
+        if len(chunk) == size and not wait_prompt(ser, False):
+            print(f"failed after sending {n} blocks")
+            return False
+        xprint('.')
+        n += 1
+    print(f" ok, sent {n} blocks")
+    return True
+
+def transmission(data):
+    """Perform STX/ETX/DLE framing and escaping of the data"""
+    out = bytearray()
+    out.append(STX)
+    for b in data:
+        if b == STX or b == ETX or b == DLE:
+            out.append(DLE)
+        out.append(b)
+    out.append(ETX)
+    return bytes(out)
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    upload_name = args.name if args.name else args.file
+
+    file_data = load_file(args.file)
+    print(f"Loaded {len(file_data)} bytes of file contents")
+
+    blocksize = bytes(str(args.blocksize).encode())
+
+    try:
+        ser = serial.Serial(port=args.port, baudrate=args.bitrate, timeout=1)
+    except serial.SerialException as e:
+        print(f"Error opening serial port {args.port}: {e}")
+        sys.exit(1)
+
+    print("Synchronising serial...", end='')
+    if not sync(ser):
+        print("\nNodeMCU not responding\n")
+        sys.exit(1)
+
+    print(f' ok\nUploading "{args.file}" as "{upload_name}"')
+
+    atexit.register(cleanup)
+
+    xprint("Sending loader")
+    ok = line_interactive_send(
+        ser, loader.replace(
+            b"@FILENAME@", upload_name.encode()).replace(
+            b"@BLOCKSIZE@", blocksize))
+
+    if ok:
+        xprint(" ok\nWaiting for go-ahead...")
+        ok = wait_line_match(ser, b"ready", 5)
+
+    if ok:
+        xprint(f" ok\nSending file contents (using blocksize {args.blocksize})")
+        ok = chunk_interactive_send(
+            ser, transmission(file_data), int(blocksize))
+    if ok:
+        xprint("Waiting for final ack...")
+        ok = wait_line_match(ser, b"done", 5)
+        ser.write(b"\n")
+
+    if not ok or not wait_prompt(ser, False):
+        print("transmission timed out")
+        sys.exit(1)
+
+    ser.close()
+    ser = None
+    print(" ok\nUpload complete.")