IDF web server module (#3502)

* Added httpd module.

Lua-interface to the standard esp_http_server component.

* Added eromfs module.

components/modules/CMakeLists.txt

@@ -7,10 +7,12 @@ set(module_srcs
   "crypto.c"
   "dht.c"
   "encoder.c"
+  "eromfs.c"
   "file.c"
   "gpio.c"
   "heaptrace.c"
   "http.c"
+  "httpd.c"
   "i2c.c"
   "i2c_hw_master.c"
   "i2c_hw_slave.c"
@@ -74,6 +76,7 @@ idf_component_register(
     "driver_can"
     "esp_http_client"
     "fatfs"
+    "esp_http_server"
     "libsodium"
     "lua"
     "mbedtls"
@@ -138,3 +141,17 @@ set_property(
   DIRECTORY "${COMPONENT_DIR}" APPEND
   PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ucg_config.h u8g2_fonts.h u8g2_displays.h
 )
+
+# eromfs generation
+add_custom_command(
+  OUTPUT eromfs.bin
+  COMMAND ${COMPONENT_DIR}/eromfs.py ${CONFIG_NODEMCU_CMODULE_EROMFS_VOLUMES}
+  DEPENDS ${SDKCONFIG_HEADER}
+)
+add_custom_target(eromfs_bin DEPENDS eromfs.bin)
+target_add_binary_data(${COMPONENT_LIB} "${CMAKE_CURRENT_BINARY_DIR}/eromfs.bin" BINARY DEPENDS eromfs_bin)
+set_property(
+  DIRECTORY "${COMPONENT_DIR}" APPEND
+  PROPERTY ADDITIONAL_MAKE_CLEAN_FILES eromfs.bin
+)
+

components/modules/Kconfig

@@ -54,6 +54,34 @@ menu "NodeMCU modules"
             Includes the encoder module. This provides hex and base64 encoding
             and decoding functionality.
 
+    config NODEMCU_CMODULE_EROMFS
+        bool "Eromfs module (embedded read-only mountable file sets)"
+        select VFS_SUPPORT_IO
+        default "n"
+        help
+            Includes the eromfs module, giving access to the embedded mountable
+            file sets (volumes) configured here. Useful for bundling file sets
+            within the main firmware image, such as website contents.
+
+    config NODEMCU_CMODULE_EROMFS_VOLUMES
+        depends on NODEMCU_CMODULE_EROMFS
+        string "File sets to embed"
+        default "volume_name=/path/to/volume_root;myvol2=../relpath"
+        help
+            List one or more volume definitions in the form of
+            VolumeName=/path/to/files where the VolumeName is the identifier
+            by which the eromfs module will refer to the volume. The path
+            may be given as either a relative or absolute path. If relative,
+            it is relative to the top-level nodemcu-firmware directory.
+            All files and directories within the specified volume root will
+            be included. Symlinks are not supported and will result in
+            failure if encountered. Multiple volumes may be declared by
+            separating the entries with a semicolon.
+
+            Note that eromfs does not support directories per se, but will
+            store the directory path as part of the filename just as SPIFFS
+            does.
+
     config NODEMCU_CMODULE_ETH
         depends on IDF_TARGET_ESP32
         select ETH_USE_ESP32_EMAC
@@ -91,6 +119,32 @@ menu "NodeMCU modules"
         help
             Includes the HTTP module (recommended).
 
+    config NODEMCU_CMODULE_HTTPD
+        bool "Httpd (web server) module"
+        default "n"
+        help
+            Includes the HTTPD module. This module uses the regular IDF
+            http server component internally.
+
+    config NODEMCU_CMODULE_HTTPD_MAX_RESPONSE_HEADERS
+        int "Max response header fields" if NODEMCU_CMODULE_HTTPD
+        default 5
+        help
+            Determines how much space to allocate for header fields in the
+            HTTP response. This value does not include header fields the
+            http server itself generates internally, but only headers
+            explicitly returned in a dynamic route handler. Typically only
+            Content-Type is needed, so for most applications the default
+            value here will suffice.
+
+    config NODEMCU_CMODULE_HTTPD_RECV_BODY_CHUNK_SIZE
+        int "Receive body chunk size" if NODEMCU_CMODULE_HTTPD
+        default 1024
+        help
+            When receiving a body payload, receive at most this many
+            bytes at a time. Higher values means reduced overhead at
+            the cost of higher memory load.
+
     config NODEMCU_CMODULE_I2C
         bool "I2C module"
         default "y"

components/modules/eromfs.c

@@ -0,0 +1,379 @@
+#include "module.h"
+#include "lauxlib.h"
+
+#include "esp_vfs.h"
+#include <errno.h>
+#include <dirent.h>
+
+#include <stdint.h>
+#include <string.h>
+
+/**
+ * The logical layout of the embedded volumes is
+ * [ volume record ]
+ * [ volume record ]
+ * ...
+ * [ index record in vol1 ]
+ * [ index record in vol1 ]
+ * ...
+ * [ file contents in vol1 ]
+ * [ file contents in vol1 ]
+ * ...
+ * [ index record in vol2 ]
+ * [ index record in vol2 ]
+ * ...
+ * [ file contents in vol2 ]
+ * [ file contents in vol2 ]
+ *
+ * Both the volume records and index records are variable length so as to not
+ * waste space in their name fields. Finding the start of the index records
+ * for a volume is by reading the offs(et) field in the volume record and
+ * jumping that many bytes forward from the start of the eromfs.bin data.
+ * Similarly, finding the file contents is by reading the index record's
+ * offs(et) field and basing that off the start of the volume index.
+ * Naturally, the start of the volume index is the same as the end of the
+ * volume header, and the start of the file contents is the same as the
+ * end of the volume index, and either of those can be worked out by
+ * reading the offs(et) field in the first record.
+ */
+
+#pragma pack(push, 1)
+typedef struct {
+  uint8_t  rec_len;
+  uint16_t offs; // index_offs
+  char     name[];
+} volume_record_t;
+
+typedef struct {
+  uint8_t  rec_len;
+  uint32_t offs; // based off index_offs
+  uint32_t len; // file_len
+  char     name[];
+} index_record_t;
+#pragma pack(pop)
+
+
+typedef struct {
+  const index_record_t *meta;
+  const char *data; // start of data
+  off_t pos;
+} file_descriptor_t;
+
+
+typedef struct {
+  DIR opaque;
+  const index_record_t *index;
+  const index_record_t *pos;
+} eromfs_DIR_t;
+
+
+extern const char eromfs_bin_start[] asm("_binary_eromfs_bin_start");
+
+// Both the volume header and the file set indices end where the next
+// type of data block commences (file set index, file contents).
+#define end_of(type, start) ((const type *)(((char *)start) + start->offs))
+
+#define eromfs_header_start ((const volume_record_t *)eromfs_bin_start)
+#define eromfs_header_end   end_of(volume_record_t, eromfs_header_start)
+
+
+/* The logic for finding a volume record by name is the same as finding a
+ * file record by name, only the data structure type varies. Hence we
+ * hide the casting and variable length record stepping behind a convenience
+ * macro here.
+ */
+#define find_entry_by_name(out, xname, start_void_p, record_t) \
+  do { \
+    const record_t *entry_ = (const record_t *)(start_void_p); \
+    const record_t *end_ = end_of(record_t, entry_); \
+    unsigned xname_len = strlen(xname); \
+    for (; entry_ < end_; \
+         entry_ = (const record_t *)(((char *)entry_) + entry_->rec_len)) \
+    { \
+      uint8_t name_len = entry_->rec_len - sizeof(record_t); \
+      if (xname_len == name_len && \
+          strncmp(xname, entry_->name, name_len) == 0) \
+      { \
+        out = entry_; \
+        break; \
+      } \
+    } \
+  } while(0);
+
+
+static int mounted_volumes = LUA_NOREF;
+
+
+static SemaphoreHandle_t fd_mutex;
+
+static file_descriptor_t fds[CONFIG_NODEMCU_MAX_OPEN_FILES];
+
+
+// --- VFS interface -----------------------------------------------------
+
+
+#define get_index() const index_record_t *index = (const index_record_t *)ctx
+
+static const index_record_t *path2entry(void *ctx, const char *path)
+{
+  while (*path == '/')
+    ++path;
+
+  get_index();
+  const index_record_t *entry = NULL;
+  find_entry_by_name(entry, path, index, index_record_t);
+  return entry;
+}
+
+
+static int eromfs_fstat(void *ctx, int fd, struct stat *st)
+{
+  memset(st, 0, sizeof(struct stat));
+  st->st_size = fds[fd].meta->len;
+  st->st_blocks = (fds[fd].meta->len + 511)/512;
+  return 0;
+}
+
+
+#ifdef CONFIG_VFS_SUPPORT_DIR
+static int eromfs_stat(void *ctx, const char *path, struct stat *st)
+{
+  const index_record_t *entry = path2entry(ctx, path);
+  if (!entry)
+    return -ENOENT;
+  memset(st, 0, sizeof(struct stat));
+  st->st_size = entry->len;
+  st->st_blocks = (entry->len + 511)/512;
+  return 0;
+}
+
+
+static DIR *eromfs_opendir(void *ctx, const char *path)
+{
+  if (strcmp(path, "/") != 0)
+    return NULL;
+
+  get_index();
+  eromfs_DIR_t *dir = calloc(1, sizeof(eromfs_DIR_t));
+  dir->index = index;
+  dir->pos = index;
+  return (DIR *)dir;
+}
+
+
+static struct dirent *eromfs_readdir(void *ctx, DIR *pdir)
+{
+  UNUSED(ctx);
+  eromfs_DIR_t *dir = (eromfs_DIR_t *)pdir;
+
+  const index_record_t *end = end_of(index_record_t, dir->index);
+  if (dir->pos >= end)
+    return NULL;
+
+  static struct dirent de = {
+    .d_ino = 0,
+    .d_type = DT_REG,
+  };
+  size_t max_len = sizeof(de.d_name);
+  size_t len = dir->pos->rec_len - sizeof(index_record_t);
+  if (len > max_len -1)
+    len = max_len - 1;
+  strncpy(de.d_name, dir->pos->name, len);
+  de.d_name[len] = 0;
+  dir->pos = (const index_record_t *)((char *)dir->pos + dir->pos->rec_len);
+  return &de;
+}
+
+
+static int eromfs_closedir(void *ctx, DIR *dir)
+{
+  UNUSED(ctx);
+  free(dir);
+  return 0;
+}
+#endif
+
+
+static int eromfs_open(void *ctx, const char *path, int flags, int mode)
+{
+  UNUSED(flags);
+  UNUSED(mode);
+  const index_record_t *entry = path2entry(ctx, path);
+  if (!entry)
+    return -ENOENT;
+
+  xSemaphoreTake(fd_mutex, portMAX_DELAY);
+  int fd = -ENFILE;
+  // max open files is guaranteed to be small; linear search is fine
+  for (unsigned i = 0; i < CONFIG_NODEMCU_MAX_OPEN_FILES; ++i)
+  {
+    if (fds[i].meta == NULL)
+    {
+      fds[i].meta = entry;
+      fds[i].data = (const char *)ctx + entry->offs;
+      fds[i].pos = 0;
+      fd = (int)i;
+    }
+  }
+  xSemaphoreGive(fd_mutex);
+
+  return fd;
+}
+
+
+static ssize_t eromfs_read(void *ctx, int fd, void *dst, size_t size)
+{
+  UNUSED(ctx);
+  size_t avail = fds[fd].meta->len - fds[fd].pos;
+  if (size > avail)
+    size = avail;
+  const char *src = fds[fd].data + fds[fd].pos;
+  memcpy(dst, src, size);
+  fds[fd].pos += size;
+  return size;
+}
+
+
+static off_t eromfs_lseek(void *ctx, int fd, off_t size, int mode)
+{
+  UNUSED(ctx);
+  off_t pos = fds[fd].pos;
+  switch(mode)
+  {
+    case SEEK_SET: pos = size; break;
+    case SEEK_CUR: pos += size; break;
+    case SEEK_END: pos = fds[fd].meta->len + size; break;
+    default:
+      return -EINVAL;
+  }
+  if (pos < 0 || pos > fds[fd].meta->len)
+    return -EINVAL;
+  fds[fd].pos = pos;
+  return pos;
+}
+
+
+static int eromfs_close(void *ctx, int fd)
+{
+  UNUSED(ctx);
+  xSemaphoreTake(fd_mutex, portMAX_DELAY);
+  fds[fd].meta = NULL;
+  fds[fd].data = NULL;
+  fds[fd].pos = 0;
+  xSemaphoreGive(fd_mutex);
+  return 0;
+}
+
+
+// --- Lua interface -----------------------------------------------------
+
+static int leromfs_list(lua_State *L)
+{
+  lua_newtable(L);
+  int t = lua_gettop(L);
+  // If this logic looks similar to the find_entry_by_name() macro, it's
+  // because it is :)  Except we're capturing all the volume names, so no
+  // easy reuse.
+  const volume_record_t *vol = eromfs_header_start;
+  const volume_record_t *end = eromfs_header_end;
+  for (; vol < end; vol = (const volume_record_t *)((char *)vol + vol->rec_len))
+  {
+    uint8_t volume_name_len = vol->rec_len - sizeof(volume_record_t);
+    lua_pushlstring(L, vol->name, volume_name_len);
+    lua_rawseti(L, t, lua_objlen(L, t) + 1);
+  }
+  return 1;
+}
+
+
+static int leromfs_mount(lua_State *L)
+{
+  const char *name = luaL_checkstring(L, 1);
+  const char *mountpt = luaL_checkstring(L, 2);
+  lua_settop(L, 2);
+
+  const volume_record_t *vol = NULL;
+  find_entry_by_name(vol, name, eromfs_bin_start, volume_record_t);
+  if (!vol)
+    return luaL_error(L, "volume %s not found", name);
+
+  const index_record_t *index_start =
+    (const index_record_t *)(eromfs_bin_start + vol->offs);
+
+  esp_vfs_t eromfs = {
+    .flags = ESP_VFS_FLAG_CONTEXT_PTR,
+    .open_p = eromfs_open,
+    .fstat_p = eromfs_fstat,
+    .read_p = eromfs_read,
+    .lseek_p = eromfs_lseek,
+    .close_p = eromfs_close,
+#ifdef CONFIG_VFS_SUPPORT_DIR
+    .stat_p = eromfs_stat,
+    .opendir_p = eromfs_opendir,
+    .readdir_p = eromfs_readdir,
+    .closedir_p = eromfs_closedir,
+#endif
+  };
+  esp_err_t err = esp_vfs_register(mountpt, &eromfs, (void *)index_start);
+  if (err != ESP_OK)
+    return luaL_error(L, "failed to mount eromfs; code %d", err);
+
+  lua_rawgeti(L, LUA_REGISTRYINDEX, mounted_volumes);
+  lua_pushvalue(L, 2);
+  lua_pushvalue(L, 1);
+  lua_rawset(L, -3); // mounted_volumes[mountpt] = name
+
+  return 0;
+}
+
+
+static int leromfs_unmount(lua_State *L)
+{
+  const char *name = luaL_checkstring(L, 1);
+  const char *mountpt = luaL_checkstring(L, 2);
+  lua_settop(L, 2);
+
+  lua_rawgeti(L, LUA_REGISTRYINDEX, mounted_volumes);
+  lua_pushvalue(L, 2);
+  lua_rawget(L, -2);
+  if (lua_isstring(L, -1))
+  {
+    const char *mounted_name = lua_tostring(L, -1);
+    if (strcmp(name, mounted_name) == 0)
+    {
+      esp_err_t err = esp_vfs_unregister(mountpt);
+      if (err != ESP_OK)
+        return luaL_error(L, "unmounting failed; code %d", err);
+      lua_pop(L, 1);
+      lua_pushvalue(L, 2);
+      lua_pushnil(L);
+      lua_rawset(L, -3); // mounted_volumes[mountpt] = nil
+      return 0;
+    }
+    else
+      return luaL_error(L,
+        "can't umount %s from %s; volume %s is mounted there",
+        name, mountpt, mounted_name);
+  }
+  else
+    return 0; // already unmounted, not an error
+}
+
+
+static int leromfs_init(lua_State *L)
+{
+  fd_mutex = xSemaphoreCreateMutex();
+
+  lua_newtable(L);
+  mounted_volumes = luaL_ref(L, LUA_REGISTRYINDEX);
+  return 0;
+}
+
+
+LROT_BEGIN(eromfs, NULL, 0)
+  LROT_FUNCENTRY( list,    leromfs_list )
+  LROT_FUNCENTRY( mount,   leromfs_mount )
+  LROT_FUNCENTRY( unmount, leromfs_unmount )
+LROT_END(eromfs, NULL, 0)
+
+NODEMCU_MODULE(EROMFS, "eromfs", eromfs, leromfs_init);

components/modules/eromfs.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import struct
+
+# [volume records]
+# reclen, index_offs, name
+# reclen, index_offs2, name2
+# [file index]
+# reclen, offs, file_len, name   # offs base index_offs
+# reclen, offs, file_len, name2  # offs base index_offs2 
+# [file contents]
+# rawdata
+# rawdata
+
+vol_names = [] # one entry per volume
+volume_indexes = [] # one entry per volume
+volume_file_contents = [] # one entry per volume
+
+for voldef in sys.argv[1:]:
+    [ name, basedir ] = voldef.split('=')
+    print(f'==> Packing volume "{name}" from {basedir}')
+    vol_names.append(name)
+    # Make relative paths relative to the top nodemcu-firmware dir; this
+    # script gets executed with build/esp-idf/modules as the current dir
+    if not os.path.isabs(basedir):
+        basedir = os.path.join(*['..', '..', '..', basedir])
+    if not os.path.isdir(basedir):
+        raise FileNotFoundError(f'source directory {basedir} not found')
+    basedir_len = len(basedir) +1
+    file_index = b''
+    file_data = b''
+    offs = 0
+    entries = []
+    index_size = 0
+    for root, subdirs, files in os.walk(basedir):
+        prefix = ('' if root == basedir else root[basedir_len:] + '/')
+        for filename in files:
+            hostrelpath = os.path.join(root, filename)
+            relpath = prefix + filename
+            size = os.path.getsize(hostrelpath)
+            rec_len = 1 + 4 + 4 + len(relpath) # reclen + offs + filelen + name
+            if rec_len > 255:
+                raise ValueError(f'excessive path length for {relpath}')
+            entries.append([ rec_len, offs, size, relpath ])
+            offs += size
+            index_size += rec_len
+            with open(hostrelpath, mode='rb') as f:
+                file_data += f.read()
+    for entry in entries:
+        [ rec_len, offs, size, relpath ] = entry
+        print('[', rec_len, index_size + offs, size, relpath, ']')
+        file_index += \
+            struct.pack('<BII', rec_len, index_size + offs, size) + \
+            relpath.encode('utf-8')
+
+    volume_indexes.append(file_index)
+    volume_file_contents.append(file_data)
+
+volume_records_len = len(vol_names) * (1 + 2) + len(''.join(vol_names))
+print(f'==> Generating volumes index ({volume_records_len} bytes)')
+with open('eromfs.bin', 'wb') as f:
+    index_offs = volume_records_len
+    for idx, name in enumerate(vol_names):
+        rec_len = 1 + 2 + len(name)
+        index_len = len(volume_indexes[idx])
+        data_len = len(volume_file_contents[idx])
+        if rec_len > 255:
+            raise ValueError(f'volume name too long for {name}')
+        if index_offs > 65535:
+            raise ValueError('volumes index overflowed; too many volumes')
+        f.write(
+            struct.pack('<BH', rec_len, index_offs) + name.encode('utf-8') \
+        )
+        print(f'- {name} (index {index_len} bytes; content {data_len} bytes)')
+        index_offs += index_len + data_len
+    for idx, index in enumerate(volume_indexes):
+        f.write(index)
+        f.write(volume_file_contents[idx])

components/modules/httpd.c

@@ -0,0 +1,754 @@
+#include "module.h"
+#include "lauxlib.h"
+
+#include "esp_http_server.h"
+
+#include "task/task.h"
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/semphr.h>
+#include <freertos/queue.h>
+
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * NodeMCU module for interfacing with the esp_http_server component.
+ * Said HTTP server runs in its own thread (RTOS task) separate from the
+ * LVM thread. As such, running dynamic route handlers in Lua requires a
+ * certain amount of thread synchronisation in order to work safely.
+ *
+ * Effectively, when a dynamic handler is invoked, the functions
+ * dynamic_handler_httpd() and dynamic_handler_lvm() will be running in
+ * lockstep. The former kicks off the latter with task_post() of the
+ * relevant HTTP request information, and then proceeds to servicing
+ * requests from dynamic_handler_lvm().
+ *
+ * There are three things dynamic_handler_lvm() may request from
+ * dynamic_handler_httpd():
+ *
+ *  - Header values. The esp_http_server component provides no method
+ *    of enumerating the received headers, so this pull approach is
+ *    necssary.
+ *
+ *  - Body data. In order to facilitate large document bodies the
+ *    body is not read in up front, but is instead requested chunk
+ *    by chunk from the dynamic handler. This allows for streaming
+ *    in e.g. a full OTA image and writing it progressively.
+ *
+ *  - Sending the response. This includes status message, content type
+ *    and any body data. The body data may either be submitted in a single
+ *    go, or a function to "pull" the body data chunk by chunk may be
+ *    given, in which case chunked encoding is used for the response body
+ *    and the content length needs not be known in advance.
+ *
+ * @author Johny Mattsson (johny.mattsson+github@gmail.com)
+ */
+
+// More wieldly names for the Kconfig settings
+#define MAX_RESPONSE_HEADERS  CONFIG_NODEMCU_CMODULE_HTTPD_MAX_RESPONSE_HEADERS
+#define RECV_BODY_CHUNK_SIZE  CONFIG_NODEMCU_CMODULE_HTTPD_RECV_BODY_CHUNK_SIZE
+
+#define REQUEST_METATABLE "httpd.req"
+
+typedef struct {
+  const char *key;
+  const char *value;
+} key_value_t;
+
+
+typedef struct {
+  const char *status_str; // e.g. "200 OK"
+  key_value_t headers[MAX_RESPONSE_HEADERS];
+  const char *content_type; // specially handled in esp_http_server
+  size_t body_len;
+  const char *body_data; // may be binary data, hence body_len above
+} response_data_t;
+
+
+// Request from the LVM thread back to the httpd thread *during* request
+// processing in a dynamic handler.
+typedef enum {
+  GET_HEADER,
+  READ_BODY_CHUNK,
+  SEND_RESPONSE,
+  SEND_PARTIAL_RESPONSE,
+} request_type_t;
+
+typedef struct {
+  size_t used;
+  char data[RECV_BODY_CHUNK_SIZE];
+} body_chunk_t;
+
+typedef struct {
+  request_type_t request_type;
+  union {
+    struct {
+      const char *name; // owned by LVM thread
+      char **value; // allocated in httpd thread, free()d in LVM thread
+    } header;
+    body_chunk_t **body_chunk; // allocated in httpd thread, free()d in LVM
+    const response_data_t *response; // owned by LVM thread
+  };
+} thread_request_t;
+
+
+typedef struct {
+  const char *key; // dynamic handler lookup key
+  const char *uri;
+  const char *query_str;
+  int method;
+  size_t body_len;
+} request_data_t;
+
+
+typedef struct {
+  const request_data_t *req_info;
+  uint32_t guard;
+} req_udata_t;
+
+
+typedef enum { INDEX_NONE, INDEX_ROOT, INDEX_ALL } index_mode_t;
+
+
+// Task handle for httpd->LVM thread task posting
+static task_handle_t dynamic_task;
+
+// Single-slot queue for passing requests from LVM->httpd thread.
+static QueueHandle_t queue;
+
+// Semaphore for releasing the LVM thread once the thread_request has been
+// processed by the httpd thread.
+static SemaphoreHandle_t done;
+
+// Server instance
+static httpd_handle_t server = NULL;
+
+// Path prefix for static files; allocated in LVM thread, used in httpd thread.
+// Needed since currently no way to free user_ctx on unregister_uri_handler()
+static char *webroot;
+
+// Auto-index mode, configured at server start.
+static index_mode_t index_mode;
+
+// Tables for keeping our registered handlers and content type strings
+// safe from garbage collection until we want them cleaned up.
+static int content_types_table_ref = LUA_NOREF;
+static int dynamic_handlers_table_ref = LUA_NOREF;
+
+// Simple guard against deadlocking by calling gethdr()/getbody() outside
+// the dynamic handler flow.
+static uint32_t guard = 0;
+
+// Known static file suffixes and their content type. Automatically registered
+// on server start.
+static const char *default_suffixes[] =
+{
+  "*.html\0text/html",
+  "*.css\0text/css",
+  "*.js\0text/javascript",
+  "*.txt\0text/plain",
+  "*.json\0application/json",
+  "*.gif\0image/gif",
+  "*.jpg\0image/jpeg",
+  "*.jpeg\0image/jpeg",
+  "*.png\0image/png",
+  "*.svg\0image/svg+xml",
+  "*.ttf\0font/ttf",
+};
+
+
+// Everybody's favourite response status
+static const char internal_err[] = "500 Internal Server Error";
+
+static const response_data_t error_resp = {
+  .status_str = internal_err,
+  .content_type = "text/plain",
+  .body_len = sizeof(internal_err) - 1,
+  .body_data = internal_err,
+};
+
+// ---- Runs in httpd task/thread -------------------------------------
+
+static bool uri_match_file_suffix_first(const char *uri_template, const char *uri_to_match, size_t match_upto)
+{
+  if (uri_template[0] == '*')
+  {
+    // uri_template in form of "*.sufx"
+    const char *suffix = uri_template + 1; // skip leading '*'
+    size_t suffix_len = strlen(suffix);
+    const char *uri_suffix = uri_to_match + match_upto - suffix_len;
+    return strncmp(suffix, uri_suffix, suffix_len) == 0;
+  }
+  else if (uri_template[0] == '\0')
+  {
+    // auto-indexer template
+    switch(index_mode)
+    {
+      case INDEX_NONE: return false;
+      case INDEX_ROOT: return (match_upto == 1) && (uri_to_match[0] == '/');
+      case INDEX_ALL: return uri_to_match[match_upto - 1] == '/';
+      default: return false;
+    }
+  }
+  else
+    return httpd_uri_match_wildcard(uri_template, uri_to_match, match_upto);
+}
+
+
+static void serve_file(httpd_req_t *req, const char *fname)
+{
+  FILE *f = fopen(fname, "r");
+  if (f)
+  {
+    char *buf = malloc(RECV_BODY_CHUNK_SIZE);
+    ssize_t n = 0;
+    while ((n = fread(buf, 1, RECV_BODY_CHUNK_SIZE, f)) > 0)
+    {
+      if (httpd_resp_send_chunk(req, buf, n) != ESP_OK)
+        break;
+    }
+    httpd_resp_send_chunk(req, buf, 0);
+    free(buf);
+  }
+  else
+    httpd_resp_send_404(req);
+
+  fclose(f);
+}
+
+
+static esp_err_t static_file_handler(httpd_req_t *req)
+{
+  httpd_resp_set_type(req, (const char *)req->user_ctx);
+
+  char *fname = NULL;
+  asprintf(&fname, "%s%s", webroot, req->uri);
+  serve_file(req, fname);
+  free(fname);
+
+  return ESP_OK;
+};
+
+
+static esp_err_t auto_index_handler(httpd_req_t *req)
+{
+  char *fname = NULL;
+  asprintf(&fname, "%s%.*s/index.html", webroot, strlen(req->uri) -1, req->uri);
+  serve_file(req, fname);
+  free(fname);
+  return ESP_OK;
+}
+
+
+static esp_err_t dynamic_handler_httpd(httpd_req_t *req)
+{
+  size_t query_len = httpd_req_get_url_query_len(req);
+  char *query = query_len ? malloc(query_len + 1) : NULL;
+  if (query_len)
+    httpd_req_get_url_query_str(req, query, query_len + 1);
+
+  request_data_t req_data = {
+    .key = (const char *)req->user_ctx,
+    .uri = req->uri,
+    .query_str = query,
+    .method = req->method,
+    .body_len = req->content_len,
+  };
+  // Pass the req info over to the LVM thread
+  task_post_medium(dynamic_task, (task_param_t)&req_data);
+
+  size_t remaining_len = req->content_len;
+  bool errored = false;
+  thread_request_t tr;
+  do {
+    // Block the httpd thread until we receive the response data, or requests
+    // for headers/body data, which can only be serviced from this thread.
+    xQueueReceive(queue, &tr, portMAX_DELAY);
+    if (tr.request_type == GET_HEADER)
+    {
+      size_t len = httpd_req_get_hdr_value_len(req, tr.header.name);
+      if (len)
+      {
+        *tr.header.value = malloc(len + 1);
+        httpd_req_get_hdr_value_str(
+          req, tr.header.name, *tr.header.value, len + 1);
+      }
+      else
+        *tr.header.value = NULL; // no such header
+    }
+    else if (tr.request_type == READ_BODY_CHUNK)
+    {
+      *tr.body_chunk = malloc(sizeof(body_chunk_t));
+      size_t to_read = (remaining_len >= RECV_BODY_CHUNK_SIZE) ?
+        RECV_BODY_CHUNK_SIZE : remaining_len;
+      remaining_len -= to_read;
+      int ret = httpd_req_recv(req, (*tr.body_chunk)->data, to_read);
+      if (ret != to_read)
+      {
+        errored = true;
+        free(*tr.body_chunk);
+        *tr.body_chunk = NULL;
+      }
+      else
+        (*tr.body_chunk)->used = to_read;
+    }
+    else if (tr.request_type == SEND_RESPONSE ||
+             tr.request_type == SEND_PARTIAL_RESPONSE)
+    {
+      if (errored)
+        httpd_resp_send_408(req);
+      else
+      {
+        bool is_partial = (tr.request_type == SEND_PARTIAL_RESPONSE);
+        const response_data_t *resp = tr.response;
+        if (!is_partial || resp->status_str)
+          httpd_resp_set_status(req, resp->status_str);
+        if (!is_partial || resp->content_type)
+          httpd_resp_set_type(req, resp->content_type);
+        for (unsigned i = 0; resp->headers[i].key; ++i)
+          httpd_resp_set_hdr(req, resp->headers[i].key, resp->headers[i].value);
+        if (!is_partial)
+          httpd_resp_send(req, resp->body_data, resp->body_len);
+        else
+        {
+          httpd_resp_send_chunk(req, resp->body_data, resp->body_len);
+          if (resp->body_data == NULL) // Was this the last chunk?
+            tr.request_type = SEND_RESPONSE; // If so, flag our exit condition
+        }
+      }
+    }
+
+    // Request processed, release LVM thread
+    xSemaphoreGive(done);
+  } while(tr.request_type != SEND_RESPONSE); // done
+
+  return ESP_OK;
+}
+
+
+// ---- helper functions ----------------------------------------------
+
+
+static int check_valid_httpd_method(lua_State *L, int idx)
+{
+  int method = luaL_checkinteger(L, idx);
+  switch (method)
+  {
+    case HTTP_GET:
+    case HTTP_HEAD:
+    case HTTP_PUT:
+    case HTTP_POST:
+    case HTTP_DELETE: break;
+    default: return luaL_error(L, "unknown method %d", method);
+  }
+  return method;
+}
+
+
+static void check_valid_guard_value(lua_State *L)
+{
+  int check = lua_tointeger(L, lua_upvalueindex(1));
+  if (check != guard)
+    luaL_error(L, "gethdr()/getbody() called outside synchronous flow");
+}
+
+
+static int lsync_get_hdr(lua_State *L)
+{
+  check_valid_guard_value(L);
+
+  const char *header_name = luaL_checkstring(L, 2);
+  char *header_val = NULL;
+  thread_request_t tr = {
+    .request_type = GET_HEADER,
+    .header = {
+      .name = header_name,
+      .value = &header_val,
+    }
+  };
+  xQueueSend(queue, &tr, portMAX_DELAY);
+  xSemaphoreTake(done, portMAX_DELAY);
+  if (header_val)
+    lua_pushstring(L, header_val);
+  else
+    lua_pushnil(L);
+  free(header_val);
+  return 1;
+}
+
+
+static int lsync_get_body_chunk(lua_State *L)
+{
+  check_valid_guard_value(L);
+
+  body_chunk_t *chunk = NULL;
+  thread_request_t tr = {
+    .request_type = READ_BODY_CHUNK,
+    .body_chunk = &chunk,
+  };
+  xQueueSend(queue, &tr, portMAX_DELAY);
+  xSemaphoreTake(done, portMAX_DELAY);
+  if (chunk)
+  {
+    if (chunk->used)
+      lua_pushlstring(L, chunk->data, chunk->used);
+    else
+      lua_pushnil(L); // end of body reached
+  }
+  else
+    return luaL_error(L, "read body failed");
+  return 1;
+}
+
+
+static int lhttpd_req_index(lua_State *L)
+{
+  req_udata_t *ud = (req_udata_t *)luaL_checkudata(L, 1, REQUEST_METATABLE);
+  const char *key = luaL_checkstring(L, 2);
+#define KEY_IS(x) (strcmp(key, x) == 0)
+  if (KEY_IS("uri"))
+    lua_pushstring(L, ud->req_info->uri);
+  else if (KEY_IS("method"))
+    lua_pushinteger(L, ud->req_info->method);
+  else if (KEY_IS("query") && ud->req_info->query_str)
+    lua_pushstring(L, ud->req_info->query_str);
+  else if (KEY_IS("headers"))
+  {
+    lua_newtable(L);
+    lua_newtable(L); // metatable
+    lua_pushinteger(L, ud->guard); // +1
+    lua_pushcclosure(L, lsync_get_hdr, 1); // -1 +1
+    lua_setfield(L, -2, "__index"); // -1
+    lua_setmetatable(L, -2); // -1
+  }
+  else if (KEY_IS("getbody"))
+  {
+    lua_pushinteger(L, guard); // +1
+    lua_pushcclosure(L, lsync_get_body_chunk, 1); // -1 +1
+  }
+  else
+    lua_pushnil(L);
+
+  return 1;
+#undef KEY_IS
+}
+
+
+static void dynamic_handler_lvm(task_param_t param, task_prio_t prio)
+{
+  UNUSED(prio);
+
+  const request_data_t *req_info = (const request_data_t *)param;
+
+  lua_State *L = lua_getstate();
+  int saved_top = lua_gettop(L);
+
+  lua_checkstack(L, MAX_RESPONSE_HEADERS*2 + 9);
+
+  response_data_t resp = error_resp;
+  thread_request_t tr = {
+    .request_type = SEND_RESPONSE,
+    .response = &resp,
+  };
+
+  lua_rawgeti(L, LUA_REGISTRYINDEX, dynamic_handlers_table_ref); // +1
+  lua_getfield(L, -1, req_info->key); // +1
+  if (lua_isfunction(L, -1))
+  {
+    // push req
+    req_udata_t *ud =
+      (req_udata_t *)lua_newuserdata(L, sizeof(req_udata_t)); // +1
+    ud->req_info = req_info;
+    ud->guard = guard;
+    luaL_getmetatable(L, REQUEST_METATABLE); // +1
+    lua_setmetatable(L, -2); // -1
+
+    int err = luaL_pcallx(L, 1, 1); // -1 +1
+    if (!err && lua_istable(L, -1))
+    {
+      // pull out response data
+      int t = lua_gettop(L); // response table index
+      lua_getfield(L, t, "status"); // +1
+      resp.status_str = luaL_optstring(L, -1, "200 OK");
+      lua_getfield(L, t, "type"); // +1
+      resp.content_type = luaL_optstring(L, -1, NULL);
+      lua_getfield(L, t, "body"); // +1
+      resp.body_data = luaL_optlstring(L, -1, NULL, &resp.body_len);
+      if (!resp.body_data)
+        resp.body_len = 0;
+      lua_getfield(L, t, "headers"); // +1
+      if (lua_istable(L, -1))
+      {
+        lua_pushnil(L); // +1
+        for (unsigned i = 0; lua_next(L, -2); ++i) // +1
+        {
+          if (i >= MAX_RESPONSE_HEADERS)
+          {
+            printf("Warning - too many response headers, ignoring some!\n");
+            break;
+          }
+          resp.headers[i].key = lua_tostring(L, -2);
+          resp.headers[i].value = lua_tostring(L, -1);
+          lua_pop(L, 1); // drop value, keep key for lua_next()
+        }
+      }
+      lua_getfield(L, t, "getbody"); // +1
+      if (lua_isfunction(L, -1))
+      {
+        // Okay, we're doing a chunked body send, so we have to repeatedly
+        // call the provided getbody() function until it returns nil
+        bool headers_cleared = false;
+        tr.request_type = SEND_PARTIAL_RESPONSE;
+next_chunk:
+        resp.body_data = NULL;
+        resp.body_len = 0;
+        err = luaL_pcallx(L, 0, 1); // -1 +1
+        resp.body_data =
+          err ? NULL : luaL_optlstring(L, -1, NULL, &resp.body_len);
+        if (resp.body_data)
+        {
+          // Toss this bit of response data over to the httpd thread
+          xQueueSend(queue, &tr, portMAX_DELAY);
+          // ...and wait until it's done sending it
+          xSemaphoreTake(done, portMAX_DELAY);
+
+          lua_pop(L, 1); // -1
+
+          if (!headers_cleared)
+          {
+            // Clear the header data; it's only used for the first chunk
+            resp.status_str = NULL;
+            resp.content_type = NULL;
+            for (unsigned i = 0; i < MAX_RESPONSE_HEADERS; ++i)
+              resp.headers[i].key = resp.headers[i].value = NULL;
+
+            headers_cleared = true;
+          }
+          lua_getfield(L, t, "getbody"); // +1
+          goto next_chunk;
+        }
+        // else, getbody() returned nil, so let the normal exit path
+        // toss the final SEND_PARTIAL_RESPONSE request over to the httpd
+      }
+    }
+  }
+
+  // Toss the response data over to the httpd thread for sending
+  xQueueSend(queue, &tr, portMAX_DELAY);
+
+  // Block until the httpd thread has finished accessing our Lua strings
+  xSemaphoreTake(done, portMAX_DELAY);
+
+  // Clean up the stack
+  lua_settop(L, saved_top);
+
+  // Make any further gethdr()/getbody() calls fail rather than deadlock
+  ++guard;
+}
+
+
+// ---- Lua interface -------------------------------------------------
+
+
+// add static route: httpd.static(uri, content_type)
+static int lhttpd_static(lua_State *L)
+{
+  if (!server)
+    return luaL_error(L, "Server not started");
+
+  const char *match = luaL_checkstring(L, 1);
+  const char *content_type = luaL_checkstring(L, 2);
+
+  if (!match[0])
+    return luaL_error(L, "Null route not supported");
+
+  // Store this in our content-type table, so the content-type string lives
+  // on, but so that we can also free it after server shutdown.
+  lua_rawgeti(L, LUA_REGISTRYINDEX, content_types_table_ref);
+  lua_pushvalue(L, 1);
+  lua_pushvalue(L, 2);
+  lua_settable(L, -3);
+
+  httpd_uri_t static_handler = {
+    .uri = match,
+    .method = HTTP_GET,
+    .handler = static_file_handler,
+    .user_ctx = (void *)content_type,
+  };
+  if (httpd_register_uri_handler(server, &static_handler) == 1)
+    lua_pushinteger(L, 1);
+  else
+    lua_pushnil(L);
+
+  return 1;
+}
+
+
+// add dynamic route: httpd.dynamic(method, uri, handler)
+static int lhttpd_dynamic(lua_State *L)
+{
+  if (!server)
+    return luaL_error(L, "Server not started");
+
+  int method = check_valid_httpd_method(L, 1);
+  const char *match = luaL_checkstring(L, 2);
+  luaL_checkfunction(L, 3);
+  lua_settop(L, 3);
+
+  if (!match[0])
+    return luaL_error(L, "Null route not supported");
+
+  // Create a key for this entry
+  const char *key = lua_pushfstring(L, "[%d]%s", method, match);
+
+  // Store this in our dynamic handlers table, so the ref lives on
+  // on, but so that we can also free it after server shutdown.
+  lua_rawgeti(L, LUA_REGISTRYINDEX, dynamic_handlers_table_ref);
+  lua_pushvalue(L, -2); // key
+  lua_pushvalue(L, 3); // handler
+  lua_settable(L, -3);
+
+  httpd_uri_t static_handler = {
+    .uri = match,
+    .method = method,
+    .handler = dynamic_handler_httpd,
+    .user_ctx = (void *)key,
+  };
+  if (httpd_register_uri_handler(server, &static_handler) == 1)
+    lua_pushinteger(L, 1);
+  else
+    lua_pushnil(L);
+
+  return 1;
+}
+
+
+// unregister route; httpd.unregister(method, uri)
+static int lhttpd_unregister(lua_State *L)
+{
+  if (!server)
+    return luaL_error(L, "Server not started");
+
+  int method = check_valid_httpd_method(L, 1);
+  const char *match = luaL_checkstring(L, 2);
+
+  if (httpd_unregister_uri_handler(server, match, method) == ESP_OK)
+    lua_pushinteger(L, 1);
+  else
+    lua_pushnil(L);
+
+  return 1;
+}
+
+
+static int lhttpd_start(lua_State *L)
+{
+  if (server)
+    return luaL_error(L, "Server already started");
+
+  luaL_checktable(L, 1);
+  lua_settop(L, 1);
+
+  lua_getfield(L, 1, "webroot");
+  const char *root = luaL_checkstring(L, -1);
+  webroot = strdup(root);
+  lua_pop(L, 1);
+
+  lua_getfield(L, 1, "max_handlers");
+  int max_handlers = luaL_optinteger(L, -1, 20);
+
+  lua_getfield(L, 1, "auto_index");
+  index_mode = (index_mode_t)luaL_optinteger(L, -1, INDEX_ROOT);
+
+  httpd_config_t config = HTTPD_DEFAULT_CONFIG();
+  config.uri_match_fn = uri_match_file_suffix_first;
+  config.max_uri_handlers = max_handlers;
+  config.max_resp_headers = MAX_RESPONSE_HEADERS;
+
+  esp_err_t err = httpd_start(&server, &config);
+  if (err != ESP_OK)
+    return luaL_error(L, "Failed to start http server; code %d", err);
+
+  // Set up our content type stash
+  lua_newtable(L);
+  content_types_table_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+  // Set up our dynamic handlers table
+  lua_newtable(L);
+  dynamic_handlers_table_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+  // Register default static suffixes
+  size_t num_suffixes = sizeof(default_suffixes)/sizeof(default_suffixes[0]);
+  for (size_t i = 0; i < num_suffixes; ++i)
+  {
+    const char *sufx = default_suffixes[i];
+    const char *content_type = strchr(sufx, '\0') + 1;
+    lua_pushcfunction(L, lhttpd_static);
+    lua_pushstring(L, sufx);
+    lua_pushstring(L, content_type);
+    lua_call(L, 2, 0);
+  }
+
+  // Auto-indexer
+  httpd_uri_t index_handler = {
+    .uri = "",
+    .method = HTTP_GET,
+    .handler = auto_index_handler,
+  };
+  httpd_register_uri_handler(server, &index_handler);
+
+  return 0;
+}
+
+
+static int lhttpd_stop(lua_State *L)
+{
+  if (server)
+  {
+    httpd_stop(server); // deletes all handlers
+    server = NULL;
+    free(webroot);
+    luaL_unref2(L, LUA_REGISTRYINDEX, content_types_table_ref);
+    luaL_unref2(L, LUA_REGISTRYINDEX, dynamic_handlers_table_ref);
+  }
+  return 0;
+}
+
+
+LROT_BEGIN(httpd_req_mt, NULL, LROT_MASK_INDEX)
+  LROT_FUNCENTRY( __index, lhttpd_req_index )
+LROT_END(httpd_req_mt, NULL, LROT_MASK_INDEX)
+
+
+LROT_BEGIN(httpd, NULL, 0)
+  LROT_FUNCENTRY( start,      lhttpd_start )
+  LROT_FUNCENTRY( stop,       lhttpd_stop )
+  LROT_FUNCENTRY( static,     lhttpd_static )
+  LROT_FUNCENTRY( dynamic,    lhttpd_dynamic )
+  LROT_FUNCENTRY( unregister, lhttpd_unregister )
+
+  LROT_NUMENTRY( GET,         HTTP_GET )
+  LROT_NUMENTRY( HEAD,        HTTP_HEAD )
+  LROT_NUMENTRY( PUT,         HTTP_PUT )
+  LROT_NUMENTRY( POST,        HTTP_POST )
+  LROT_NUMENTRY( DELETE,      HTTP_DELETE )
+
+  LROT_NUMENTRY( INDEX_NONE,  INDEX_NONE )
+  LROT_NUMENTRY( INDEX_ROOT,  INDEX_ROOT )
+  LROT_NUMENTRY( INDEX_ALL,   INDEX_ALL  )
+LROT_END(httpd, NULL, 0)
+
+
+static int lhttpd_init(lua_State *L)
+{
+  dynamic_task = task_get_id(dynamic_handler_lvm);
+  queue = xQueueCreate(1, sizeof(thread_request_t));
+  done = xSemaphoreCreateBinary();
+
+  luaL_rometatable(L, REQUEST_METATABLE, LROT_TABLEREF(httpd_req_mt));
+
+  return 0;
+}
+
+NODEMCU_MODULE(HTTPD, "httpd", httpd, lhttpd_init);

docs/modules/eromfs.md

@@ -0,0 +1,90 @@
+# EROMFS Module
+| Since  | Origin / Contributor  | Maintainer  | Source  |
+| :----- | :-------------------- | :---------- | :------ |
+| 2021-11-13 | [Johny Mattsson](https://github.com/jmattsson) |[Johny Mattsson](https://github.com/jmattsson) | [heaptrace.c](../../components/modules/eromfs.c)|
+
+EROMFS (Embedded Read-Only Mountable File Sets) provides a convenient mechanism
+for bundling file sets into the firmware image itself. The main use cases
+envisaged for this is static web site content and default "skeleton" files
+that may be used to populate SPIFFS on first boot.
+
+When enabling the `eromfs` module one or more file sets ("volumes") must be
+declared. Each such volume is identified by name, and may be mounted anywhere
+supported by the [Virtual File System](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/vfs.html). Once mounted, the included
+files are available on a read-only basis to any thread wanting to access them.
+
+Note that EROMFS does not support directories per se, but will store the
+directory path as part of the filename just as SPIFFS does. As such it is
+only possible to list the root of the volume, not subdirectories (since
+they don't exist).
+
+## eromfs.list
+Returns a list of the bundled file sets (volumes).
+
+#### Syntax
+```lua
+eromfs.list()
+```
+
+#### Parameters
+None.
+
+#### Returns
+An array with the names of the bundled volumes.
+
+#### Example
+```lua
+for _, volname in ipairs(eromfs.list()) do print(volname) end
+```
+
+## eromfs.mount
+Mounts a volume at a specified point in the virtual file system.
+
+Note that it is technically possible to mount a volume multiple times on
+different mount points. The benefit of doing so however is questionable.
+
+#### Syntax
+```lua
+eromfs.mount(volname, mountpt)
+```
+
+#### Parameters
+- `volname` the name of the volume to mount, e.g. `myvol`.
+- `mountpt` where to mount said volume. Must start with '/', e.g. `/myvol`.
+
+#### Returns
+`nil` on success. Raises an error if the named volume cannot be found, or
+cannot be mounted.
+
+#### Example
+```lua
+-- Assumes the volume named "myvol" exists
+eromfs.mount('myvol', '/somewhere')
+for name,size in pairs(file.list('/somewhere')) do print(name, size) end
+```
+
+## eromfs.unmount
+Unmounts the specified EROMFS volume from the given mount point.
+
+#### Syntax
+```lua
+eromfs.unmount(volname, mountpt)
+```
+
+#### Parameters
+- `volname` the name of the volume to mount.
+- `mountpt` the current mount point of the volume.
+
+#### Returns
+`nil` if:
+- the volume was successfully unmounted; or
+- the volume was not currently mounted at the given mount point
+
+Raises an error if:
+- the unmounting fails for some reason; or
+- a different EROMFS volume is mounted on the given mount point
+
+#### Example
+```lua
+eromfs.unmount('myvol', '/somewhere')
+```

docs/modules/httpd.md

@@ -0,0 +1,286 @@
+# LEDC Module
+| Since  | Origin / Contributor  | Maintainer  | Source  |
+| :----- | :-------------------- | :---------- | :------ |
+| 2021-11-07 | [Johny Mattsson](https://github.com/jmattsson) | [Johny Mattsson](https://github.com/jmattsson) | [httpd.c](../../components/modules/httpd.c)|
+
+This module provides an interface to Espressif's [web server component](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/protocols/esp_http_server.html).
+
+
+# HTTPD Overview
+
+The httpd module implements support for both static file serving and dynamic
+content generation. For static files, all files need to reside under a
+common prefix (the "webroot") in the (virtual) filesystem. The module does
+not care whether the underlying file system supports directories or not,
+so files may be served from SPIFFS, FAT filesystems, or whatever else
+may be mounted. If you wish to include the static website contents within
+the firmware image itself, considering using the [EROMFS](eromfs.md) module.
+
+Unlike the default behaviour of the Espressif web server, this module serves
+static files based on file extensions primarily. Static routes are typically
+defined as a file extension (e.g. \*.html) and the `Content-Type` such files
+should be served as. A number of file extensions are included by default
+and should cover the basic needs:
+
+  - \*.html (text/html)
+  - \*.css (text/css)
+  - \*.js (text/javascript)
+  - \*.json (application/json)
+  - \*.gif (image/gif)
+  - \*.jpg (image/jpeg)
+  - \*.jpeg (image/jpeg)
+  - \*.png (image/png)
+  - \*.svg (image/svg+xml)
+  - \*.ttf (font/ttf)
+
+The native Espressif approach may also be used if you prefer, but is harder
+to work with. Both schemes can coexist in most cases without issues. When
+using the native approach, URI wildcard matching is supported.
+
+Dynamic routes may be registered, which when accessed by a client will result
+in a Lua function being invoked. This function may then generate whatever
+content is applicable, for example obtaining a sensor value and returning it.
+
+Note that if you are writing sensor data to files and serving those files
+statically you will be susceptible to race conditions where the file contents
+may not be available from the outside. This is due to the web server running
+in its own FreeRTOS thread and serving files directly from that thread
+concurrently with the Lua VM running as usual. It is therefore safer to
+instead serve such content on a dynamic route, even if all that route does
+is reads the file and serves that.
+
+An example of such a setup:
+```lua
+function handler(req)
+  local f = io.open('/path/to/mysensordata.csv', 'r')
+  return {
+    status = "200 OK",
+    type = "text/plain",
+    getbody = function()
+      local data = f:read(512) -- pick a suiteable chunk size here
+      if not data then f:close() end
+      return data
+    end,
+  }
+end
+
+httpd.dynamic(httpd.GET, "/mysensordata", handler)
+```
+
+## httpd.start()
+Starts the web server. The server has to be started before routes can be
+configured.
+
+#### Syntax
+```lua
+httpd.start({
+  webroot = "<static file prefix>",
+  max_handlers = 20,
+  auto_index = httpd.INDEX_NONE || httpd.INDEX_ROOT || httpd.INDEX_ALL,
+})
+```
+
+#### Parameters
+A single configuration table is provided, with the following possible fields:
+
+  - `webroot` (mandatory) This sets the prefix used when serving static files.
+  For example, with `webroot` set to "web", a HTTP request for "/index.html"
+  will result in the httpd module trying to serve the file "web/index.html"
+  from the file system. Do NOT set this to the empty string, as that would
+  provide remote access to your entire virtual file system, including special
+  files such as virtual device files (e.g. "/dev/uart1") which would likely
+  present a serious security issue.
+  - `max_handlers` (optional) Configures the maximum number of route handlers
+  the server will support. Default value is 20, which includes both the
+  standard static file extension handlers and any user-provided handlers.
+  Raising this will result in a bit of additional memory being used. Adjust
+  if and when necessary.
+  - `auto_index` Sets the indexer mode to be used. Most web servers
+  automatically go looking for an "index.html" file when a directory is
+  requested. For example, when pointing your web browser to a web site
+  for the first time, e.g. http://www.example.com/ the actual request will
+  come through for "/", which in turn commonly gets translated to "/index.html"
+  on the server. This behaviour can be enabled in this module as well. There
+  are three modes provided:
+    - `httpd.INDEX_NONE` No automatic translation to "index.html" is provided.
+    - `httpd.INDEX_ROOT` Only the root ("/") is translated to "/index.html".
+    - `httpd.INDEX_ALL` Any path ending with a "/" has "index.html" appended.
+    For example, a request for "subdir/" would become "subdir/index.html",
+    which in turn might result in the file "web/subdir/index.html" being
+    served (if the `webroot` was set to "web").
+    The default value is `httpd.INDEX_ROOT`.
+
+#### Returns
+`nil`
+
+#### Example
+```lua
+httpd.start({ webroot = "web", auto_index = httpd.INDEX_ALL })
+```
+
+## httpd.stop()
+
+Stops the web server. All registered route handlers are removed.
+
+#### Syntax
+```lua
+httpd.stop()
+```
+
+#### Parameters
+None.
+
+#### Returns
+`nil`
+
+
+## httpd.static()
+
+Registers a static route handler.
+
+#### Syntax
+```
+httpd.static(route, content_type)
+```
+
+#### Parameters
+- `route` The route prefix. Typically in the form of \*.ext to serve all files
+with the ".ext" extension statically. Refer to the Espressif [documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/protocols/esp_http_server.html)
+if you wish to use the native Espressif style of static routes instead.
+- `content_type` The value to send in the `Content-Type` header for this file
+type.
+
+#### Returns
+An error code on failure, or `nil` on success. The error code is the value
+returned from the `httpd_register_uri_handler()` function.
+
+#### Example
+```lua
+httpd.start({ webroot = "web" })
+httpd.static("*.csv", "text/csv") -- Serve CSV files under web/
+```
+
+## httpd.dynamic()
+
+Registers a dynamic route handler.
+
+#### Syntax
+```lua
+httpd.dynamic(method, route, handler)
+```
+
+#### Parameters
+- `method` The HTTP method this route applies to. One of:
+  - `httpd.GET`
+  - `httpd.HEAD`
+  - `httpd.PUT`
+  - `httpd.POST`
+  - `httpd.DELETE`
+- `route` The route prefix. Be mindful of any trailing "/" as that may interact
+with the `auto_index` functionality.
+- `handler` The route handler function - `handler(req)`. The provided request
+object `req` has the following fields/functions:
+  - `method` The request method. Same as the `method` parameter above. If the
+  same function is registered for several methods, this field can be used to
+  determine the method the request used.
+  - `uri` The requested URI. Includes both path and query string (if
+  applicable).
+  - `query` The query string on its own. Not decoded.
+  - `headers` A table-like object in which request headers may be looked up.
+  Note that due to the Espressif API not providing a way to iterate over all
+  headers this table will appear empty if fed to `pairs()`.
+  - `getbody()` A function which may be called to read in the request body
+  incrementally. The size of each chunk is set via the Kconfig option
+  "Receive body chunk size". When this function returns `nil` the end of
+  the body has been reached. May raise an error if reading the body fails
+  for some reason (e.g. timeout, network error).
+
+Note that the provided `req` object is _only valid_ within the scope of this
+single invocation of the handler. Attempts to store away the request and use
+it later _will_ fail.
+
+#### Returns
+A table with the response data to send to the requesting client:
+```lua
+{
+  status = "200 OK",
+  type = "text/plain",
+  headers = {
+    ['X-Extra'] = "My custom header value"
+  },
+  body = "Hello, Lua!",
+  getbody = dynamic_content_generator_func,
+}
+```
+Supported fields:
+- `status` The status code and string to send. If not included "200 OK" is used.
+Other common strings would be "404 Not Found", "400 Bad Request" and everybody's
+favourite "500 Internal Server Error".
+- `type` The value for the `Content-Type` header. The Espressif web server
+component handles this header specially, which is why it's provided here and
+not within the `headers` table.
+- `body` The full content body to send.
+- `getbody` A function to source the body content from, similar to the way
+the request body is read in. This function will be called repeatedly and the
+returned string from each invocation will be sent as a chunk to the client.
+Once this function returns `nil` the body is deemed to be complete and no
+further calls to the function will be made. It is guaranteed that the
+function will be called until it returns `nil` even if the sending of the
+content encounters an error. This ensures that any resource cleanup
+necessary will still take place in such circumstances (e.g. file closing).
+
+Only one of `body` and `getbody` should be specified.
+
+#### Example
+```lua
+httpd.start({ webroot = "web" })
+
+function put_foo(req)
+  local body_len = tonumber(req.headers['content-length']) or 0
+  if body_len < 4096
+  then
+    local f = io.open("/upload/foo.txt", "w")
+    local body = req.getbody()
+    while body
+    do
+      f:write(body)
+      body = req.getbody()
+    end
+    f:close()
+    return { status = "201 Created" }
+  else
+    return { status = "400 Bad Request" }
+  end
+end
+
+httpd.dynamic(httpd.PUT, "/foo", put_foo)
+```
+
+## httpd.unregister()
+
+Unregisters a previously registered handler. The default handlers may be
+unregistered.
+
+#### Syntax
+```lua
+httpd.unregister(method, route)
+```
+
+#### Parameters
+- `method` The method the route was registered for. One of:
+  - `httpd.GET`
+  - `httpd.HEAD`
+  - `httpd.PUT`
+  - `httpd.POST`
+  - `httpd.DELETE`
+- `route` The route prefix.
+
+#### Returns
+`1` on success, `nil` on failure (including if the route was not registered).
+
+#### Example
+Unregistering one of the default static handlers:
+```lua
+httpd.start({ webroot = "web" })
+httpd.unregister(httpd.GET, "*.jpeg")
+```

mkdocs.yml

@@ -53,6 +53,7 @@ pages:
       - 'gpio': 'modules/gpio.md'
       - 'heaptrace': 'modules/heaptrace.md'
       - 'http': 'modules/http.md'
+      - 'httpd': 'modules/httpd.md'
       - 'i2c': 'modules/i2c.md'
       - 'i2s': 'modules/i2s.md'
       - 'ledc': 'modules/ledc.md'

sdkconfig.defaults

@@ -27,7 +27,7 @@ CONFIG_LWIP_SO_REUSE=y
 
 # Decrease the duration of sockets in TIME_WAIT
 # see https://github.com/nodemcu/nodemcu-firmware/issues/1836
-CONFIG_TCP_MSL=5000
+CONFIG_LWIP_TCP_MSL=5000
 
 # Disable esp-idf's bluetooth component by default.
 # The bthci module is also disabled and will enable bt when selected