From 670bf2d6a6601c24ce54fc373e656f3a653901de Mon Sep 17 00:00:00 2001 From: Johny Mattsson Date: Wed, 26 Jan 2022 11:32:24 +1100 Subject: [PATCH 1/4] Make UART buffer sizes configurable. (#3500) Also corrects assumption about number of UARTs available. --- components/platform/Kconfig | 78 +++++++++++++++++++++++++ components/platform/include/cpu_esp32.h | 2 +- components/platform/include/platform.h | 2 - components/platform/platform.c | 20 ++++++- 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/components/platform/Kconfig b/components/platform/Kconfig index 66848cbc..bb162bfc 100644 --- a/components/platform/Kconfig +++ b/components/platform/Kconfig @@ -64,4 +64,82 @@ menu "NodeMCU platform config" Embedded LUA Flash Store size. Set to zero to use an LFS partition instead of embedding the LFS within the NodeMCU firmware itself. + menu "UART buffer size configuration" + + config NODEMCU_UART_AT_LEAST_2 + bool + default y if IDF_TARGET_ESP32 + default y if IDF_TARGET_ESP32S2 + default y if IDF_TARGET_ESP32S3 + default y if IDF_TARGET_ESP32C3 + default y if IDF_TARGET_ESP32H2 + + config NODEMCU_UART_AT_LEAST_3 + bool + default y if IDF_TARGET_ESP32 + default y if IDF_TARGET_ESP32S3 + + config NODEMCU_UART_DRIVER_BUF_SIZE_RX0 + int "RX buffer size for UART0" + default 512 + help + The rx buffer size to use for UART0. + This is a buffer used to/from the UART ISR. For high-speed scenarios, + it can be very helpful to increase the rx size. The ISR trying + to write to a full rx buffer results in lost data and an error + event posted. + + config NODEMCU_UART_DRIVER_BUF_SIZE_TX0 + int "TX buffer size for UART0" + default 512 + help + The tx buffer size to use for UART0. + This is a buffer used to/from the UART ISR. In some scenarios it + can be helpful to increase the tx size. Writing to a full tx buffer + blocks the calling RTOS task. + + config NODEMCU_UART_DRIVER_BUF_SIZE_RX1 + int "RX buffer size for UART1" + depends on NODEMCU_UART_AT_LEAST_2 + default 512 + help + The rx buffer size to use for UART1. + This is a buffer used to/from the UART ISR. For high-speed scenarios, + it can be very helpful to increase the rx size. The ISR trying + to write to a full rx buffer results in lost data and an error + event posted. + + config NODEMCU_UART_DRIVER_BUF_SIZE_TX1 + int "TX buffer size for UART1" + depends on NODEMCU_UART_AT_LEAST_2 + default 512 + help + The tx buffer size to use for UART1. + This is a buffer used to/from the UART ISR. In some scenarios it + can be helpful to increase the tx size. Writing to a full tx buffer + blocks the calling RTOS task. + + config NODEMCU_UART_DRIVER_BUF_SIZE_RX2 + int "RX buffer size for UART2" + depends on NODEMCU_UART_AT_LEAST_3 + default 512 + help + The rx buffer size to use for UART2. + This is a buffer used to/from the UART ISR. For high-speed scenarios, + it can be very helpful to increase the rx size. The ISR trying + to write to a full rx buffer results in lost data and an error + event posted. + + config NODEMCU_UART_DRIVER_BUF_SIZE_TX2 + int "TX buffer size for UART2" + depends on NODEMCU_UART_AT_LEAST_3 + default 512 + help + The tx buffer size to use for UART2. + This is a buffer used to/from the UART ISR. In some scenarios it + can be helpful to increase the tx size. Writing to a full tx buffer + blocks the calling RTOS task. + + endmenu + endmenu diff --git a/components/platform/include/cpu_esp32.h b/components/platform/include/cpu_esp32.h index 2f5e42eb..f8ae6968 100644 --- a/components/platform/include/cpu_esp32.h +++ b/components/platform/include/cpu_esp32.h @@ -4,7 +4,7 @@ #include "sdkconfig.h" #include "esp_spi_flash.h" -#define NUM_UART 3 +#define NUM_UART SOC_UART_NUM #define INTERNAL_FLASH_SECTOR_SIZE SPI_FLASH_SEC_SIZE #define INTERNAL_FLASH_WRITE_UNIT_SIZE 4 diff --git a/components/platform/include/platform.h b/components/platform/include/platform.h index 5fef74cd..786e9ab5 100644 --- a/components/platform/include/platform.h +++ b/components/platform/include/platform.h @@ -48,8 +48,6 @@ int platform_gpio_output_exists( unsigned gpio ); // ***************************************************************************** // UART subsection -#define UART_BUFFER_SIZE 512 - enum { PLATFORM_UART_MODE_UART = 0x0, diff --git a/components/platform/platform.c b/components/platform/platform.c index 6198b2a7..59228b82 100644 --- a/components/platform/platform.c +++ b/components/platform/platform.c @@ -34,6 +34,24 @@ int platform_gpio_output_exists( unsigned gpio ) { return GPIO_IS_VALID_OUTPUT_G #define PLATFORM_UART_EVENT_RX (UART_EVENT_MAX + 3) #define PLATFORM_UART_EVENT_BREAK (UART_EVENT_MAX + 4) +typedef struct { + unsigned rx_buf_sz; + unsigned tx_buf_sz; +} uart_buf_sz_cfg_t; + +static const uart_buf_sz_cfg_t uart_buf_sz_cfg[] = { +{ .rx_buf_sz = CONFIG_NODEMCU_UART_DRIVER_BUF_SIZE_RX0 +0, + .tx_buf_sz = CONFIG_NODEMCU_UART_DRIVER_BUF_SIZE_TX0 +0 }, +#if NUM_UART > 1 +{ .rx_buf_sz = CONFIG_NODEMCU_UART_DRIVER_BUF_SIZE_RX1 +0, + .tx_buf_sz = CONFIG_NODEMCU_UART_DRIVER_BUF_SIZE_TX1 +0 }, +#endif +#if NUM_UART > 2 +{ .rx_buf_sz = CONFIG_NODEMCU_UART_DRIVER_BUF_SIZE_RX2 +0, + .tx_buf_sz = CONFIG_NODEMCU_UART_DRIVER_BUF_SIZE_TX2 +0 }, +#endif +}; + typedef struct { unsigned id; int type; @@ -295,7 +313,7 @@ int platform_uart_start( unsigned id ) uart_status_t *us = & uart_status[id]; - esp_err_t ret = uart_driver_install(id, UART_BUFFER_SIZE, UART_BUFFER_SIZE, 3, & us->queue, 0); + esp_err_t ret = uart_driver_install(id, uart_buf_sz_cfg[id].rx_buf_sz, uart_buf_sz_cfg[id].tx_buf_sz, 3, & us->queue, 0); if(ret != ESP_OK) { return -1; } From 8f9295b09e909d44375bab9fda2f0c302b071962 Mon Sep 17 00:00:00 2001 From: Philip Gladstone Date: Tue, 25 Jan 2022 20:54:17 -0500 Subject: [PATCH 2/4] Update to the released version of idf4.4 (#3498) * Update to the released version of idf4.4 * Try to get the CI Build to work in all cases --- components/modules/node.c | 9 ++++++--- components/modules/pulsecnt.c | 11 ++++++----- sdk/esp32-esp-idf | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/components/modules/node.c b/components/modules/node.c index ba4d76cc..11b19077 100644 --- a/components/modules/node.c +++ b/components/modules/node.c @@ -106,14 +106,17 @@ static int node_bootreason( lua_State *L) case EXT_CPU_RESET: #endif case DEEPSLEEP_RESET: -#if defined(CONFIG_IDF_TARGET_ESP32) || defined(CONFIG_IDF_TARGET_ESP32C3) +#if defined(CONFIG_IDF_TARGET_ESP32) case SDIO_RESET: #endif -#if defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32S3) +#if defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32S3) || defined(CONFIG_IDF_TARGET_ESP32C3) case GLITCH_RTC_RESET: #endif -#if defined(CONFIG_IDF_TARGET_ESP32S3) +#if defined(CONFIG_IDF_TARGET_ESP32S3) || defined(CONFIG_IDF_TARGET_ESP32C3) case EFUSE_RESET: + case USB_UART_CHIP_RESET: + case USB_JTAG_CHIP_RESET: + case POWER_GLITCH_RESET: #endif case TG0WDT_SYS_RESET: case TG1WDT_SYS_RESET: diff --git a/components/modules/pulsecnt.c b/components/modules/pulsecnt.c index 99d4b0e1..4f4df34b 100644 --- a/components/modules/pulsecnt.c +++ b/components/modules/pulsecnt.c @@ -19,6 +19,7 @@ CONDITIONS OF ANY KIND, either express or implied. #include "platform.h" #include "task/task.h" #include "driver/pcnt.h" +#include "soc/pcnt_struct.h" #include "esp_log.h" #include "lextra.h" @@ -135,17 +136,17 @@ static void pulsecnt_task(task_param_t param, task_prio_t prio) /*0: positive value to zero; 1: negative value to zero; 2: counter value negative ; 3: counter value positive*/ // uint8_t moving_to = status & 0x00000003u; // get first two bits - if (status & PCNT_STATUS_THRES1_M) { + if (status & PCNT_EVT_THRES_1) { // printf("THRES1 EVT\n"); thr1 = true; // evt_count = pc->thresh1; } - if (status & PCNT_STATUS_THRES0_M) { + if (status & PCNT_EVT_THRES_0) { // printf("THRES0 EVT\n"); thr0 = true; // evt_count = pc->thresh0; } - if (status & PCNT_STATUS_L_LIM_M) { + if (status & PCNT_EVT_L_LIM) { // printf("L_LIM EVT\n"); l_lim = true; /* @@ -164,7 +165,7 @@ static void pulsecnt_task(task_param_t param, task_prio_t prio) } */ } - if (status & PCNT_STATUS_H_LIM_M) { + if (status & PCNT_EVT_H_LIM) { // printf("H_LIM EVT\n"); h_lim = true; /* @@ -183,7 +184,7 @@ static void pulsecnt_task(task_param_t param, task_prio_t prio) } */ } - if (status & PCNT_STATUS_ZERO_M) { + if (status & PCNT_EVT_ZERO) { // printf("ZERO EVT\n"); zero = true; // evt_count = 0; diff --git a/sdk/esp32-esp-idf b/sdk/esp32-esp-idf index a20df743..d83021a6 160000 --- a/sdk/esp32-esp-idf +++ b/sdk/esp32-esp-idf @@ -1 +1 @@ -Subproject commit a20df743f1c51e6d65b021ed2ffd3081a2feec64 +Subproject commit d83021a6e8550b4d462e11d61aaab0214dc03f5a From e5892a728640020011da1554bdf66fc1662cc5c1 Mon Sep 17 00:00:00 2001 From: Philip Gladstone Date: Thu, 27 Jan 2022 01:02:56 -0500 Subject: [PATCH 3/4] Give the SPI module a chance of working... (#3496) --- components/modules/spi_master.c | 15 +++++++++++++++ docs/modules/spi.md | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/components/modules/spi_master.c b/components/modules/spi_master.c index 39c80478..e3b761ee 100644 --- a/components/modules/spi_master.c +++ b/components/modules/spi_master.c @@ -6,6 +6,7 @@ #include "lextra.h" #include "driver/spi_master.h" +#include "driver/gpio.h" #include "esp_heap_caps.h" #include "esp_log.h" @@ -107,6 +108,7 @@ static int lspi_device_transfer( lua_State *L ) // lua_getfield( L, stack, "mode" ); trans.flags |= options_flags[ luaL_checkoption( L, -1, options[0], options ) ]; + lua_pop(L, 1); // data_len = 0; data = opt_checklstring( L, "txdata", "", &data_len ); @@ -296,6 +298,19 @@ static int lspi_host_device( lua_State *L ) CONFIG_DEVICE_FROM_BOOL_FIELD(halfduplex, SPI_DEVICE_HALFDUPLEX); CONFIG_DEVICE_FROM_BOOL_FIELD(clk_as_cs, SPI_DEVICE_CLK_AS_CS); lua_settop( L, stack ); + + // + if (config.spics_io_num >= 0) { + // Configure the gpio + gpio_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.mode = GPIO_MODE_OUTPUT; + cfg.pin_bit_mask = 1ULL << config.spics_io_num; + if (!no_err(gpio_config (&cfg)) || + !no_err(gpio_set_level (config.spics_io_num, (config.flags & SPI_DEVICE_POSITIVE_CS) ? 0 : 1))) { + return luaL_error(L, "failed to configure gpio"); + } + } // // fill remaining config entries config.queue_size = 1; diff --git a/docs/modules/spi.md b/docs/modules/spi.md index c3abe44f..caa2fd92 100644 --- a/docs/modules/spi.md +++ b/docs/modules/spi.md @@ -147,8 +147,8 @@ device:transfer(txdata) #### Parameters `trans` table containing the elements of the transaction: -- `command` data for command phase, amount of bits was defined during device creation, optional -- `address` data for address phase, amount of bits was defined during device creation, optional +- `cmd` data for command phase, amount of bits was defined during device creation, optional +- `addr` data for address phase, amount of bits was defined during device creation, optional - `txdata` string of data to be sent to the device, optional - `rxlen` number of bytes to be received, optional - `mode` optional, one of From cb434811ca668f1ee919ac45d9a251abf2145fe5 Mon Sep 17 00:00:00 2001 From: Johny Mattsson Date: Sat, 5 Mar 2022 13:51:54 +1100 Subject: [PATCH 4/4] IDF web server module (#3502) * Added httpd module. Lua-interface to the standard esp_http_server component. * Added eromfs module. --- components/modules/CMakeLists.txt | 17 + components/modules/Kconfig | 54 +++ components/modules/eromfs.c | 379 +++++++++++++++ components/modules/eromfs.py | 80 ++++ components/modules/httpd.c | 754 ++++++++++++++++++++++++++++++ docs/modules/eromfs.md | 90 ++++ docs/modules/httpd.md | 286 ++++++++++++ mkdocs.yml | 1 + sdkconfig.defaults | 2 +- 9 files changed, 1662 insertions(+), 1 deletion(-) create mode 100644 components/modules/eromfs.c create mode 100755 components/modules/eromfs.py create mode 100644 components/modules/httpd.c create mode 100644 docs/modules/eromfs.md create mode 100644 docs/modules/httpd.md diff --git a/components/modules/CMakeLists.txt b/components/modules/CMakeLists.txt index 6b73b8df..e1f68d5d 100644 --- a/components/modules/CMakeLists.txt +++ b/components/modules/CMakeLists.txt @@ -7,10 +7,12 @@ set(module_srcs "crypto.c" "dht.c" "encoder.c" + "eromfs.c" "file.c" "gpio.c" "heaptrace.c" "http.c" + "httpd.c" "i2c.c" "i2c_hw_master.c" "i2c_hw_slave.c" @@ -74,6 +76,7 @@ idf_component_register( "driver_can" "esp_http_client" "fatfs" + "esp_http_server" "libsodium" "lua" "mbedtls" @@ -138,3 +141,17 @@ set_property( DIRECTORY "${COMPONENT_DIR}" APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ucg_config.h u8g2_fonts.h u8g2_displays.h ) + +# eromfs generation +add_custom_command( + OUTPUT eromfs.bin + COMMAND ${COMPONENT_DIR}/eromfs.py ${CONFIG_NODEMCU_CMODULE_EROMFS_VOLUMES} + DEPENDS ${SDKCONFIG_HEADER} +) +add_custom_target(eromfs_bin DEPENDS eromfs.bin) +target_add_binary_data(${COMPONENT_LIB} "${CMAKE_CURRENT_BINARY_DIR}/eromfs.bin" BINARY DEPENDS eromfs_bin) +set_property( + DIRECTORY "${COMPONENT_DIR}" APPEND + PROPERTY ADDITIONAL_MAKE_CLEAN_FILES eromfs.bin +) + diff --git a/components/modules/Kconfig b/components/modules/Kconfig index 4d92ad8c..b07b6d0e 100644 --- a/components/modules/Kconfig +++ b/components/modules/Kconfig @@ -54,6 +54,34 @@ menu "NodeMCU modules" Includes the encoder module. This provides hex and base64 encoding and decoding functionality. + config NODEMCU_CMODULE_EROMFS + bool "Eromfs module (embedded read-only mountable file sets)" + select VFS_SUPPORT_IO + default "n" + help + Includes the eromfs module, giving access to the embedded mountable + file sets (volumes) configured here. Useful for bundling file sets + within the main firmware image, such as website contents. + + config NODEMCU_CMODULE_EROMFS_VOLUMES + depends on NODEMCU_CMODULE_EROMFS + string "File sets to embed" + default "volume_name=/path/to/volume_root;myvol2=../relpath" + help + List one or more volume definitions in the form of + VolumeName=/path/to/files where the VolumeName is the identifier + by which the eromfs module will refer to the volume. The path + may be given as either a relative or absolute path. If relative, + it is relative to the top-level nodemcu-firmware directory. + All files and directories within the specified volume root will + be included. Symlinks are not supported and will result in + failure if encountered. Multiple volumes may be declared by + separating the entries with a semicolon. + + Note that eromfs does not support directories per se, but will + store the directory path as part of the filename just as SPIFFS + does. + config NODEMCU_CMODULE_ETH depends on IDF_TARGET_ESP32 select ETH_USE_ESP32_EMAC @@ -91,6 +119,32 @@ menu "NodeMCU modules" help Includes the HTTP module (recommended). + config NODEMCU_CMODULE_HTTPD + bool "Httpd (web server) module" + default "n" + help + Includes the HTTPD module. This module uses the regular IDF + http server component internally. + + config NODEMCU_CMODULE_HTTPD_MAX_RESPONSE_HEADERS + int "Max response header fields" if NODEMCU_CMODULE_HTTPD + default 5 + help + Determines how much space to allocate for header fields in the + HTTP response. This value does not include header fields the + http server itself generates internally, but only headers + explicitly returned in a dynamic route handler. Typically only + Content-Type is needed, so for most applications the default + value here will suffice. + + config NODEMCU_CMODULE_HTTPD_RECV_BODY_CHUNK_SIZE + int "Receive body chunk size" if NODEMCU_CMODULE_HTTPD + default 1024 + help + When receiving a body payload, receive at most this many + bytes at a time. Higher values means reduced overhead at + the cost of higher memory load. + config NODEMCU_CMODULE_I2C bool "I2C module" default "y" diff --git a/components/modules/eromfs.c b/components/modules/eromfs.c new file mode 100644 index 00000000..bd296c00 --- /dev/null +++ b/components/modules/eromfs.c @@ -0,0 +1,379 @@ +#include "module.h" +#include "lauxlib.h" + +#include "esp_vfs.h" +#include +#include + +#include +#include + +/** + * The logical layout of the embedded volumes is + * [ volume record ] + * [ volume record ] + * ... + * [ index record in vol1 ] + * [ index record in vol1 ] + * ... + * [ file contents in vol1 ] + * [ file contents in vol1 ] + * ... + * [ index record in vol2 ] + * [ index record in vol2 ] + * ... + * [ file contents in vol2 ] + * [ file contents in vol2 ] + * + * Both the volume records and index records are variable length so as to not + * waste space in their name fields. Finding the start of the index records + * for a volume is by reading the offs(et) field in the volume record and + * jumping that many bytes forward from the start of the eromfs.bin data. + * Similarly, finding the file contents is by reading the index record's + * offs(et) field and basing that off the start of the volume index. + * Naturally, the start of the volume index is the same as the end of the + * volume header, and the start of the file contents is the same as the + * end of the volume index, and either of those can be worked out by + * reading the offs(et) field in the first record. + */ + +#pragma pack(push, 1) +typedef struct { + uint8_t rec_len; + uint16_t offs; // index_offs + char name[]; +} volume_record_t; + +typedef struct { + uint8_t rec_len; + uint32_t offs; // based off index_offs + uint32_t len; // file_len + char name[]; +} index_record_t; +#pragma pack(pop) + + +typedef struct { + const index_record_t *meta; + const char *data; // start of data + off_t pos; +} file_descriptor_t; + + +typedef struct { + DIR opaque; + const index_record_t *index; + const index_record_t *pos; +} eromfs_DIR_t; + + +extern const char eromfs_bin_start[] asm("_binary_eromfs_bin_start"); + +// Both the volume header and the file set indices end where the next +// type of data block commences (file set index, file contents). +#define end_of(type, start) ((const type *)(((char *)start) + start->offs)) + +#define eromfs_header_start ((const volume_record_t *)eromfs_bin_start) +#define eromfs_header_end end_of(volume_record_t, eromfs_header_start) + + +/* The logic for finding a volume record by name is the same as finding a + * file record by name, only the data structure type varies. Hence we + * hide the casting and variable length record stepping behind a convenience + * macro here. + */ +#define find_entry_by_name(out, xname, start_void_p, record_t) \ + do { \ + const record_t *entry_ = (const record_t *)(start_void_p); \ + const record_t *end_ = end_of(record_t, entry_); \ + unsigned xname_len = strlen(xname); \ + for (; entry_ < end_; \ + entry_ = (const record_t *)(((char *)entry_) + entry_->rec_len)) \ + { \ + uint8_t name_len = entry_->rec_len - sizeof(record_t); \ + if (xname_len == name_len && \ + strncmp(xname, entry_->name, name_len) == 0) \ + { \ + out = entry_; \ + break; \ + } \ + } \ + } while(0); + + +static int mounted_volumes = LUA_NOREF; + + +static SemaphoreHandle_t fd_mutex; + +static file_descriptor_t fds[CONFIG_NODEMCU_MAX_OPEN_FILES]; + + +// --- VFS interface ----------------------------------------------------- + + +#define get_index() const index_record_t *index = (const index_record_t *)ctx + +static const index_record_t *path2entry(void *ctx, const char *path) +{ + while (*path == '/') + ++path; + + get_index(); + const index_record_t *entry = NULL; + find_entry_by_name(entry, path, index, index_record_t); + return entry; +} + + +static int eromfs_fstat(void *ctx, int fd, struct stat *st) +{ + memset(st, 0, sizeof(struct stat)); + st->st_size = fds[fd].meta->len; + st->st_blocks = (fds[fd].meta->len + 511)/512; + return 0; +} + + +#ifdef CONFIG_VFS_SUPPORT_DIR +static int eromfs_stat(void *ctx, const char *path, struct stat *st) +{ + const index_record_t *entry = path2entry(ctx, path); + if (!entry) + return -ENOENT; + memset(st, 0, sizeof(struct stat)); + st->st_size = entry->len; + st->st_blocks = (entry->len + 511)/512; + return 0; +} + + +static DIR *eromfs_opendir(void *ctx, const char *path) +{ + if (strcmp(path, "/") != 0) + return NULL; + + get_index(); + eromfs_DIR_t *dir = calloc(1, sizeof(eromfs_DIR_t)); + dir->index = index; + dir->pos = index; + return (DIR *)dir; +} + + +static struct dirent *eromfs_readdir(void *ctx, DIR *pdir) +{ + UNUSED(ctx); + eromfs_DIR_t *dir = (eromfs_DIR_t *)pdir; + + const index_record_t *end = end_of(index_record_t, dir->index); + if (dir->pos >= end) + return NULL; + + static struct dirent de = { + .d_ino = 0, + .d_type = DT_REG, + }; + size_t max_len = sizeof(de.d_name); + size_t len = dir->pos->rec_len - sizeof(index_record_t); + if (len > max_len -1) + len = max_len - 1; + strncpy(de.d_name, dir->pos->name, len); + de.d_name[len] = 0; + dir->pos = (const index_record_t *)((char *)dir->pos + dir->pos->rec_len); + return &de; +} + + +static int eromfs_closedir(void *ctx, DIR *dir) +{ + UNUSED(ctx); + free(dir); + return 0; +} +#endif + + +static int eromfs_open(void *ctx, const char *path, int flags, int mode) +{ + UNUSED(flags); + UNUSED(mode); + const index_record_t *entry = path2entry(ctx, path); + if (!entry) + return -ENOENT; + + xSemaphoreTake(fd_mutex, portMAX_DELAY); + int fd = -ENFILE; + // max open files is guaranteed to be small; linear search is fine + for (unsigned i = 0; i < CONFIG_NODEMCU_MAX_OPEN_FILES; ++i) + { + if (fds[i].meta == NULL) + { + fds[i].meta = entry; + fds[i].data = (const char *)ctx + entry->offs; + fds[i].pos = 0; + fd = (int)i; + } + } + xSemaphoreGive(fd_mutex); + + return fd; +} + + +static ssize_t eromfs_read(void *ctx, int fd, void *dst, size_t size) +{ + UNUSED(ctx); + size_t avail = fds[fd].meta->len - fds[fd].pos; + if (size > avail) + size = avail; + const char *src = fds[fd].data + fds[fd].pos; + memcpy(dst, src, size); + fds[fd].pos += size; + return size; +} + + +static off_t eromfs_lseek(void *ctx, int fd, off_t size, int mode) +{ + UNUSED(ctx); + off_t pos = fds[fd].pos; + switch(mode) + { + case SEEK_SET: pos = size; break; + case SEEK_CUR: pos += size; break; + case SEEK_END: pos = fds[fd].meta->len + size; break; + default: + return -EINVAL; + } + if (pos < 0 || pos > fds[fd].meta->len) + return -EINVAL; + fds[fd].pos = pos; + return pos; +} + + +static int eromfs_close(void *ctx, int fd) +{ + UNUSED(ctx); + xSemaphoreTake(fd_mutex, portMAX_DELAY); + fds[fd].meta = NULL; + fds[fd].data = NULL; + fds[fd].pos = 0; + xSemaphoreGive(fd_mutex); + return 0; +} + + +// --- Lua interface ----------------------------------------------------- + +static int leromfs_list(lua_State *L) +{ + lua_newtable(L); + int t = lua_gettop(L); + // If this logic looks similar to the find_entry_by_name() macro, it's + // because it is :) Except we're capturing all the volume names, so no + // easy reuse. + const volume_record_t *vol = eromfs_header_start; + const volume_record_t *end = eromfs_header_end; + for (; vol < end; vol = (const volume_record_t *)((char *)vol + vol->rec_len)) + { + uint8_t volume_name_len = vol->rec_len - sizeof(volume_record_t); + lua_pushlstring(L, vol->name, volume_name_len); + lua_rawseti(L, t, lua_objlen(L, t) + 1); + } + return 1; +} + + +static int leromfs_mount(lua_State *L) +{ + const char *name = luaL_checkstring(L, 1); + const char *mountpt = luaL_checkstring(L, 2); + lua_settop(L, 2); + + const volume_record_t *vol = NULL; + find_entry_by_name(vol, name, eromfs_bin_start, volume_record_t); + if (!vol) + return luaL_error(L, "volume %s not found", name); + + const index_record_t *index_start = + (const index_record_t *)(eromfs_bin_start + vol->offs); + + esp_vfs_t eromfs = { + .flags = ESP_VFS_FLAG_CONTEXT_PTR, + .open_p = eromfs_open, + .fstat_p = eromfs_fstat, + .read_p = eromfs_read, + .lseek_p = eromfs_lseek, + .close_p = eromfs_close, +#ifdef CONFIG_VFS_SUPPORT_DIR + .stat_p = eromfs_stat, + .opendir_p = eromfs_opendir, + .readdir_p = eromfs_readdir, + .closedir_p = eromfs_closedir, +#endif + }; + esp_err_t err = esp_vfs_register(mountpt, &eromfs, (void *)index_start); + if (err != ESP_OK) + return luaL_error(L, "failed to mount eromfs; code %d", err); + + lua_rawgeti(L, LUA_REGISTRYINDEX, mounted_volumes); + lua_pushvalue(L, 2); + lua_pushvalue(L, 1); + lua_rawset(L, -3); // mounted_volumes[mountpt] = name + + return 0; +} + + +static int leromfs_unmount(lua_State *L) +{ + const char *name = luaL_checkstring(L, 1); + const char *mountpt = luaL_checkstring(L, 2); + lua_settop(L, 2); + + lua_rawgeti(L, LUA_REGISTRYINDEX, mounted_volumes); + lua_pushvalue(L, 2); + lua_rawget(L, -2); + if (lua_isstring(L, -1)) + { + const char *mounted_name = lua_tostring(L, -1); + if (strcmp(name, mounted_name) == 0) + { + esp_err_t err = esp_vfs_unregister(mountpt); + if (err != ESP_OK) + return luaL_error(L, "unmounting failed; code %d", err); + lua_pop(L, 1); + lua_pushvalue(L, 2); + lua_pushnil(L); + lua_rawset(L, -3); // mounted_volumes[mountpt] = nil + return 0; + } + else + return luaL_error(L, + "can't umount %s from %s; volume %s is mounted there", + name, mountpt, mounted_name); + } + else + return 0; // already unmounted, not an error +} + + +static int leromfs_init(lua_State *L) +{ + fd_mutex = xSemaphoreCreateMutex(); + + lua_newtable(L); + mounted_volumes = luaL_ref(L, LUA_REGISTRYINDEX); + return 0; +} + + +LROT_BEGIN(eromfs, NULL, 0) + LROT_FUNCENTRY( list, leromfs_list ) + LROT_FUNCENTRY( mount, leromfs_mount ) + LROT_FUNCENTRY( unmount, leromfs_unmount ) +LROT_END(eromfs, NULL, 0) + +NODEMCU_MODULE(EROMFS, "eromfs", eromfs, leromfs_init); diff --git a/components/modules/eromfs.py b/components/modules/eromfs.py new file mode 100755 index 00000000..ad3b79e7 --- /dev/null +++ b/components/modules/eromfs.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import sys +import os +import struct + +# [volume records] +# reclen, index_offs, name +# reclen, index_offs2, name2 +# [file index] +# reclen, offs, file_len, name # offs base index_offs +# reclen, offs, file_len, name2 # offs base index_offs2 +# [file contents] +# rawdata +# rawdata + +vol_names = [] # one entry per volume +volume_indexes = [] # one entry per volume +volume_file_contents = [] # one entry per volume + +for voldef in sys.argv[1:]: + [ name, basedir ] = voldef.split('=') + print(f'==> Packing volume "{name}" from {basedir}') + vol_names.append(name) + # Make relative paths relative to the top nodemcu-firmware dir; this + # script gets executed with build/esp-idf/modules as the current dir + if not os.path.isabs(basedir): + basedir = os.path.join(*['..', '..', '..', basedir]) + if not os.path.isdir(basedir): + raise FileNotFoundError(f'source directory {basedir} not found') + basedir_len = len(basedir) +1 + file_index = b'' + file_data = b'' + offs = 0 + entries = [] + index_size = 0 + for root, subdirs, files in os.walk(basedir): + prefix = ('' if root == basedir else root[basedir_len:] + '/') + for filename in files: + hostrelpath = os.path.join(root, filename) + relpath = prefix + filename + size = os.path.getsize(hostrelpath) + rec_len = 1 + 4 + 4 + len(relpath) # reclen + offs + filelen + name + if rec_len > 255: + raise ValueError(f'excessive path length for {relpath}') + entries.append([ rec_len, offs, size, relpath ]) + offs += size + index_size += rec_len + with open(hostrelpath, mode='rb') as f: + file_data += f.read() + for entry in entries: + [ rec_len, offs, size, relpath ] = entry + print('[', rec_len, index_size + offs, size, relpath, ']') + file_index += \ + struct.pack(' Generating volumes index ({volume_records_len} bytes)') +with open('eromfs.bin', 'wb') as f: + index_offs = volume_records_len + for idx, name in enumerate(vol_names): + rec_len = 1 + 2 + len(name) + index_len = len(volume_indexes[idx]) + data_len = len(volume_file_contents[idx]) + if rec_len > 255: + raise ValueError(f'volume name too long for {name}') + if index_offs > 65535: + raise ValueError('volumes index overflowed; too many volumes') + f.write( + struct.pack(' +#include +#include + +#include +#include + +/** + * NodeMCU module for interfacing with the esp_http_server component. + * Said HTTP server runs in its own thread (RTOS task) separate from the + * LVM thread. As such, running dynamic route handlers in Lua requires a + * certain amount of thread synchronisation in order to work safely. + * + * Effectively, when a dynamic handler is invoked, the functions + * dynamic_handler_httpd() and dynamic_handler_lvm() will be running in + * lockstep. The former kicks off the latter with task_post() of the + * relevant HTTP request information, and then proceeds to servicing + * requests from dynamic_handler_lvm(). + * + * There are three things dynamic_handler_lvm() may request from + * dynamic_handler_httpd(): + * + * - Header values. The esp_http_server component provides no method + * of enumerating the received headers, so this pull approach is + * necssary. + * + * - Body data. In order to facilitate large document bodies the + * body is not read in up front, but is instead requested chunk + * by chunk from the dynamic handler. This allows for streaming + * in e.g. a full OTA image and writing it progressively. + * + * - Sending the response. This includes status message, content type + * and any body data. The body data may either be submitted in a single + * go, or a function to "pull" the body data chunk by chunk may be + * given, in which case chunked encoding is used for the response body + * and the content length needs not be known in advance. + * + * @author Johny Mattsson (johny.mattsson+github@gmail.com) + */ + +// More wieldly names for the Kconfig settings +#define MAX_RESPONSE_HEADERS CONFIG_NODEMCU_CMODULE_HTTPD_MAX_RESPONSE_HEADERS +#define RECV_BODY_CHUNK_SIZE CONFIG_NODEMCU_CMODULE_HTTPD_RECV_BODY_CHUNK_SIZE + +#define REQUEST_METATABLE "httpd.req" + +typedef struct { + const char *key; + const char *value; +} key_value_t; + + +typedef struct { + const char *status_str; // e.g. "200 OK" + key_value_t headers[MAX_RESPONSE_HEADERS]; + const char *content_type; // specially handled in esp_http_server + size_t body_len; + const char *body_data; // may be binary data, hence body_len above +} response_data_t; + + +// Request from the LVM thread back to the httpd thread *during* request +// processing in a dynamic handler. +typedef enum { + GET_HEADER, + READ_BODY_CHUNK, + SEND_RESPONSE, + SEND_PARTIAL_RESPONSE, +} request_type_t; + +typedef struct { + size_t used; + char data[RECV_BODY_CHUNK_SIZE]; +} body_chunk_t; + +typedef struct { + request_type_t request_type; + union { + struct { + const char *name; // owned by LVM thread + char **value; // allocated in httpd thread, free()d in LVM thread + } header; + body_chunk_t **body_chunk; // allocated in httpd thread, free()d in LVM + const response_data_t *response; // owned by LVM thread + }; +} thread_request_t; + + +typedef struct { + const char *key; // dynamic handler lookup key + const char *uri; + const char *query_str; + int method; + size_t body_len; +} request_data_t; + + +typedef struct { + const request_data_t *req_info; + uint32_t guard; +} req_udata_t; + + +typedef enum { INDEX_NONE, INDEX_ROOT, INDEX_ALL } index_mode_t; + + +// Task handle for httpd->LVM thread task posting +static task_handle_t dynamic_task; + +// Single-slot queue for passing requests from LVM->httpd thread. +static QueueHandle_t queue; + +// Semaphore for releasing the LVM thread once the thread_request has been +// processed by the httpd thread. +static SemaphoreHandle_t done; + +// Server instance +static httpd_handle_t server = NULL; + +// Path prefix for static files; allocated in LVM thread, used in httpd thread. +// Needed since currently no way to free user_ctx on unregister_uri_handler() +static char *webroot; + +// Auto-index mode, configured at server start. +static index_mode_t index_mode; + +// Tables for keeping our registered handlers and content type strings +// safe from garbage collection until we want them cleaned up. +static int content_types_table_ref = LUA_NOREF; +static int dynamic_handlers_table_ref = LUA_NOREF; + +// Simple guard against deadlocking by calling gethdr()/getbody() outside +// the dynamic handler flow. +static uint32_t guard = 0; + +// Known static file suffixes and their content type. Automatically registered +// on server start. +static const char *default_suffixes[] = +{ + "*.html\0text/html", + "*.css\0text/css", + "*.js\0text/javascript", + "*.txt\0text/plain", + "*.json\0application/json", + "*.gif\0image/gif", + "*.jpg\0image/jpeg", + "*.jpeg\0image/jpeg", + "*.png\0image/png", + "*.svg\0image/svg+xml", + "*.ttf\0font/ttf", +}; + + +// Everybody's favourite response status +static const char internal_err[] = "500 Internal Server Error"; + +static const response_data_t error_resp = { + .status_str = internal_err, + .content_type = "text/plain", + .body_len = sizeof(internal_err) - 1, + .body_data = internal_err, +}; + +// ---- Runs in httpd task/thread ------------------------------------- + +static bool uri_match_file_suffix_first(const char *uri_template, const char *uri_to_match, size_t match_upto) +{ + if (uri_template[0] == '*') + { + // uri_template in form of "*.sufx" + const char *suffix = uri_template + 1; // skip leading '*' + size_t suffix_len = strlen(suffix); + const char *uri_suffix = uri_to_match + match_upto - suffix_len; + return strncmp(suffix, uri_suffix, suffix_len) == 0; + } + else if (uri_template[0] == '\0') + { + // auto-indexer template + switch(index_mode) + { + case INDEX_NONE: return false; + case INDEX_ROOT: return (match_upto == 1) && (uri_to_match[0] == '/'); + case INDEX_ALL: return uri_to_match[match_upto - 1] == '/'; + default: return false; + } + } + else + return httpd_uri_match_wildcard(uri_template, uri_to_match, match_upto); +} + + +static void serve_file(httpd_req_t *req, const char *fname) +{ + FILE *f = fopen(fname, "r"); + if (f) + { + char *buf = malloc(RECV_BODY_CHUNK_SIZE); + ssize_t n = 0; + while ((n = fread(buf, 1, RECV_BODY_CHUNK_SIZE, f)) > 0) + { + if (httpd_resp_send_chunk(req, buf, n) != ESP_OK) + break; + } + httpd_resp_send_chunk(req, buf, 0); + free(buf); + } + else + httpd_resp_send_404(req); + + fclose(f); +} + + +static esp_err_t static_file_handler(httpd_req_t *req) +{ + httpd_resp_set_type(req, (const char *)req->user_ctx); + + char *fname = NULL; + asprintf(&fname, "%s%s", webroot, req->uri); + serve_file(req, fname); + free(fname); + + return ESP_OK; +}; + + +static esp_err_t auto_index_handler(httpd_req_t *req) +{ + char *fname = NULL; + asprintf(&fname, "%s%.*s/index.html", webroot, strlen(req->uri) -1, req->uri); + serve_file(req, fname); + free(fname); + return ESP_OK; +} + + +static esp_err_t dynamic_handler_httpd(httpd_req_t *req) +{ + size_t query_len = httpd_req_get_url_query_len(req); + char *query = query_len ? malloc(query_len + 1) : NULL; + if (query_len) + httpd_req_get_url_query_str(req, query, query_len + 1); + + request_data_t req_data = { + .key = (const char *)req->user_ctx, + .uri = req->uri, + .query_str = query, + .method = req->method, + .body_len = req->content_len, + }; + // Pass the req info over to the LVM thread + task_post_medium(dynamic_task, (task_param_t)&req_data); + + size_t remaining_len = req->content_len; + bool errored = false; + thread_request_t tr; + do { + // Block the httpd thread until we receive the response data, or requests + // for headers/body data, which can only be serviced from this thread. + xQueueReceive(queue, &tr, portMAX_DELAY); + if (tr.request_type == GET_HEADER) + { + size_t len = httpd_req_get_hdr_value_len(req, tr.header.name); + if (len) + { + *tr.header.value = malloc(len + 1); + httpd_req_get_hdr_value_str( + req, tr.header.name, *tr.header.value, len + 1); + } + else + *tr.header.value = NULL; // no such header + } + else if (tr.request_type == READ_BODY_CHUNK) + { + *tr.body_chunk = malloc(sizeof(body_chunk_t)); + size_t to_read = (remaining_len >= RECV_BODY_CHUNK_SIZE) ? + RECV_BODY_CHUNK_SIZE : remaining_len; + remaining_len -= to_read; + int ret = httpd_req_recv(req, (*tr.body_chunk)->data, to_read); + if (ret != to_read) + { + errored = true; + free(*tr.body_chunk); + *tr.body_chunk = NULL; + } + else + (*tr.body_chunk)->used = to_read; + } + else if (tr.request_type == SEND_RESPONSE || + tr.request_type == SEND_PARTIAL_RESPONSE) + { + if (errored) + httpd_resp_send_408(req); + else + { + bool is_partial = (tr.request_type == SEND_PARTIAL_RESPONSE); + const response_data_t *resp = tr.response; + if (!is_partial || resp->status_str) + httpd_resp_set_status(req, resp->status_str); + if (!is_partial || resp->content_type) + httpd_resp_set_type(req, resp->content_type); + for (unsigned i = 0; resp->headers[i].key; ++i) + httpd_resp_set_hdr(req, resp->headers[i].key, resp->headers[i].value); + if (!is_partial) + httpd_resp_send(req, resp->body_data, resp->body_len); + else + { + httpd_resp_send_chunk(req, resp->body_data, resp->body_len); + if (resp->body_data == NULL) // Was this the last chunk? + tr.request_type = SEND_RESPONSE; // If so, flag our exit condition + } + } + } + + // Request processed, release LVM thread + xSemaphoreGive(done); + } while(tr.request_type != SEND_RESPONSE); // done + + return ESP_OK; +} + + +// ---- helper functions ---------------------------------------------- + + +static int check_valid_httpd_method(lua_State *L, int idx) +{ + int method = luaL_checkinteger(L, idx); + switch (method) + { + case HTTP_GET: + case HTTP_HEAD: + case HTTP_PUT: + case HTTP_POST: + case HTTP_DELETE: break; + default: return luaL_error(L, "unknown method %d", method); + } + return method; +} + + +static void check_valid_guard_value(lua_State *L) +{ + int check = lua_tointeger(L, lua_upvalueindex(1)); + if (check != guard) + luaL_error(L, "gethdr()/getbody() called outside synchronous flow"); +} + + +static int lsync_get_hdr(lua_State *L) +{ + check_valid_guard_value(L); + + const char *header_name = luaL_checkstring(L, 2); + char *header_val = NULL; + thread_request_t tr = { + .request_type = GET_HEADER, + .header = { + .name = header_name, + .value = &header_val, + } + }; + xQueueSend(queue, &tr, portMAX_DELAY); + xSemaphoreTake(done, portMAX_DELAY); + if (header_val) + lua_pushstring(L, header_val); + else + lua_pushnil(L); + free(header_val); + return 1; +} + + +static int lsync_get_body_chunk(lua_State *L) +{ + check_valid_guard_value(L); + + body_chunk_t *chunk = NULL; + thread_request_t tr = { + .request_type = READ_BODY_CHUNK, + .body_chunk = &chunk, + }; + xQueueSend(queue, &tr, portMAX_DELAY); + xSemaphoreTake(done, portMAX_DELAY); + if (chunk) + { + if (chunk->used) + lua_pushlstring(L, chunk->data, chunk->used); + else + lua_pushnil(L); // end of body reached + } + else + return luaL_error(L, "read body failed"); + return 1; +} + + +static int lhttpd_req_index(lua_State *L) +{ + req_udata_t *ud = (req_udata_t *)luaL_checkudata(L, 1, REQUEST_METATABLE); + const char *key = luaL_checkstring(L, 2); +#define KEY_IS(x) (strcmp(key, x) == 0) + if (KEY_IS("uri")) + lua_pushstring(L, ud->req_info->uri); + else if (KEY_IS("method")) + lua_pushinteger(L, ud->req_info->method); + else if (KEY_IS("query") && ud->req_info->query_str) + lua_pushstring(L, ud->req_info->query_str); + else if (KEY_IS("headers")) + { + lua_newtable(L); + lua_newtable(L); // metatable + lua_pushinteger(L, ud->guard); // +1 + lua_pushcclosure(L, lsync_get_hdr, 1); // -1 +1 + lua_setfield(L, -2, "__index"); // -1 + lua_setmetatable(L, -2); // -1 + } + else if (KEY_IS("getbody")) + { + lua_pushinteger(L, guard); // +1 + lua_pushcclosure(L, lsync_get_body_chunk, 1); // -1 +1 + } + else + lua_pushnil(L); + + return 1; +#undef KEY_IS +} + + +static void dynamic_handler_lvm(task_param_t param, task_prio_t prio) +{ + UNUSED(prio); + + const request_data_t *req_info = (const request_data_t *)param; + + lua_State *L = lua_getstate(); + int saved_top = lua_gettop(L); + + lua_checkstack(L, MAX_RESPONSE_HEADERS*2 + 9); + + response_data_t resp = error_resp; + thread_request_t tr = { + .request_type = SEND_RESPONSE, + .response = &resp, + }; + + lua_rawgeti(L, LUA_REGISTRYINDEX, dynamic_handlers_table_ref); // +1 + lua_getfield(L, -1, req_info->key); // +1 + if (lua_isfunction(L, -1)) + { + // push req + req_udata_t *ud = + (req_udata_t *)lua_newuserdata(L, sizeof(req_udata_t)); // +1 + ud->req_info = req_info; + ud->guard = guard; + luaL_getmetatable(L, REQUEST_METATABLE); // +1 + lua_setmetatable(L, -2); // -1 + + int err = luaL_pcallx(L, 1, 1); // -1 +1 + if (!err && lua_istable(L, -1)) + { + // pull out response data + int t = lua_gettop(L); // response table index + lua_getfield(L, t, "status"); // +1 + resp.status_str = luaL_optstring(L, -1, "200 OK"); + lua_getfield(L, t, "type"); // +1 + resp.content_type = luaL_optstring(L, -1, NULL); + lua_getfield(L, t, "body"); // +1 + resp.body_data = luaL_optlstring(L, -1, NULL, &resp.body_len); + if (!resp.body_data) + resp.body_len = 0; + lua_getfield(L, t, "headers"); // +1 + if (lua_istable(L, -1)) + { + lua_pushnil(L); // +1 + for (unsigned i = 0; lua_next(L, -2); ++i) // +1 + { + if (i >= MAX_RESPONSE_HEADERS) + { + printf("Warning - too many response headers, ignoring some!\n"); + break; + } + resp.headers[i].key = lua_tostring(L, -2); + resp.headers[i].value = lua_tostring(L, -1); + lua_pop(L, 1); // drop value, keep key for lua_next() + } + } + lua_getfield(L, t, "getbody"); // +1 + if (lua_isfunction(L, -1)) + { + // Okay, we're doing a chunked body send, so we have to repeatedly + // call the provided getbody() function until it returns nil + bool headers_cleared = false; + tr.request_type = SEND_PARTIAL_RESPONSE; +next_chunk: + resp.body_data = NULL; + resp.body_len = 0; + err = luaL_pcallx(L, 0, 1); // -1 +1 + resp.body_data = + err ? NULL : luaL_optlstring(L, -1, NULL, &resp.body_len); + if (resp.body_data) + { + // Toss this bit of response data over to the httpd thread + xQueueSend(queue, &tr, portMAX_DELAY); + // ...and wait until it's done sending it + xSemaphoreTake(done, portMAX_DELAY); + + lua_pop(L, 1); // -1 + + if (!headers_cleared) + { + // Clear the header data; it's only used for the first chunk + resp.status_str = NULL; + resp.content_type = NULL; + for (unsigned i = 0; i < MAX_RESPONSE_HEADERS; ++i) + resp.headers[i].key = resp.headers[i].value = NULL; + + headers_cleared = true; + } + lua_getfield(L, t, "getbody"); // +1 + goto next_chunk; + } + // else, getbody() returned nil, so let the normal exit path + // toss the final SEND_PARTIAL_RESPONSE request over to the httpd + } + } + } + + // Toss the response data over to the httpd thread for sending + xQueueSend(queue, &tr, portMAX_DELAY); + + // Block until the httpd thread has finished accessing our Lua strings + xSemaphoreTake(done, portMAX_DELAY); + + // Clean up the stack + lua_settop(L, saved_top); + + // Make any further gethdr()/getbody() calls fail rather than deadlock + ++guard; +} + + +// ---- Lua interface ------------------------------------------------- + + +// add static route: httpd.static(uri, content_type) +static int lhttpd_static(lua_State *L) +{ + if (!server) + return luaL_error(L, "Server not started"); + + const char *match = luaL_checkstring(L, 1); + const char *content_type = luaL_checkstring(L, 2); + + if (!match[0]) + return luaL_error(L, "Null route not supported"); + + // Store this in our content-type table, so the content-type string lives + // on, but so that we can also free it after server shutdown. + lua_rawgeti(L, LUA_REGISTRYINDEX, content_types_table_ref); + lua_pushvalue(L, 1); + lua_pushvalue(L, 2); + lua_settable(L, -3); + + httpd_uri_t static_handler = { + .uri = match, + .method = HTTP_GET, + .handler = static_file_handler, + .user_ctx = (void *)content_type, + }; + if (httpd_register_uri_handler(server, &static_handler) == 1) + lua_pushinteger(L, 1); + else + lua_pushnil(L); + + return 1; +} + + +// add dynamic route: httpd.dynamic(method, uri, handler) +static int lhttpd_dynamic(lua_State *L) +{ + if (!server) + return luaL_error(L, "Server not started"); + + int method = check_valid_httpd_method(L, 1); + const char *match = luaL_checkstring(L, 2); + luaL_checkfunction(L, 3); + lua_settop(L, 3); + + if (!match[0]) + return luaL_error(L, "Null route not supported"); + + // Create a key for this entry + const char *key = lua_pushfstring(L, "[%d]%s", method, match); + + // Store this in our dynamic handlers table, so the ref lives on + // on, but so that we can also free it after server shutdown. + lua_rawgeti(L, LUA_REGISTRYINDEX, dynamic_handlers_table_ref); + lua_pushvalue(L, -2); // key + lua_pushvalue(L, 3); // handler + lua_settable(L, -3); + + httpd_uri_t static_handler = { + .uri = match, + .method = method, + .handler = dynamic_handler_httpd, + .user_ctx = (void *)key, + }; + if (httpd_register_uri_handler(server, &static_handler) == 1) + lua_pushinteger(L, 1); + else + lua_pushnil(L); + + return 1; +} + + +// unregister route; httpd.unregister(method, uri) +static int lhttpd_unregister(lua_State *L) +{ + if (!server) + return luaL_error(L, "Server not started"); + + int method = check_valid_httpd_method(L, 1); + const char *match = luaL_checkstring(L, 2); + + if (httpd_unregister_uri_handler(server, match, method) == ESP_OK) + lua_pushinteger(L, 1); + else + lua_pushnil(L); + + return 1; +} + + +static int lhttpd_start(lua_State *L) +{ + if (server) + return luaL_error(L, "Server already started"); + + luaL_checktable(L, 1); + lua_settop(L, 1); + + lua_getfield(L, 1, "webroot"); + const char *root = luaL_checkstring(L, -1); + webroot = strdup(root); + lua_pop(L, 1); + + lua_getfield(L, 1, "max_handlers"); + int max_handlers = luaL_optinteger(L, -1, 20); + + lua_getfield(L, 1, "auto_index"); + index_mode = (index_mode_t)luaL_optinteger(L, -1, INDEX_ROOT); + + httpd_config_t config = HTTPD_DEFAULT_CONFIG(); + config.uri_match_fn = uri_match_file_suffix_first; + config.max_uri_handlers = max_handlers; + config.max_resp_headers = MAX_RESPONSE_HEADERS; + + esp_err_t err = httpd_start(&server, &config); + if (err != ESP_OK) + return luaL_error(L, "Failed to start http server; code %d", err); + + // Set up our content type stash + lua_newtable(L); + content_types_table_ref = luaL_ref(L, LUA_REGISTRYINDEX); + + // Set up our dynamic handlers table + lua_newtable(L); + dynamic_handlers_table_ref = luaL_ref(L, LUA_REGISTRYINDEX); + + // Register default static suffixes + size_t num_suffixes = sizeof(default_suffixes)/sizeof(default_suffixes[0]); + for (size_t i = 0; i < num_suffixes; ++i) + { + const char *sufx = default_suffixes[i]; + const char *content_type = strchr(sufx, '\0') + 1; + lua_pushcfunction(L, lhttpd_static); + lua_pushstring(L, sufx); + lua_pushstring(L, content_type); + lua_call(L, 2, 0); + } + + // Auto-indexer + httpd_uri_t index_handler = { + .uri = "", + .method = HTTP_GET, + .handler = auto_index_handler, + }; + httpd_register_uri_handler(server, &index_handler); + + return 0; +} + + +static int lhttpd_stop(lua_State *L) +{ + if (server) + { + httpd_stop(server); // deletes all handlers + server = NULL; + free(webroot); + luaL_unref2(L, LUA_REGISTRYINDEX, content_types_table_ref); + luaL_unref2(L, LUA_REGISTRYINDEX, dynamic_handlers_table_ref); + } + return 0; +} + + +LROT_BEGIN(httpd_req_mt, NULL, LROT_MASK_INDEX) + LROT_FUNCENTRY( __index, lhttpd_req_index ) +LROT_END(httpd_req_mt, NULL, LROT_MASK_INDEX) + + +LROT_BEGIN(httpd, NULL, 0) + LROT_FUNCENTRY( start, lhttpd_start ) + LROT_FUNCENTRY( stop, lhttpd_stop ) + LROT_FUNCENTRY( static, lhttpd_static ) + LROT_FUNCENTRY( dynamic, lhttpd_dynamic ) + LROT_FUNCENTRY( unregister, lhttpd_unregister ) + + LROT_NUMENTRY( GET, HTTP_GET ) + LROT_NUMENTRY( HEAD, HTTP_HEAD ) + LROT_NUMENTRY( PUT, HTTP_PUT ) + LROT_NUMENTRY( POST, HTTP_POST ) + LROT_NUMENTRY( DELETE, HTTP_DELETE ) + + LROT_NUMENTRY( INDEX_NONE, INDEX_NONE ) + LROT_NUMENTRY( INDEX_ROOT, INDEX_ROOT ) + LROT_NUMENTRY( INDEX_ALL, INDEX_ALL ) +LROT_END(httpd, NULL, 0) + + +static int lhttpd_init(lua_State *L) +{ + dynamic_task = task_get_id(dynamic_handler_lvm); + queue = xQueueCreate(1, sizeof(thread_request_t)); + done = xSemaphoreCreateBinary(); + + luaL_rometatable(L, REQUEST_METATABLE, LROT_TABLEREF(httpd_req_mt)); + + return 0; +} + +NODEMCU_MODULE(HTTPD, "httpd", httpd, lhttpd_init); diff --git a/docs/modules/eromfs.md b/docs/modules/eromfs.md new file mode 100644 index 00000000..91e76d2d --- /dev/null +++ b/docs/modules/eromfs.md @@ -0,0 +1,90 @@ +# EROMFS Module +| Since | Origin / Contributor | Maintainer | Source | +| :----- | :-------------------- | :---------- | :------ | +| 2021-11-13 | [Johny Mattsson](https://github.com/jmattsson) |[Johny Mattsson](https://github.com/jmattsson) | [heaptrace.c](../../components/modules/eromfs.c)| + +EROMFS (Embedded Read-Only Mountable File Sets) provides a convenient mechanism +for bundling file sets into the firmware image itself. The main use cases +envisaged for this is static web site content and default "skeleton" files +that may be used to populate SPIFFS on first boot. + +When enabling the `eromfs` module one or more file sets ("volumes") must be +declared. Each such volume is identified by name, and may be mounted anywhere +supported by the [Virtual File System](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/vfs.html). Once mounted, the included +files are available on a read-only basis to any thread wanting to access them. + +Note that EROMFS does not support directories per se, but will store the +directory path as part of the filename just as SPIFFS does. As such it is +only possible to list the root of the volume, not subdirectories (since +they don't exist). + +## eromfs.list +Returns a list of the bundled file sets (volumes). + +#### Syntax +```lua +eromfs.list() +``` + +#### Parameters +None. + +#### Returns +An array with the names of the bundled volumes. + +#### Example +```lua +for _, volname in ipairs(eromfs.list()) do print(volname) end +``` + +## eromfs.mount +Mounts a volume at a specified point in the virtual file system. + +Note that it is technically possible to mount a volume multiple times on +different mount points. The benefit of doing so however is questionable. + +#### Syntax +```lua +eromfs.mount(volname, mountpt) +``` + +#### Parameters +- `volname` the name of the volume to mount, e.g. `myvol`. +- `mountpt` where to mount said volume. Must start with '/', e.g. `/myvol`. + +#### Returns +`nil` on success. Raises an error if the named volume cannot be found, or +cannot be mounted. + +#### Example +```lua +-- Assumes the volume named "myvol" exists +eromfs.mount('myvol', '/somewhere') +for name,size in pairs(file.list('/somewhere')) do print(name, size) end +``` + +## eromfs.unmount +Unmounts the specified EROMFS volume from the given mount point. + +#### Syntax +```lua +eromfs.unmount(volname, mountpt) +``` + +#### Parameters +- `volname` the name of the volume to mount. +- `mountpt` the current mount point of the volume. + +#### Returns +`nil` if: +- the volume was successfully unmounted; or +- the volume was not currently mounted at the given mount point + +Raises an error if: +- the unmounting fails for some reason; or +- a different EROMFS volume is mounted on the given mount point + +#### Example +```lua +eromfs.unmount('myvol', '/somewhere') +``` diff --git a/docs/modules/httpd.md b/docs/modules/httpd.md new file mode 100644 index 00000000..f651f5c3 --- /dev/null +++ b/docs/modules/httpd.md @@ -0,0 +1,286 @@ +# LEDC Module +| Since | Origin / Contributor | Maintainer | Source | +| :----- | :-------------------- | :---------- | :------ | +| 2021-11-07 | [Johny Mattsson](https://github.com/jmattsson) | [Johny Mattsson](https://github.com/jmattsson) | [httpd.c](../../components/modules/httpd.c)| + +This module provides an interface to Espressif's [web server component](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/protocols/esp_http_server.html). + + +# HTTPD Overview + +The httpd module implements support for both static file serving and dynamic +content generation. For static files, all files need to reside under a +common prefix (the "webroot") in the (virtual) filesystem. The module does +not care whether the underlying file system supports directories or not, +so files may be served from SPIFFS, FAT filesystems, or whatever else +may be mounted. If you wish to include the static website contents within +the firmware image itself, considering using the [EROMFS](eromfs.md) module. + +Unlike the default behaviour of the Espressif web server, this module serves +static files based on file extensions primarily. Static routes are typically +defined as a file extension (e.g. \*.html) and the `Content-Type` such files +should be served as. A number of file extensions are included by default +and should cover the basic needs: + + - \*.html (text/html) + - \*.css (text/css) + - \*.js (text/javascript) + - \*.json (application/json) + - \*.gif (image/gif) + - \*.jpg (image/jpeg) + - \*.jpeg (image/jpeg) + - \*.png (image/png) + - \*.svg (image/svg+xml) + - \*.ttf (font/ttf) + +The native Espressif approach may also be used if you prefer, but is harder +to work with. Both schemes can coexist in most cases without issues. When +using the native approach, URI wildcard matching is supported. + +Dynamic routes may be registered, which when accessed by a client will result +in a Lua function being invoked. This function may then generate whatever +content is applicable, for example obtaining a sensor value and returning it. + +Note that if you are writing sensor data to files and serving those files +statically you will be susceptible to race conditions where the file contents +may not be available from the outside. This is due to the web server running +in its own FreeRTOS thread and serving files directly from that thread +concurrently with the Lua VM running as usual. It is therefore safer to +instead serve such content on a dynamic route, even if all that route does +is reads the file and serves that. + +An example of such a setup: +```lua +function handler(req) + local f = io.open('/path/to/mysensordata.csv', 'r') + return { + status = "200 OK", + type = "text/plain", + getbody = function() + local data = f:read(512) -- pick a suiteable chunk size here + if not data then f:close() end + return data + end, + } +end + +httpd.dynamic(httpd.GET, "/mysensordata", handler) +``` + +## httpd.start() +Starts the web server. The server has to be started before routes can be +configured. + +#### Syntax +```lua +httpd.start({ + webroot = "", + max_handlers = 20, + auto_index = httpd.INDEX_NONE || httpd.INDEX_ROOT || httpd.INDEX_ALL, +}) +``` + +#### Parameters +A single configuration table is provided, with the following possible fields: + + - `webroot` (mandatory) This sets the prefix used when serving static files. + For example, with `webroot` set to "web", a HTTP request for "/index.html" + will result in the httpd module trying to serve the file "web/index.html" + from the file system. Do NOT set this to the empty string, as that would + provide remote access to your entire virtual file system, including special + files such as virtual device files (e.g. "/dev/uart1") which would likely + present a serious security issue. + - `max_handlers` (optional) Configures the maximum number of route handlers + the server will support. Default value is 20, which includes both the + standard static file extension handlers and any user-provided handlers. + Raising this will result in a bit of additional memory being used. Adjust + if and when necessary. + - `auto_index` Sets the indexer mode to be used. Most web servers + automatically go looking for an "index.html" file when a directory is + requested. For example, when pointing your web browser to a web site + for the first time, e.g. http://www.example.com/ the actual request will + come through for "/", which in turn commonly gets translated to "/index.html" + on the server. This behaviour can be enabled in this module as well. There + are three modes provided: + - `httpd.INDEX_NONE` No automatic translation to "index.html" is provided. + - `httpd.INDEX_ROOT` Only the root ("/") is translated to "/index.html". + - `httpd.INDEX_ALL` Any path ending with a "/" has "index.html" appended. + For example, a request for "subdir/" would become "subdir/index.html", + which in turn might result in the file "web/subdir/index.html" being + served (if the `webroot` was set to "web"). + The default value is `httpd.INDEX_ROOT`. + +#### Returns +`nil` + +#### Example +```lua +httpd.start({ webroot = "web", auto_index = httpd.INDEX_ALL }) +``` + +## httpd.stop() + +Stops the web server. All registered route handlers are removed. + +#### Syntax +```lua +httpd.stop() +``` + +#### Parameters +None. + +#### Returns +`nil` + + +## httpd.static() + +Registers a static route handler. + +#### Syntax +``` +httpd.static(route, content_type) +``` + +#### Parameters +- `route` The route prefix. Typically in the form of \*.ext to serve all files +with the ".ext" extension statically. Refer to the Espressif [documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/protocols/esp_http_server.html) +if you wish to use the native Espressif style of static routes instead. +- `content_type` The value to send in the `Content-Type` header for this file +type. + +#### Returns +An error code on failure, or `nil` on success. The error code is the value +returned from the `httpd_register_uri_handler()` function. + +#### Example +```lua +httpd.start({ webroot = "web" }) +httpd.static("*.csv", "text/csv") -- Serve CSV files under web/ +``` + +## httpd.dynamic() + +Registers a dynamic route handler. + +#### Syntax +```lua +httpd.dynamic(method, route, handler) +``` + +#### Parameters +- `method` The HTTP method this route applies to. One of: + - `httpd.GET` + - `httpd.HEAD` + - `httpd.PUT` + - `httpd.POST` + - `httpd.DELETE` +- `route` The route prefix. Be mindful of any trailing "/" as that may interact +with the `auto_index` functionality. +- `handler` The route handler function - `handler(req)`. The provided request +object `req` has the following fields/functions: + - `method` The request method. Same as the `method` parameter above. If the + same function is registered for several methods, this field can be used to + determine the method the request used. + - `uri` The requested URI. Includes both path and query string (if + applicable). + - `query` The query string on its own. Not decoded. + - `headers` A table-like object in which request headers may be looked up. + Note that due to the Espressif API not providing a way to iterate over all + headers this table will appear empty if fed to `pairs()`. + - `getbody()` A function which may be called to read in the request body + incrementally. The size of each chunk is set via the Kconfig option + "Receive body chunk size". When this function returns `nil` the end of + the body has been reached. May raise an error if reading the body fails + for some reason (e.g. timeout, network error). + +Note that the provided `req` object is _only valid_ within the scope of this +single invocation of the handler. Attempts to store away the request and use +it later _will_ fail. + +#### Returns +A table with the response data to send to the requesting client: +```lua +{ + status = "200 OK", + type = "text/plain", + headers = { + ['X-Extra'] = "My custom header value" + }, + body = "Hello, Lua!", + getbody = dynamic_content_generator_func, +} +``` +Supported fields: +- `status` The status code and string to send. If not included "200 OK" is used. +Other common strings would be "404 Not Found", "400 Bad Request" and everybody's +favourite "500 Internal Server Error". +- `type` The value for the `Content-Type` header. The Espressif web server +component handles this header specially, which is why it's provided here and +not within the `headers` table. +- `body` The full content body to send. +- `getbody` A function to source the body content from, similar to the way +the request body is read in. This function will be called repeatedly and the +returned string from each invocation will be sent as a chunk to the client. +Once this function returns `nil` the body is deemed to be complete and no +further calls to the function will be made. It is guaranteed that the +function will be called until it returns `nil` even if the sending of the +content encounters an error. This ensures that any resource cleanup +necessary will still take place in such circumstances (e.g. file closing). + +Only one of `body` and `getbody` should be specified. + +#### Example +```lua +httpd.start({ webroot = "web" }) + +function put_foo(req) + local body_len = tonumber(req.headers['content-length']) or 0 + if body_len < 4096 + then + local f = io.open("/upload/foo.txt", "w") + local body = req.getbody() + while body + do + f:write(body) + body = req.getbody() + end + f:close() + return { status = "201 Created" } + else + return { status = "400 Bad Request" } + end +end + +httpd.dynamic(httpd.PUT, "/foo", put_foo) +``` + +## httpd.unregister() + +Unregisters a previously registered handler. The default handlers may be +unregistered. + +#### Syntax +```lua +httpd.unregister(method, route) +``` + +#### Parameters +- `method` The method the route was registered for. One of: + - `httpd.GET` + - `httpd.HEAD` + - `httpd.PUT` + - `httpd.POST` + - `httpd.DELETE` +- `route` The route prefix. + +#### Returns +`1` on success, `nil` on failure (including if the route was not registered). + +#### Example +Unregistering one of the default static handlers: +```lua +httpd.start({ webroot = "web" }) +httpd.unregister(httpd.GET, "*.jpeg") +``` diff --git a/mkdocs.yml b/mkdocs.yml index 89e349e1..7a7acf8e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -53,6 +53,7 @@ pages: - 'gpio': 'modules/gpio.md' - 'heaptrace': 'modules/heaptrace.md' - 'http': 'modules/http.md' + - 'httpd': 'modules/httpd.md' - 'i2c': 'modules/i2c.md' - 'i2s': 'modules/i2s.md' - 'ledc': 'modules/ledc.md' diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 1f6f711f..617c1c31 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -27,7 +27,7 @@ CONFIG_LWIP_SO_REUSE=y # Decrease the duration of sockets in TIME_WAIT # see https://github.com/nodemcu/nodemcu-firmware/issues/1836 -CONFIG_TCP_MSL=5000 +CONFIG_LWIP_TCP_MSL=5000 # Disable esp-idf's bluetooth component by default. # The bthci module is also disabled and will enable bt when selected