From 73b13e4197ae0e0f83f3202810c3422ab054af7e Mon Sep 17 00:00:00 2001 From: tomsci Date: Thu, 8 Nov 2018 19:24:18 +0000 Subject: [PATCH] Add ESP32 http module (#2540) * ESP32: Added http module * add asynchronous flavor for context:request() (cherry picked from commit e65b90cc8fc5296f7fe6cae1978835e06a9f44bb) * http: More asynchronous support, more options * Fix docs typo * Code review comments from @devsaurus Fixes some cleanup issues with asynchronous mode * Added http.md to mkdocs.yml * Align connection:close() to template --- components/modules/Kconfig | 6 + components/modules/http.c | 853 +++++++++++++++++++++++++++++++++++ docs/en/modules/http.md | 248 ++++++++++ lua_examples/http-client.lua | 40 -- lua_examples/httptest.lua | 151 +++++++ mkdocs.yml | 3 +- 6 files changed, 1260 insertions(+), 41 deletions(-) create mode 100644 components/modules/http.c create mode 100644 docs/en/modules/http.md delete mode 100644 lua_examples/http-client.lua create mode 100644 lua_examples/httptest.lua diff --git a/components/modules/Kconfig b/components/modules/Kconfig index ffc9fa7e..cb17df53 100644 --- a/components/modules/Kconfig +++ b/components/modules/Kconfig @@ -105,6 +105,12 @@ config LUA_MODULE_GPIO help Includes the GPIO module (recommended). +config LUA_MODULE_HTTP + bool "HTTP module" + default "y" + help + Includes the HTTP module (recommended). + config LUA_MODULE_I2C bool "I2C module" default "y" diff --git a/components/modules/http.c b/components/modules/http.c new file mode 100644 index 00000000..3ce8c79c --- /dev/null +++ b/components/modules/http.c @@ -0,0 +1,853 @@ +#include "module.h" +#include "lauxlib.h" +#include "lmem.h" +#include + +#include "esp_http_client.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "esp_task.h" +#include "task/task.h" + +#include +#define TAG "http" + +typedef struct list_item list_item; + +enum { + ConnectionCallback = 0, + HeadersCallback, + DataCallback, + CompleteCallback, + + // after this are other refs which aren't callbacks + + ContextRef, + PostDataRef, + CertRef, + CountRefs // Must be last +}; + +static char const * const CALLBACK_NAME[] = { + "connect", + "headers", + "data", + "complete", + NULL +}; + +typedef enum { + Async = 0, + Connected = 1, + InCallback = 2, + AckPending = 3, + ShouldCloseInRtosTask = 4, +} Flag; + +typedef struct +{ + esp_http_client_handle_t client; + int refs[CountRefs]; + list_item *headers; + int16_t status_code; + uint16_t flags; + TaskHandle_t perform_rtos_task; // NULL if we're not in the middle of an async request +} lhttp_context_t; + +struct list_item +{ + struct list_item *next; + uint32_t len; + char data[1]; +}; + +typedef struct +{ + int id; + lhttp_context_t *context; + union { + list_item *headers; + struct { + uint32_t data_len; + char data[1]; + }; + }; +} lhttp_event; + +static const char http_context_mt[] = "http.context"; +#define DELAY_ACK (-99) // Chosen not to conflict with any other esp_err_t +#define HTTP_REQUEST_COMPLETE (-1) + +static task_handle_t lhttp_request_task_id, lhttp_event_task_id; +#define CHECK_CONNECTION_IDLE(ctx) \ + if (ctx->perform_rtos_task || context_flag(ctx, InCallback)) { \ + return luaL_error(L, "Cannot modify connection while a request is active"); \ + } + +static void context_setflag(lhttp_context_t *context, Flag flag) +{ + context->flags |= 1 << flag; +} + +static void context_clearflag(lhttp_context_t *context, Flag flag) +{ + context->flags &= ~(1 << flag); +} + +static bool context_flag(lhttp_context_t const *context, Flag flag) +{ + return context->flags & (1 << flag); +} + +static void context_setflagbool(lhttp_context_t *context, Flag flag, bool val) +{ + if (val) { + context_setflag(context, flag); + } else { + context_clearflag(context, flag); + } +} + +static void context_setref(lua_State *L, lhttp_context_t *context, int index) +{ + assert(index >= 0 && index < CountRefs); + luaL_unref(L, LUA_REGISTRYINDEX, context->refs[index]); + int ref = luaL_ref(L, LUA_REGISTRYINDEX); + if (ref == LUA_REFNIL) { + ref = LUA_NOREF; + } + context->refs[index] = ref; +} + +static void context_unsetref(lua_State *L, lhttp_context_t *context, int index) +{ + assert(index >= 0 && index < CountRefs); + int ref = context->refs[index]; + context->refs[index] = LUA_NOREF; + luaL_unref(L, LUA_REGISTRYINDEX, ref); +} + +static int context_close(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + if (!context->client) { + // Nothing to do + } else if (context->perform_rtos_task) { + // Can only be closed from within a callback or while there's a delayed ack pending + if (context_flag(context, InCallback) || context_flag(context, AckPending)) { + context_setflag(context, ShouldCloseInRtosTask); + } else { + return luaL_error(L, "Cannot close an ongoing async request outside of a callback or pending ack"); + } + } else { + esp_http_client_close(context->client); + } + + return 0; +} + +static int context_gc(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + assert(context->refs[ContextRef] == LUA_NOREF); // No way to get GC'd if we still hold a ref to ourselves + for (int i = 0; i < CountRefs; i++) { + context_unsetref(L, context, i); + } + list_item *hdr = context->headers; + while (hdr) { + list_item *next = hdr->next; + free(hdr); + hdr = next; + } + context->headers = NULL; + if (context->client) { + esp_http_client_cleanup(context->client); + context->client = NULL; + } + return 0; +} + +static lhttp_context_t *context_new(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)lua_newuserdata(L, sizeof(lhttp_context_t)); + context->client = NULL; + context->status_code = 0; + context->headers = NULL; + for (int i = 0; i < CountRefs; i++) { + context->refs[i] = LUA_NOREF; + } + context->flags = 0; + context->perform_rtos_task = NULL; + luaL_getmetatable(L, http_context_mt); + lua_setmetatable(L, -2); + return context; +} + +static void context_seterr(lhttp_context_t *context, esp_err_t err) +{ + if (err > 0 && err <= INT16_MAX) { + context->status_code = -err; + } else if (err) { + context->status_code = -1; + } +} + +static void perform_rtos_task(void *pvParameters) +{ + lhttp_context_t *context = (lhttp_context_t *)pvParameters; + + ulTaskNotifyTake(pdTRUE, 0); // ensure that notification counter is reset + esp_err_t err = esp_http_client_perform(context->client); + if (err) { + ESP_LOGW(TAG, "esp_http_client_perform returned error %d", err); + context_seterr(context, err); + } + ESP_LOGD(TAG, "perform_rtos_task completed, remaining stack = %d bytes", uxTaskGetStackHighWaterMark(NULL)); + task_post_low(lhttp_request_task_id, (task_param_t)context); + vTaskSuspend(NULL); +} + +static int make_callback(lhttp_context_t *context, int id, void *data, size_t data_len); + +static void lhttp_request_task(task_param_t param, task_prio_t prio) +{ + lhttp_context_t *context = (lhttp_context_t *)param; + + // the rtos task for esp_http_client_perform suspended, reap it + vTaskDelete(context->perform_rtos_task); + context->perform_rtos_task = NULL; + + make_callback(context, HTTP_REQUEST_COMPLETE, NULL, 0); + lua_State *L = lua_getstate(); + context_unsetref(L, context, ContextRef); +} + +// note: this function is called both in synchronous mode and in asynchronous mode +static esp_err_t http_event_cb(esp_http_client_event_t *evt) +{ + // ESP_LOGI(TAG, "http_event_cb %d", evt->event_id); + lhttp_context_t *context = (lhttp_context_t *)evt->user_data; + switch (evt->event_id) { + case HTTP_EVENT_ON_CONNECTED: { + context_setflag(context, Connected); + return make_callback(context, HTTP_EVENT_ON_CONNECTED, NULL, 0); + } + case HTTP_EVENT_ON_HEADER: { + size_t keylen = strlen(evt->header_key); + size_t vallen = strlen(evt->header_value); + list_item *hdr = (list_item *)malloc(sizeof(list_item) + keylen + vallen + 1); // +1 for final null + if (!hdr) { + return ESP_ERR_NO_MEM; + } + hdr->next = context->headers; + context->headers = hdr; + hdr->len = keylen; + memcpy(hdr->data, evt->header_key, keylen + 1); + memcpy(hdr->data + keylen + 1, evt->header_value, vallen + 1); + break; + } + case HTTP_EVENT_ON_DATA: { + if (context->headers) { + context->status_code = esp_http_client_get_status_code(evt->client); + int err = make_callback(context, HTTP_EVENT_ON_HEADER, NULL, 0); + if (err) return err; + } + return make_callback(context, evt->event_id, evt->data, evt->data_len); + } + case HTTP_EVENT_ON_FINISH: { + if (context->headers) { + // Might still be set, if there wasn't any data in the request + context->status_code = esp_http_client_get_status_code(evt->client); + int err = make_callback(context, HTTP_EVENT_ON_HEADER, NULL, 0); + if (err) return err; + } + // Given when HTTP_EVENT_ON_FINISH is dispatched (before the + // http_should_keep_alive check in esp_http_client_perform) I don't think + // there's any benefit to exposing this event + // int ret = make_callback(context, HTTP_EVENT_ON_FINISH, NULL, 0); + break; + } + case HTTP_EVENT_DISCONNECTED: + context_clearflag(context, Connected); + break; + default: + break; + } + return ESP_OK; +} + +// Task posted from http thread when there's an event +static void lhttp_event_task(task_param_t param, task_prio_t prio) +{ + esp_http_client_event_t *evt = (esp_http_client_event_t *)param; + lhttp_context_t *context = (lhttp_context_t *)evt->user_data; + + context_setflag(context, AckPending); + int result = http_event_cb(evt); + if (context->perform_rtos_task && result != DELAY_ACK) { + context_clearflag(context, AckPending); + xTaskNotifyGive(context->perform_rtos_task); + } +} + +static esp_err_t http_event_handler(esp_http_client_event_t *evt) +{ + lhttp_context_t *context = (lhttp_context_t *)evt->user_data; + + if (context->perform_rtos_task) { + // asynchronous mode: we're called from perform_rtos_task context + // 1. post to Lua task + task_post_high(lhttp_event_task_id, (task_param_t)evt); + // 2. wait for ack from Lua land + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + if (context_flag(context, ShouldCloseInRtosTask)) { + context_clearflag(context, ShouldCloseInRtosTask); + esp_http_client_close(context->client); + return ESP_FAIL; + } else { + return ESP_OK; + } + } else { + // Call directly + return http_event_cb(evt); + } +} + +static int make_callback(lhttp_context_t *context, int id, void *data, size_t data_len) +{ + lua_State *L = lua_getstate(); + lua_settop(L, 0); + + switch (id) { + case HTTP_EVENT_ON_CONNECTED: + lua_rawgeti(L, LUA_REGISTRYINDEX, context->refs[ConnectionCallback]); + break; + case HTTP_EVENT_ON_HEADER: { + lua_rawgeti(L, LUA_REGISTRYINDEX, context->refs[HeadersCallback]); + lua_pushinteger(L, context->status_code); + lua_newtable(L); + list_item *item = context->headers; + while (item) { + lua_pushlstring(L, item->data, item->len); // key + // Lowercase all header names + luaL_getmetafield(L, -1, "lower"); + lua_insert(L, -2); + lua_call(L, 1, 1); + char *val = item->data + item->len + 1; + lua_pushstring(L, val); + lua_settable(L, -3); + list_item *next = item->next; + free(item); + item = next; + } + context->headers = NULL; + break; + } + case HTTP_EVENT_ON_DATA: + lua_rawgeti(L, LUA_REGISTRYINDEX, context->refs[DataCallback]); + lua_pushinteger(L, context->status_code); + lua_pushlstring(L, data, data_len); + break; + case HTTP_REQUEST_COMPLETE: + lua_rawgeti(L, LUA_REGISTRYINDEX, context->refs[CompleteCallback]); + lua_pushinteger(L, context->status_code); + lua_pushboolean(L, context_flag(context, Connected)); + break; + default: + break; + } + int result = ESP_OK; + if (lua_type(L, 1) == LUA_TFUNCTION) { + // Don't set InCallback for complete callback, we use that to determine + // whether a connection is busy or not, and during a complete it's not + if (id != HTTP_REQUEST_COMPLETE) { + context_setflag(context, InCallback); + } + int err = lua_pcall(L, lua_gettop(L) - 1, 1, 0); + context_clearflag(context, InCallback); + if (err) { + const char *msg = lua_type(L, -1) == LUA_TSTRING ? lua_tostring(L, -1) : ""; + ESP_LOGW(TAG, "Error returned from callback for HTTP event %d: %s", id, msg); + result = ESP_FAIL; + } else { + bool delay_ack = (lua_tointeger(L, -1) == DELAY_ACK); + if (delay_ack) { + if (!context_flag(context, Async)) { + luaL_error(L, "Cannot delay acknowledgment of a callback when using synchronous callbacks"); + } else if (id != HTTP_EVENT_ON_DATA) { + luaL_error(L, "Cannot delay acknowledgment of callbacks other than 'data'"); + } + result = DELAY_ACK; + } + } + } + lua_settop(L, 0); + return result; +} + +// headers_idx must be absolute idx +static void set_headers(lua_State *L, int headers_idx, esp_http_client_handle_t client) +{ + if (lua_isnoneornil(L, headers_idx)) { + return; + } + lua_pushnil(L); + while (lua_next(L, headers_idx) != 0) { + /* uses 'key' (at index -2) and 'value' (at index -1) */ + if (lua_type(L, -2) == LUA_TSTRING && lua_type(L, -1) == LUA_TSTRING) { + const char *key = lua_tostring(L, -2); + const char *val = lua_tostring(L, -1); + esp_http_client_set_header(client, key, val); + } + /* removes 'value'; keeps 'key' for next iteration */ + lua_pop(L, 1); + } +} + +// Similar to luaL_argcheck() but using the options table rather than a raw index +static bool get_option(lua_State *L, const char *name, int required_type) +{ + if (!lua_istable(L, -1)) { + return false; + } + + lua_getfield(L, -1, name); + int type = lua_type(L, -1); + if (type == LUA_TNIL) { + // Option not present + lua_pop(L, 1); + return false; + } + if (type != required_type) { + luaL_error(L, "Bad option '%s' to createConnection (%s expected, got %s)", + name, lua_typename(L, required_type), lua_typename(L, type)); + } + return true; +} + +static int check_optint(lua_State *L, const char *name, int default_val) +{ + if (get_option(L, name, LUA_TNUMBER)) { + int result = lua_tointeger(L, -1); + lua_pop(L, 1); + return result; + } else { + return default_val; + } +} + +static bool check_optbool(lua_State *L, const char *name, bool default_val) +{ + if (get_option(L, name, LUA_TBOOLEAN)) { + int result = lua_toboolean(L, -1); + lua_pop(L, 1); + return !!result; + } else { + return default_val; + } +} + +// Options assumed to be on top of stack +static void parse_options(lua_State *L, lhttp_context_t *context, esp_http_client_config_t *config) +{ + config->timeout_ms = check_optint(L, "timeout", 10*1000); // Same default as old http module + config->buffer_size = check_optint(L, "bufsz", DEFAULT_HTTP_BUF_SIZE); + int redirects = check_optint(L, "max_redirects", -1); // -1 means "not specified" here + if (redirects == 0) { + config->disable_auto_redirect = true; + } else if (redirects > 0) { + config->max_redirection_count = redirects; + } + // Note, config->is_async is always set to false regardless of what we set + // the Async flag to, because of how we configure the tasks we always want + // esp_http_client_perform to run in its 'is_async=false' mode. + context_setflagbool(context, Async, check_optbool(L, "async", false)); + + if (get_option(L, "cert", LUA_TSTRING)) { + const char *cert = lua_tostring(L, -1); + context_setref(L, context, CertRef); + config->cert_pem = cert; + } + + // This function doesn't set headers because we need the connection to be created first +} + +// http.createConnection([url, [method,] [options]) +static int http_lapi_createConnection(lua_State *L) +{ + lua_settop(L, 3); + const char *url = NULL; + if (!lua_isnoneornil(L, 1)) { + url = luaL_checkstring(L, 1); + } + int method = HTTP_METHOD_GET; + if (lua_type(L, 2) == LUA_TNUMBER) { + method = luaL_checkint(L, 2); + if (method < 0 || method >= HTTP_METHOD_MAX) { + return luaL_error(L, "Bad HTTP method %d", method); + } + } else { + // No method, make sure options on top + lua_settop(L, 2); + } + lhttp_context_t *context = context_new(L); // context now on top of stack + lua_insert(L, -2); // Move context below options + + esp_http_client_config_t config = { + .url = url, + .event_handler = http_event_handler, + .method = method, + .is_async = false, + .user_data = context, + }; + parse_options(L, context, &config); + + context->client = esp_http_client_init(&config); + if (!context->client) { + return luaL_error(L, "esp_http_client_init failed"); + } + + if (lua_istable(L, -1)) { + lua_getfield(L, -1, "headers"); + set_headers(L, lua_gettop(L), context->client); + lua_pop(L, 1); // headers + } + lua_pop(L, 1); // options + + // Note we do not take a ref to the context itself until http_lapi_request(), + // because otherwise we'd leak any object that was constructed but never used. + return 1; +} + +// context:on(name, fn) +static int http_lapi_on(lua_State *L) +{ + lua_settop(L, 3); + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + int callback_idx = luaL_checkoption(L, 2, NULL, CALLBACK_NAME); + context_setref(L, context, callback_idx); + lua_settop(L, 1); + return 1; // Return context, for chained calls +} + +// context:request() +static int http_lapi_request(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + CHECK_CONNECTION_IDLE(context); + + // context now owns itself for the duration of the request + assert(context->refs[ContextRef] == LUA_NOREF); + lua_pushvalue(L, 1); + context_setref(L, context, ContextRef); + + if (context_flag(context, Async)) { + if (xTaskCreate(perform_rtos_task, + "http_task", + 4096, + (void *)context, + ESP_TASK_MAIN_PRIO + 1, + &context->perform_rtos_task) != pdPASS) { + context_unsetref(L, context, ContextRef); + return luaL_error(L, "cannot create rtos task"); + } + return 0; + } else { + esp_err_t err = esp_http_client_perform(context->client); + // Note, the above call invalidates the Lua stack + + if (err) { + ESP_LOGW(TAG, "Error %d from esp_http_client_perform", err); + context_seterr(context, err); + } + make_callback(context, HTTP_REQUEST_COMPLETE, NULL, 0); + + lua_pushinteger(L, context->status_code); + lua_pushboolean(L, context_flag(context, Connected)); + context_unsetref(L, context, ContextRef); + return 2; + } +} + +// connection:seturl(url) +static int http_lapi_seturl(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + CHECK_CONNECTION_IDLE(context); + esp_err_t err = esp_http_client_set_url(context->client, luaL_checkstring(L, 2)); + if (err) { + return luaL_error(L, "esp_http_client_set_url returned %d", err); + } + return 0; +} + +// connection:setmethod(method) +static int http_lapi_setmethod(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + CHECK_CONNECTION_IDLE(context); + int method = luaL_checkint(L, 2); + if (method < 0 || method >= HTTP_METHOD_MAX) { + return luaL_error(L, "Bad HTTP method %d", method); + } + esp_http_client_set_method(context->client, method); + return 0; +} + +// connection:setheader(name, val) +static int http_lapi_setheader(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + CHECK_CONNECTION_IDLE(context); + const char *name = luaL_checkstring(L, 2); + const char *value = luaL_optstring(L, 3, NULL); + esp_http_client_set_header(context->client, name, value); + return 0; +} + +// context:setpostdata(data) +static int http_lapi_setpostdata(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + CHECK_CONNECTION_IDLE(context); + + size_t postdata_sz; + const char *postdata = luaL_optlstring(L, 2, NULL, &postdata_sz); + esp_http_client_set_method(context->client, HTTP_METHOD_POST); + esp_http_client_set_post_field(context->client, postdata, (int)postdata_sz); + context_setref(L, context, PostDataRef); + return 0; +} + +// context:ack() +static int http_lapi_ack(lua_State *L) +{ + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, 1, http_context_mt); + if (!context_flag(context, AckPending)) { + return luaL_error(L, "No asynchronous callback pending"); + } + if (context_flag(context, InCallback)) { + // Unblocking the HTTP task to potentially trigger more callbacks while + // we're still processing this callback is just too complicated + return luaL_error(L, "Cannot call ack() from within the callback itself"); + } + context_clearflag(context, AckPending); + xTaskNotifyGive(context->perform_rtos_task); + return 0; +} + +//// One-shot functions http.get(), http.post() etc follow //// + +// args: statusCode, headers +static int http_accumulate_headers(lua_State *L) +{ + int cache_table = lua_upvalueindex(1); + lua_rawseti(L, cache_table, 2); // cache_table[2] = headers + lua_rawseti(L, cache_table, 1); // cache_table[1] = statusCode + lua_pushinteger(L, 2); + lua_rawseti(L, cache_table, 0); // Use zero for len + return 0; +} + +// args: statusCode, data +static int http_accumulate_data(lua_State *L) +{ + int cache_table = lua_upvalueindex(1); + lua_rawgeti(L, cache_table, 0); + int n = lua_tointeger(L, -1); + lua_pop(L, 1); + lua_rawseti(L, cache_table, n + 1); // top of stack is data + lua_pushinteger(L, n + 1); + lua_rawseti(L, cache_table, 0); + return 0; +} + +static int http_accumulate_complete(lua_State *L) +{ + lua_settop(L, 1); // Don't care about any of the args except status_code + int context_idx = lua_upvalueindex(1); + int cache_table = lua_upvalueindex(2); + lua_pushvalue(L, lua_upvalueindex(3)); // The callback fn + lua_insert(L, 1); // Put callback fn first, status_code second + + // Now concat data + luaL_Buffer b; + luaL_buffinit(L, &b); + for (int i = 3; ; i++) { + lua_rawgeti(L, cache_table, i); + if lua_isnoneornil(L, -1) { + lua_pop(L, 1); + break; + } + luaL_addvalue(&b); + // Remove from table, don't need any more + lua_pushnil(L); + lua_rawseti(L, cache_table, i); + } + luaL_pushresult(&b); // data now pushed + if (lua_isnoneornil(L, 1)) { + // No callback fn so must be sync, meaning just need to stash headers and data in the context + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, context_idx, http_context_mt); + + // steal some completion refs, nothing's going to need them again in a one-shot + context_setref(L, context, DataCallback); // pops data + lua_rawgeti(L, cache_table, 2); // headers + context_setref(L, context, HeadersCallback); + } else { + lua_rawgeti(L, cache_table, 2); // headers + lua_call(L, 3, 0); + } + return 0; +} + +static int make_oneshot_request(lua_State *L, int callback_idx) +{ + // context must be on top of stack + lhttp_context_t *context = (lhttp_context_t *)luaL_checkudata(L, -1, http_context_mt); + const bool async = context_flag(context, Async); + lua_pushvalue(L, -1); // Need this later + + // Make sure we always send Connection: close for oneshots + esp_http_client_set_header(context->client, "Connection", "close"); + + lua_newtable(L); // cache table + + lua_pushvalue(L, -1); // dup cache table + lua_pushcclosure(L, http_accumulate_headers, 1); + context_setref(L, context, HeadersCallback); + + lua_pushvalue(L, -1); // dup cache table + lua_pushcclosure(L, http_accumulate_data, 1); + context_setref(L, context, DataCallback); + + // Don't dup cache table, it's in the right place on the stack and we don't need it again + lua_pushvalue(L, callback_idx); + lua_pushcclosure(L, http_accumulate_complete, 3); // context, cache table, callback + context_setref(L, context, CompleteCallback); + + // Finally, call request + lua_pushcfunction(L, http_lapi_request); + lua_pushvalue(L, -2); // context + lua_call(L, 1, 0); + + if (async) { + return 0; + } else { + // Have to return the data here. context is guaranteed still valid because + // we made sure to keep a reference to it on our stack + lua_pushinteger(L, context->status_code); + // Retrieve the data we stashed in context in http_accumulate_complete + lua_rawgeti(L, LUA_REGISTRYINDEX, context->refs[DataCallback]); + lua_rawgeti(L, LUA_REGISTRYINDEX, context->refs[HeadersCallback]); + return 3; + } +} + +// http.get(url, [options,] [callback]) +static int http_lapi_get(lua_State *L) +{ + luaL_checkstring(L, 1); + if (lua_isfunction(L, 2)) { + lua_pushnil(L); + lua_insert(L, 2); + } + lua_settop(L, 3); + if (lua_isnil(L, 2)) { + lua_newtable(L); + lua_replace(L, 2); + } + // Now 1 = url, 2 = non-nil options, 3 = [callback] + + luaL_argcheck(L, lua_istable(L, 2), 2, "options must be nil or a table"); + bool async = lua_isfunction(L, 3); + luaL_argcheck(L, lua_isnil(L, 3) || async, 3, "callback must be nil or a function"); + + // Override options.async based on whether callback present + lua_pushboolean(L, async); + lua_setfield(L, 2, "async"); + + // Setup call to createConnection + lua_pushcfunction(L, http_lapi_createConnection); + lua_pushvalue(L, 1); // url + lua_pushinteger(L, HTTP_METHOD_GET); + lua_pushvalue(L, 2); // options + + lua_call(L, 3, 1); // returns context + + return make_oneshot_request(L, 3); +} + +// http.post(url, options, body[, callback]) +static int http_lapi_post(lua_State *L) +{ + lua_settop(L, 4); + + luaL_checkstring(L, 1); + if (lua_isnil(L, 2)) { + lua_newtable(L); + lua_replace(L, 2); + } + // Now 1 = url, 2 = non-nil options, 3 = body, 4 = [callback] + + luaL_argcheck(L, lua_istable(L, 2), 2, "options must be nil or a table"); + luaL_checkstring(L, 3); + bool async = lua_isfunction(L, 4); + luaL_argcheck(L, lua_isnil(L, 4) || async, 4, "callback must be nil or a function"); + + // Override options.async based on whether callback present + lua_pushboolean(L, async); + lua_setfield(L, 2, "async"); + + // Setup call to createConnection + lua_pushcfunction(L, http_lapi_createConnection); + lua_pushvalue(L, 1); // url + lua_pushinteger(L, HTTP_METHOD_POST); + lua_pushvalue(L, 2); // options + + lua_call(L, 3, 1); // returns context + + lua_pushcfunction(L, http_lapi_setpostdata); + lua_pushvalue(L, -2); // context + lua_pushvalue(L, 3); // body + lua_call(L, 2, 0); + + return make_oneshot_request(L, 4); // 4 = callback idx +} + +static const LUA_REG_TYPE http_map[] = { + { LSTRKEY("createConnection"), LFUNCVAL(http_lapi_createConnection) }, + { LSTRKEY("GET"), LNUMVAL(HTTP_METHOD_GET) }, + { LSTRKEY("POST"), LNUMVAL(HTTP_METHOD_POST) }, + { LSTRKEY("DELETE"), LNUMVAL(HTTP_METHOD_DELETE) }, + { LSTRKEY("HEAD"), LNUMVAL(HTTP_METHOD_HEAD) }, + { LSTRKEY("DELAYACK"), LNUMVAL(DELAY_ACK) }, + { LSTRKEY("ACKNOW"), LNUMVAL(0) }, // Doesn't really matter what this is + { LSTRKEY("get"), LFUNCVAL(http_lapi_get) }, + { LSTRKEY("post"), LFUNCVAL(http_lapi_post) }, + { LNILKEY, LNILVAL } +}; + +static const LUA_REG_TYPE http_context_map[] = { + { LSTRKEY("on"), LFUNCVAL(http_lapi_on) }, + { LSTRKEY("request"), LFUNCVAL(http_lapi_request) }, + { LSTRKEY("setmethod"), LFUNCVAL(http_lapi_setmethod) }, + { LSTRKEY("setheader"), LFUNCVAL(http_lapi_setheader) }, + { LSTRKEY("seturl"), LFUNCVAL(http_lapi_seturl) }, + { LSTRKEY("setpostdata"), LFUNCVAL(http_lapi_setpostdata) }, + { LSTRKEY("close"), LFUNCVAL(context_close) }, + { LSTRKEY("ack"), LFUNCVAL(http_lapi_ack) }, + { LSTRKEY("__gc"), LFUNCVAL(context_gc) }, + { LSTRKEY("__index"), LROVAL(http_context_map) }, + { LNILKEY, LNILVAL } +}; + +static int luaopen_http(lua_State *L) +{ + luaL_rometatable(L, http_context_mt, (void *)http_context_map); + lhttp_request_task_id = task_get_id(lhttp_request_task); + lhttp_event_task_id = task_get_id(lhttp_event_task); + return 0; +} + +NODEMCU_MODULE(HTTP, "http", http_map, luaopen_http); diff --git a/docs/en/modules/http.md b/docs/en/modules/http.md new file mode 100644 index 00000000..1e4aac8b --- /dev/null +++ b/docs/en/modules/http.md @@ -0,0 +1,248 @@ +# HTTP Module +| Since | Origin / Contributor | Maintainer | Source | +| :----- | :-------------------- | :---------- | :------ | +| 2018-10-27 | [Tom Sutcliffe](https://github.com/tomsci) | [Tom Sutcliffe](https://github.com/tomsci) | [http.c](../../../components/modules/http.c)| + +HTTP *client* module that provides an interface to do GET/POST/PUT/DELETE over HTTP and HTTPS, as well as customized requests. It can support large requests with an API similar to that of the `net` module. Multiple concurrent HTTP requests are supported. Both synchronous and asynchronous modes are supported. + +For each operation it is possible to provide custom HTTP headers or override standard headers. By default the `Host` header is deduced from the URL and `User-Agent` is `ESP32 HTTP Client/1.0`. Requests are always sent as `HTTP/1.1`. Keep-alive is supported (unless using the one-shot APIs) by default, disable by adding a `Connection: close` header or by explicitly closing the connection once complete. + +HTTP redirects (HTTP status 300-308) are followed automatically up to a limit of 10 to avoid redirect loops. This behavior may be customized by setting the `max_redirects` option. + +Whenever headers are returned or passed into a callback, the header names are always lower cased. If there are multiple headers of the same name, then only the last one is returned. + +## http.createConnection() + +Creates a connection object which can be configured and then executed. Note this function does not actually open the connection to the remote server, that doesn't happen until [`connection:request()`](#connectionrequest) is called. + +#### Syntax +`http.createConnection(url, [method,] [options])` + +#### Parameters +- `url` The URL to fetch, including the `http://` or `https://` prefix. Required. +- `method` The HTTP method to use, one of `http.GET`, `http.POST`, `http.DELETE` or `http.HEAD`. Optional and may be omitted, the default is `http.GET`. +- `options` An optional table containing any or all of: + - `async` If true, the request is processed asynchronously, meaning [`request()`](#connectionrequest) returns immediately rather than blocking until the connection is complete and all callbacks have been made. Some other connection APIs behave differently in asynchronous mode, see their documentation for details. If not specified, the default is `false`, meaning requests are processed synchronously. + - `bufsz` The size in bytes of the temporary buffer used for reading data. If not specified, the default is `512`. + - `cert` A [PEM-encoded](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) certificate (or certificates). If specified, the server's TLS certificate must chain back to one of these root or intermediate certificates otherwise the request will fail. This option is ignored for HTTP requests (unless they redirect to an HTTPS URL). + - `headers` Table of headers to add to the request. + - `max_redirects` Maximum number of `30x` redirects to follow before giving up. If not specified, the default is `10`. Specify `0` to disable following redirects entirely. + - `timeout` Network timeout, in milliseconds. If not specified, the default is `10000` (10 seconds). + +#### Returns +The connection object. + +#### Example +```lua +headers = { + Connection = "close", + ["If-Modified-Since"] = "Sat, 27 Oct 2018 00:00:00 GMT", +} +connection = http.createConnection("http://httpbin.org/", http.GET, { headers=headers } ) +connection:on("complete", function(status) + print("Request completed with status code =", status) +end) +connection:request() +``` + +# http connection objects + +## connection:on() +Set a callback to be called when a certain event occurs. + +#### Syntax +`connection:on(event[, callback])` + +#### Parameters +- `event` One of + - `"connect"` Called when the connection is first established. Callback receives no arguments. + - `"headers"` Called once the HTTP headers from the remote end have been received. Callback is called as `callback(status_code, headers_table)`. + - `"data"` Can be called multiple times, each time more (non-headers) data is received. Callback is called as `callback(status_code, data)`. + - `"complete"` Called once all data has been received. Callback is called as `callback(status_code, connected)` where `connected` is `true` if the connection is still open. +- `callback` a function to be called when the given event occurs. Can be `nil` to remove a previously configured callback. + +If an error occurs, the `status_code` in the callback will be a negative number. The only callback guaranteed to be called in an error situation is `complete`. + +If the connection is asynchronous, the `data` callback may optionally return the constant `http.DELAYACK` to indicate that no further callbacks should be made until [`connection:ack()`](#connectionack) is called. This is useful to slow down the arrival of data in cases where the system is not ready to receive any more. If desired, non-delayed acks may be indicated by returning `http.ACKNOW` but this is not required since it's the default behaviour anyway. + +Note, you must not attempt to reuse `connection` until the `complete` callback has been called, or (in synchronous mode) until `request()` returns. You may reuse the connection from within the `complete` callback (in either mode). You may call `connection:close()` from within any callback to indicate that the connection should be aborted; in any callback other than `complete`, this will result in a subsequent `complete` callback. + +#### Returns +`nil` + +#### Example +See [`http.createConnection()`](#httpcreateconnection). + +## connection:request() +Opens the connection to the server and issues the request. If `async` was set to `true` in the connection options, then this function returns immediately. Otherwise, the function doesn't return until all callbacks have been made and the request is complete. If the server supports Keep-Alive and `Connection = "close"` wasn't specified in the headers, the connection will remain open after completion, ready for another call to `request()` (possibly after setting a new URL with `connection:seturl()`). + +#### Syntax +`connection:request()` + +#### Parameters +None + +#### Returns +In asynchronous mode, always returns `nil`. + +In synchronous mode, it returns 2 results, `status_code, connected` where `connected` is `true` if the connection is still open. In that case, if you don't wish to use the socket again, you should call `connection:close()` to ensure the socket is not kept open unnecessarily. When a connection is garbage collected any remaining sockets will be closed. + +## connection:setmethod() +Sets the connection method. Useful if making multiple requests of different types on a single connection. Errors if called while a request is in progress. + +#### Syntax +`connection:setmethod(method)` + +#### Parameters +- `method` one of `http.GET`, `http.POST`, `http.HEAD`, `http.DELETE`. + +#### Returns +`nil` + +#### Example +```lua +connection:setmethod(http.POST) +``` + +## connection:seturl() +Sets the connection URL. Useful if making multiple requests on a single connection. Errors if called while a request is in progress. + +#### Syntax +`connection:seturl(url)` + +#### Parameters +- `url` Required. The URL to use for the next `request()` call. + +#### Returns +`nil` + +#### Example +```lua +connection = http.createConnection("http://httpbin.org/") +connection:request() +-- Make another request +connection:seturl("http://httpbin.org/ip") +connection:request() +``` + +## connection:setheader() +Sets an individual header in the request. Header names are case-insensitive, but it is recommended to match the case of any headers automatically added by the underlying library (for example: `Connection`, `Content-Type`, `User-Agent` etc). Errors if called while a request is in progress. + +#### Syntax +`connection:setheader(name[, value])` + +#### Parameters +- `name` name of the header to set. +- `value` what to set it to. Must be a string, or `nil` to unset it. + +## connection:setpostdata() +Sets the POST data to be used for this request. Also sets the method to `http.POST` if it isn't already. If a `Content-Type` header has not already been set, also sets that to `application/x-www-form-urlencoded`. Errors if called while a request is in progress. + +#### Syntax +`connection:setpostdata([data])` + +#### Parameters +`data` - The data to POST. Unless a custom `Content-Type` header has been set, this data should be in `application/x-www-form-urlencoded` format. Can be `nil` to unset what to post and the `Content-Type` header. + +#### Returns +`nil` + +## connection:ack() +Completes a callback that was previously delayed by returning `http.DELAYACK`. This unblocks the request and allows subsequent callbacks to occur. Errors if the most recent callback wasn't an asynchronous one that was delayed. + +#### Syntax +`connection:ack()` + +#### Parameters +None + +#### Returns +`nil` + +## connection:close() +Closes the connection if it is still open. Note this does not reset any configured URL or callbacks on the connection object, it just closes the underlying TCP/IP socket. If called from within a callback, this aborts the connection, but the connection object must not be reused until the `complete` callback occurs (waiting until after `request()` returns is also allowed, if the request was synchronous). In the case of a connection in asynchronous mode, will error if called from outside of a callback while a request is ongoing, _unless_ the last callback returned `http.DELAYACK` _and_ `connection:ack()` has not yet been called. In such a situation, `connection:ack()` must still be called, after the call to `connection:close()`. An asynchronous connection that is still open but not ongoing (ie the `complete` callback has taken place) may be closed without restriction. + +#### Syntax +`connection:close()` + +#### Parameters +None + +#### Returns +`nil` + +# One shot HTTP APIs +These APIs are wrappers around the connection object based APIs above. They allow simpler code to be written for one-off requests where complex connection object management is not required. Note however that they buffer the incoming data and therefore are limited by free memory in how large a request they can handle. The EGC can also artificially limit this - try setting `node.egc.setmode(node.egc.ON_ALLOC_FAILURE)` for more optimal memory management behavior. If however there is physically not enough RAM to buffer the entire request, then the connection object based APIs must be used instead so each individual `data` callback can be processed separately. + +All one-shot APIs add the header `Connection: close` regardless of what the `options.headers` parameter contains, and can be executed either synchronously or asynchronously, depending on whether a `callback` is specified, regardless of any `options.async` setting. + +* Synchronous mode: The call to `get()`/`post()` does not return until the request is complete, and the results of the request are returned from the call. +* Asynchronous mode: The call returns immediately (with no results), the results of the request are given as arguments to the `callback` function. + +If more advanced control over the request is required, the connection object based APIs [`http.createConnection()`](#httpcreateconnection) and [`connection:request()`](#connectionrequest) should be used instead. + +## http.get() +Make an HTTP GET request. If a `callback` is specifed then the function operates in asynchronous mode, otherwise it is synchronous. + +#### Syntax +`http.get(url, [options,] [callback])` + +#### Parameters +- `url` The URL to fetch, including the `http://` or `https://` prefix +- `options` Same options as [`http.createConnection()`](#httpcreateconnection), except that `async` is set for you based on whether a `callback` is specified or not. May be `nil` or omitted. +- `callback` Should be `nil` or omitted to specify synchronous behaviour, otherwise a callback function to be invoked when the response has been received or an error occurred, which is called with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` will be a negative number. + +#### Returns +In synchronous mode, returns 3 results `status_code, body, headers` once the request has completed. In asynchronous mode, returns `nil` immediately. + +#### Example +```lua +-- Asynchronous mode +http.get("http://httpbin.org/ip", function(code, data) + if (code < 0) then + print("HTTP request failed") + else + print(code, data) + end + end) + +-- Synchronous mode +code, data = http.get("http://httpbin.org/ip") +if (code < 0) then + print("HTTP request failed") +else + print(code, data) +end +``` + +## http.post() + +Executes a single HTTP POST request and closes the connection. If a `callback` is specifed then the function operates in asynchronous mode, otherwise it is synchronous. + +#### Syntax +`http.post(url, options, body[, callback])` + +#### Parameters +- `url` The URL to fetch, including the `http://` or `https://` prefix +- `options` Same options as [`http.createConnection()`](#httpcreateconnection), except that `async` is set for you based on whether a `callback` is specified or not. May be `nil`. +- `body` The body to post. Required and must already be encoded in the appropriate format, but may be empty. See [`connection:setpostdata()`](#connectionsetpostdata) for more information. +- `callback` Should be `nil` or omitted to specify synchronous mode, otherwise a callback function to be invoked when the response has been received or an error occurred, which is called with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` will be a negative number. + +#### Returns +In synchronous mode, returns 3 results `status_code, body, headers` once the request has completed. In asynchronous mode, returns `nil` immediately. + +#### Example +```lua +headers = { + ["Content-Type"] = "application/json", +} +body = '{"hello":"world"}' +http.post("http://httpbin.org/post", headers, body, + function(code, data) + if (code < 0) then + print("HTTP request failed") + else + print(code, data) + end + end) +``` diff --git a/lua_examples/http-client.lua b/lua_examples/http-client.lua deleted file mode 100644 index 5041509a..00000000 --- a/lua_examples/http-client.lua +++ /dev/null @@ -1,40 +0,0 @@ --- Support HTTP and HTTPS, For example --- HTTP POST Example with JSON header and body -http.post("http://somewhere.acceptjson.com/", - "Content-Type: application/json\r\n", - "{\"hello\":\"world\"}", - function(code, data) - print(code) - print(data) - end) - --- HTTPS GET Example with NULL header -http.get("https://www.vowstar.com/nodemcu/","", - function(code, data) - print(code) - print(data) - end) --- You will get --- > 200 --- hello nodemcu - --- HTTPS DELETE Example with NULL header and body -http.delete("https://10.0.0.2:443","","", - function(code, data) - print(code) - print(data) - end) - --- HTTPS PUT Example with NULL header and body -http.put("https://testput.somewhere/somewhereyouput.php","","", - function(code, data) - print(code) - print(data) - end) - --- HTTP RAW Request Example, use more HTTP/HTTPS request method -http.request("http://www.apple.com:80/library/test/success.html","GET","","", - function(code, data) - print(code) - print(data) - end) diff --git a/lua_examples/httptest.lua b/lua_examples/httptest.lua new file mode 100644 index 00000000..a426455a --- /dev/null +++ b/lua_examples/httptest.lua @@ -0,0 +1,151 @@ +--dofile"httptest.lua" + +function assertEquals(a, b) + if a ~= b then + error(string.format("%q not equal to %q", tostring(a), tostring(b)), 2) + end +end + +-- Skipping test because my internet connection always resolves invalid DNS... +function SKIP_test_bad_dns() + do return end + + local c = http.createConnection("http://nope.invalid") + local seenConnect, seenFinish + c:on("connect", function() + seenConnect = true + end) + c:on("complete", function(statusCode) + seenFinish = statusCode + end) + local status, connected = c:request() + assert(status < 0) + assertEquals(connected, false) + assertEquals(seenConnect, false) + assert(seenFinish < 0) +end + +function test_simple() + local c = http.createConnection("http://httpbin.org/", { headers = { Connection = "close" } }) + local seenConnect, seenHeaders, headersStatusCode + local dataBytes = 0 + c:on("connect", function() seenConnect = true end) + c:on("headers", function(statusCode, headers) + headersStatusCode = statusCode + seenHeaders = headers + end) + c:on("data", function(statusCode, data) + assertEquals(statusCode, headersStatusCode) + dataBytes = dataBytes + #data + end) + + local status, connected = c:request() + + assertEquals(seenConnect, true) + assertEquals(headersStatusCode, 200) + assert(seenHeaders ~= nil) + assert(dataBytes > 0) + assertEquals(status, 200) + assertEquals(connected, false) +end + +function test_keepalive() + local c = http.createConnection("http://httpbin.org/") + local seenConnect, seenHeaders, seenData, seenFinish + c:on("connect", function() seenConnect = true end) + c:on("headers", function(status, headers) + seenHeaders = headers + end) + c:on("data", function(status, data) + seenData = data + end) + c:on("complete", function(status) + seenFinish = status + end) + + local status, connected = c:request() + assertEquals(seenConnect, true) + assertEquals(status, 200) + assertEquals(connected, true) + assert(seenHeaders) + assertEquals(seenFinish, 200) + + c:seturl("http://httpbin.org/user-agent") + c:setheader("Connection", "close") + seenConnect = false + seenData = nil + seenHeaders = nil + seenFinish = nil + status, connected = c:request() + assertEquals(status, 200) + assertEquals(connected, false) + assertEquals(seenConnect, false) -- You don't get another connect callback + assert(seenHeaders) -- But you do get new headers + assertEquals(seenData, '{\n "user-agent": "ESP32 HTTP Client/1.0"\n}\n') + assertEquals(seenFinish, 200) + c:close() +end + +function test_oneshot_get() + local options = { + headers = { + ["user-agent"] = "ESP32 HTTP Module User Agent test", + }, + } + local code, data = http.get("http://httpbin.org/user-agent", options) + assertEquals(code, 200) + assertEquals(data, '{\n "user-agent": "ESP32 HTTP Module User Agent test"\n}\n') + + -- And async version + print("test_oneshot_get async starting") + http.get("http://httpbin.org/user-agent", options, function(code, data) + assertEquals(code, 200) + assertEquals(data, '{\n "user-agent": "ESP32 HTTP Module User Agent test"\n}\n') + print("test_oneshot_get async completed") + end) +end + +function test_404() + local status = http.get("http://httpbin.org/status/404") + assertEquals(status, 404) +end + +function test_post() + local status, data = http.post("http://httpbin.org/post", nil, "hello=world&answer=42") + assertEquals(status, 200) + assert(data:match('"hello": "world"')) + assert(data:match('"answer": "42"')) + + status, data = http.post("http://httpbin.org/post", {headers = {["Content-Type"] = "application/json"}}, '{"hello":"world"}') + assertEquals(status, 200) + assert(data:match('"hello": "world"')) +end + +function test_redirects() + local status = http.createConnection("http://httpbin.org/redirect/3", { max_redirects = 0 }):request() + assertEquals(status, -28673) -- -ESP_ERR_HTTP_MAX_REDIRECT + + status = http.createConnection("http://httpbin.org/redirect/3", { max_redirects = 2 }):request() + assertEquals(status, -28673) + + status = http.createConnection("http://httpbin.org/redirect/3"):request() + assertEquals(status, 200) +end + +-- Doesn't seem to work... +function SKIP_test_timeout() + local status = http.createConnection("http://httpbin.org/delay/10", { timeout = 2000 }):request() + assertEquals(status, -1) +end + +local tests = {} +for k, v in pairs(_G) do + if type(v) == "function" and k:match("^test_") then + table.insert(tests, k) + end +end +table.sort(tests) +for _, t in ipairs(tests) do + print(t) + _G[t]() +end diff --git a/mkdocs.yml b/mkdocs.yml index af0bf086..90ab83eb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,8 +40,9 @@ pages: - 'encoder': 'en/modules/encoder.md' - 'file': 'en/modules/file.md' - 'gpio': 'en/modules/gpio.md' + - 'http': 'en/modules/http.md' - 'i2c': 'en/modules/i2c.md' - - 'i2s': 'en/modules/i2s.md' + - 'i2s': 'en/modules/i2s.md' - 'ledc': 'en/modules/ledc.md' - 'mqtt': 'en/modules/mqtt.md' - 'net': 'en/modules/net.md'