HTTP module can now chain requests (#1629)

* Fix up the HTTP module to use less memory

app/http/httpclient.c

@@ -402,9 +402,20 @@ static void ICACHE_FLASH_ATTR http_disconnect_callback( void * arg )
 		}
 		if ( req->callback_handle != NULL ) /* Callback is optional. */
 		{
-			req->callback_handle( body, http_status, req->buffer );
+                  char *req_buffer = req->buffer;
-		}
+                  req->buffer = NULL;
-		http_free_req( req );
+                  http_callback_t req_callback;
+                  req_callback = req->callback_handle;
+
+		  http_free_req( req );
+
+                  req_callback( body, http_status, &req_buffer );
+                  if (req_buffer) {
+                    os_free(req_buffer);
+                  }
+		} else {
+		  http_free_req( req );
+                }
 	}
 	/* Fix memory leak. */
 	espconn_delete( conn );
@@ -449,7 +460,7 @@ static void ICACHE_FLASH_ATTR http_dns_callback( const char * hostname, ip_addr_
 		HTTPCLIENT_ERR( "DNS failed for %s", hostname );
 		if ( req->callback_handle != NULL )
 		{
-			req->callback_handle( "", -1, "" );
+			req->callback_handle( "", -1, NULL );
 		}
 		http_free_req( req );
 	}

app/http/httpclient.h

@@ -53,7 +53,7 @@ static const char log_prefix[] = "HTTP client: ";
  * A successful request corresponds to an HTTP status code of 200 (OK).
  * More info at http://en.wikipedia.org/wiki/List_of_HTTP_status_codes
  */
-typedef void (* http_callback_t)(char * response_body, int http_status, char * full_response);
+typedef void (* http_callback_t)(char * response_body, int http_status, char ** full_response_p);
 
 /*
  * Call this function to skip URL parsing if the arguments are already in separate variables.

app/modules/http.c

@@ -3,6 +3,7 @@
  * vowstar@gmail.com
  * 2015-12-29
 *******************************************************************************/
+#include <c_stdlib.h>
 #include "module.h"
 #include "lauxlib.h"
 #include "platform.h"
@@ -11,8 +12,10 @@
 
 static int http_callback_registry  = LUA_NOREF;
 
-static void http_callback( char * response, int http_status, char * full_response )
+static void http_callback( char * response, int http_status, char ** full_response_p )
 {
+  const char *full_response = full_response_p ? *full_response_p : NULL;
+
 #if defined(HTTPCLIENT_DEBUG_ON)
   dbg_printf( "http_status=%d\n", http_status );
   if ( http_status != HTTP_STATUS_GENERIC_ERROR )
@@ -33,15 +36,70 @@ static void http_callback( char * response, int http_status, char * full_respons
     if ( http_status != HTTP_STATUS_GENERIC_ERROR && response)
     {
       lua_pushstring(L, response);
+      lua_newtable(L);
+
+      const char *p = full_response;
+
+      // Need to skip the HTTP/1.1 header line
+      while (*p && *p != '\n') {
+        p++;
+      }
+      if (*p == '\n') {
+        p++;
+      }
+
+      while (*p && *p != '\r' && *p != '\n') {
+        const char *eol = p;
+        while (*eol && *eol != '\r') {
+          eol++;
+        }
+
+        const char *colon = p;
+        while (*colon != ':' && colon < eol) {
+          colon++;
+        }
+
+        if (*colon != ':') {
+          break;
+        }
+
+        const char *value = colon + 1;
+        while (*value == ' ') {
+          value++;
+        }
+
+        luaL_Buffer b;
+        luaL_buffinit(L, &b);
+        while (p < colon) {
+          luaL_addchar(&b, tolower((unsigned char) *p));
+          p++;
+        }
+        luaL_pushresult(&b);
+
+        lua_pushlstring(L, value, eol - value);
+        lua_settable(L, -3);
+
+        p = eol + 1;
+        if (*p == '\n') {
+          p++;
+        }
+      }
     }
     else
     {
       lua_pushnil(L);
+      lua_pushnil(L);
+    }
+
+    if (full_response_p && *full_response_p) {
+      c_free(*full_response_p);
+      *full_response_p = NULL;
     }
-    lua_call(L, 2, 0); // With 2 arguments and 0 result
 
     luaL_unref(L, LUA_REGISTRYINDEX, http_callback_registry);
     http_callback_registry = LUA_NOREF;
+
+    lua_call(L, 3, 0); // With 3 arguments and 0 result
   }
 }
 
@@ -71,8 +129,7 @@ static int http_lapi_request( lua_State *L )
 
   if (lua_type(L, 5) == LUA_TFUNCTION || lua_type(L, 5) == LUA_TLIGHTFUNCTION) {
     lua_pushvalue(L, 5);  // copy argument (func) to the top of stack
-    if (http_callback_registry != LUA_NOREF)
+    luaL_unref(L, LUA_REGISTRYINDEX, http_callback_registry);
-      luaL_unref(L, LUA_REGISTRYINDEX, http_callback_registry);
     http_callback_registry = luaL_ref(L, LUA_REGISTRYINDEX);
   }
 

docs/en/modules/http.md

@@ -7,7 +7,7 @@ Basic HTTP *client* module that provides an interface to do GET/POST/PUT/DELETE
 
 !!! attention
 
-    It is **not** possible to execute concurrent HTTP requests using this module. Starting a new request before the previous has completed will result in undefined behavior. Use [`node.task.post()`](https://nodemcu.readthedocs.io/en/master/en/modules/node/#nodetaskpost) in the callbacks of your calls to start subsequent calls if you want to chain them (see [#1258](https://github.com/nodemcu/nodemcu-firmware/issues/1258)).
+    It is **not** possible to execute concurrent HTTP requests using this module. 
 
 Each request method takes a callback which is invoked when the response has been received from the server. The first argument is the status code, which is either a regular HTTP status code, or -1 to denote a DNS, connection or out-of-memory failure, or a timeout (currently at 10 seconds).
 
@@ -15,6 +15,9 @@ For each operation it is possible to provide custom HTTP headers or override sta
 
 HTTP redirects (HTTP status 300-308) are followed automatically up to a limit of 20 to avoid the dreaded redirect loops.
 
+When the callback is invoked, it is passed the HTTP status code, the body as it was received, and a table of the response headers. All the header names have been lower cased
+to make it easy to access. If there are multiple headers of the same name, then only the last one is returned.
+
 **SSL/TLS support**
 
 Take note of constraints documented in the [net module](net.md). 
@@ -30,7 +33,7 @@ Executes a HTTP DELETE request. Note that concurrent requests are not supported.
 - `url` The URL to fetch, including the `http://` or `https://` prefix
 - `headers` Optional additional headers to append, *including \r\n*; may be `nil`
 - `body` The body to post; must already be encoded in the appropriate format, but may be empty
-- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code` and `body`
+- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers`
 
 #### Returns
 `nil`
@@ -59,7 +62,7 @@ Executes a HTTP GET request. Note that concurrent requests are not supported.
 #### Parameters
 - `url` The URL to fetch, including the `http://` or `https://` prefix
 - `headers` Optional additional headers to append, *including \r\n*; may be `nil`
-- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code` and `body`
+- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers`
 
 #### Returns
 `nil`
@@ -86,7 +89,7 @@ Executes a HTTP POST request. Note that concurrent requests are not supported.
 - `url` The URL to fetch, including the `http://` or `https://` prefix
 - `headers` Optional additional headers to append, *including \r\n*; may be `nil`
 - `body` The body to post; must already be encoded in the appropriate format, but may be empty
-- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code` and `body`
+- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers`
 
 #### Returns
 `nil`
@@ -116,7 +119,7 @@ Executes a HTTP PUT request. Note that concurrent requests are not supported.
 - `url` The URL to fetch, including the `http://` or `https://` prefix
 - `headers` Optional additional headers to append, *including \r\n*; may be `nil`
 - `body` The body to post; must already be encoded in the appropriate format, but may be empty
-- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code` and `body`
+- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers`
 
 #### Returns
 `nil`
@@ -147,7 +150,7 @@ Execute a custom HTTP request for any HTTP method. Note that concurrent requests
 - `method` The HTTP method to use, e.g. "GET", "HEAD", "OPTIONS" etc
 - `headers` Optional additional headers to append, *including \r\n*; may be `nil`
 - `body` The body to post; must already be encoded in the appropriate format, but may be empty
-- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code` and `body`
+- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers`
 
 #### Returns
 `nil`