Make HTTP module follow redirects (#1450)

Fixes #1366

app/http/httpclient.c

@@ -20,6 +20,8 @@
 #include "httpclient.h"
 #include "stdlib.h"
 
+#define REDIRECTION_FOLLOW_MAX 20
+
 /* Internal state. */
 typedef struct request_args_t {
 	char		* hostname;
@@ -31,6 +33,7 @@ typedef struct request_args_t {
 	char		* post_data;
 	char		* buffer;
 	int		buffer_size;
+	int		redirect_follow_count;
 	int		timeout;
 	os_timer_t	timeout_timer;
 	http_callback_t callback_handle;
@@ -307,26 +310,92 @@ static void ICACHE_FLASH_ATTR http_disconnect_callback( void * arg )
 			else  
 			{
 				http_status	= atoi( req->buffer + strlen( version_1_0 ) );
-			        body            = (char *) os_strstr(req->buffer, "\r\n\r\n");
 
-				if (NULL == body) {
+				char *locationOffset = (char *) os_strstr( req->buffer, "Location:" );
-					  /* Find missing body */
+				if ( locationOffset == NULL ) {
-					  HTTPCLIENT_DEBUG("Body shouldn't be NULL\n");
+					locationOffset = (char *) os_strstr( req->buffer, "location:" );
-					  /* To avoid NULL body */
-					  body = "";
-				} else {                                        
-					  /* Skip CR & LF */
-					  body = body + 4;
 				}
-								   
+
-				if ( os_strstr( req->buffer, "Transfer-Encoding: chunked" ) )
+				if ( locationOffset != NULL && http_status >= 300 && http_status <= 308 ) {
-				{
+					if (req->redirect_follow_count < REDIRECTION_FOLLOW_MAX) {
-					int	body_size = req->buffer_size - (body - req->buffer);
+						locationOffset += strlen("location:");
-					char	chunked_decode_buffer[body_size];
+
-					os_memset( chunked_decode_buffer, 0, body_size );
+						while (*locationOffset == ' ') { // skip url leading white-space
-					/* Chuncked data */
+							locationOffset++;
-					http_chunked_decode( body, chunked_decode_buffer );
+						}
-					os_memcpy( body, chunked_decode_buffer, body_size );
+
+						char *locationOffsetEnd = (char *) os_strstr(locationOffset, "\r\n");
+						if ( locationOffsetEnd == NULL ) {
+							HTTPCLIENT_DEBUG( "Found Location header but was incomplete\n" );
+							http_status = -1;
+						} else {
+							*locationOffsetEnd = '\0';
+							req->redirect_follow_count++;
+
+							// Check if url is absolute
+							bool url_has_protocol =
+								os_strncmp( locationOffset, "http://", strlen( "http://" ) ) == 0 ||
+								os_strncmp( locationOffset, "https://", strlen( "https://" ) ) == 0;
+
+							if ( url_has_protocol ) {
+								http_request( locationOffset, req->method, req->headers,
+									req->post_data, req->callback_handle, req->redirect_follow_count );
+							} else {
+								if ( os_strncmp( locationOffset, "/", 1 ) == 0) { // relative and full path
+									http_raw_request( req->hostname, req->port, req->secure, req->method,
+										locationOffset, req->headers, req->post_data, req->callback_handle, req->redirect_follow_count );
+								} else { // relative and relative path
+
+									// find last /
+									const char *pathFolderEnd = strrchr(req->path, '/');
+
+									int pathFolderLength = pathFolderEnd - req->path;
+									pathFolderLength++; // use the '/'
+									int locationLength = strlen(locationOffset);
+									locationLength++; // use the '\0'
+
+									// append pathFolder with given relative path
+									char *completeRelativePath = (char *) os_malloc(pathFolderLength + locationLength);
+									os_memcpy( completeRelativePath, req->path, pathFolderLength );
+									os_memcpy( completeRelativePath + pathFolderLength, locationOffset, locationLength);
+
+									http_raw_request( req->hostname, req->port, req->secure, req->method,
+										completeRelativePath, req->headers, req->post_data, req->callback_handle, req->redirect_follow_count );
+
+									os_free( completeRelativePath );
+								}
+							}
+							http_free_req( req );
+							espconn_delete( conn );
+							os_free( conn );
+							return;
+						}
+					} else {
+						HTTPCLIENT_DEBUG("Too many redirections\n");
+						http_status = -1;
+					}
+				} else {
+					body = (char *) os_strstr(req->buffer, "\r\n\r\n");
+
+					if (NULL == body) {
+						  /* Find missing body */
+						  HTTPCLIENT_DEBUG("Body shouldn't be NULL\n");
+						  /* To avoid NULL body */
+						  body = "";
+					} else {
+						  /* Skip CR & LF */
+						  body = body + 4;
+					}
+
+					if ( os_strstr( req->buffer, "Transfer-Encoding: chunked" ) )
+					{
+						int	body_size = req->buffer_size - (body - req->buffer);
+						char	chunked_decode_buffer[body_size];
+						os_memset( chunked_decode_buffer, 0, body_size );
+						/* Chuncked data */
+						http_chunked_decode( body, chunked_decode_buffer );
+						os_memcpy( body, chunked_decode_buffer, body_size );
+					}
 				}
 			}
 		}
@@ -419,7 +488,7 @@ static void ICACHE_FLASH_ATTR http_dns_callback( const char * hostname, ip_addr_
 }
 
 
-void ICACHE_FLASH_ATTR http_raw_request( const char * hostname, int port, bool secure, const char * method, const char * path, const char * headers, const char * post_data, http_callback_t callback_handle )
+void ICACHE_FLASH_ATTR http_raw_request( const char * hostname, int port, bool secure, const char * method, const char * path, const char * headers, const char * post_data, http_callback_t callback_handle, int redirect_follow_count )
 {
 	HTTPCLIENT_DEBUG( "DNS request\n" );
 
@@ -436,6 +505,7 @@ void ICACHE_FLASH_ATTR http_raw_request( const char * hostname, int port, bool s
 	req->buffer[0]		= '\0';                                         /* Empty string. */
 	req->callback_handle	= callback_handle;
 	req->timeout		= HTTP_REQUEST_TIMEOUT_MS;
+	req->redirect_follow_count = redirect_follow_count;
 
 	ip_addr_t	addr;
 	err_t		error = espconn_gethostbyname( (struct espconn *) req,  /* It seems we don't need a real espconn pointer here. */
@@ -468,7 +538,7 @@ void ICACHE_FLASH_ATTR http_raw_request( const char * hostname, int port, bool s
  * <host> can be a hostname or an IP address
  * <port> is optional
  */
-void ICACHE_FLASH_ATTR http_request( const char * url, const char * method, const char * headers, const char * post_data, http_callback_t callback_handle )
+void ICACHE_FLASH_ATTR http_request( const char * url, const char * method, const char * headers, const char * post_data, http_callback_t callback_handle, int redirect_follow_count )
 {
 	/*
 	 * FIXME: handle HTTP auth with http://user:pass@host/
@@ -541,7 +611,7 @@ void ICACHE_FLASH_ATTR http_request( const char * url, const char * method, cons
 	HTTPCLIENT_DEBUG( "port=%d\n", port );
 	HTTPCLIENT_DEBUG( "method=%s\n", method );
 	HTTPCLIENT_DEBUG( "path=%s\n", path );
-	http_raw_request( hostname, port, secure, method, path, headers, post_data, callback_handle );
+	http_raw_request( hostname, port, secure, method, path, headers, post_data, callback_handle, redirect_follow_count);
 }
 
 
@@ -552,25 +622,25 @@ void ICACHE_FLASH_ATTR http_request( const char * url, const char * method, cons
  */
 void ICACHE_FLASH_ATTR http_post( const char * url, const char * headers, const char * post_data, http_callback_t callback_handle )
 {
-	http_request( url, "POST", headers, post_data, callback_handle );
+	http_request( url, "POST", headers, post_data, callback_handle, 0 );
 }
 
 
 void ICACHE_FLASH_ATTR http_get( const char * url, const char * headers, http_callback_t callback_handle )
 {
-	http_request( url, "GET", headers, NULL, callback_handle );
+	http_request( url, "GET", headers, NULL, callback_handle, 0 );
 }
 
 
 void ICACHE_FLASH_ATTR http_delete( const char * url, const char * headers, const char * post_data, http_callback_t callback_handle )
 {
-	http_request( url, "DELETE", headers, post_data, callback_handle );
+	http_request( url, "DELETE", headers, post_data, callback_handle, 0 );
 }
 
 
 void ICACHE_FLASH_ATTR http_put( const char * url, const char * headers, const char * post_data, http_callback_t callback_handle )
 {
-	http_request( url, "PUT", headers, post_data, callback_handle );
+	http_request( url, "PUT", headers, post_data, callback_handle, 0 );
 }
 
 

app/http/httpclient.h

@@ -51,15 +51,15 @@ typedef void (* http_callback_t)(char * response_body, int http_status, char * f
 /*
  * Call this function to skip URL parsing if the arguments are already in separate variables.
  */
-void ICACHE_FLASH_ATTR http_raw_request(const char * hostname, int port, bool secure, const char * method, const char * path, const char * headers, const char * post_data, http_callback_t callback_handle);
+void ICACHE_FLASH_ATTR http_raw_request(const char * hostname, int port, bool secure, const char * method, const char * path, const char * headers, const char * post_data, http_callback_t callback_handle, int redirect_follow_count);
 
 /*
  * Request data from URL use custom method.
  * The data should be encoded as any format.
  * Try:
- * http_request("http://httpbin.org/post", "OPTIONS", "Content-type: text/plain", "Hello world", http_callback_example);
+ * http_request("http://httpbin.org/post", "OPTIONS", "Content-type: text/plain", "Hello world", http_callback_example, 0);
  */
-void ICACHE_FLASH_ATTR http_request(const char * url, const char * method, const char * headers, const char * post_data, http_callback_t callback_handle);
+void ICACHE_FLASH_ATTR http_request(const char * url, const char * method, const char * headers, const char * post_data, http_callback_t callback_handle, int redirect_follow_count);
 
 /*
  * Post data to a web form.

app/modules/http.c

@@ -76,7 +76,7 @@ static int http_lapi_request( lua_State *L )
     http_callback_registry = luaL_ref(L, LUA_REGISTRYINDEX);
   }
 
-  http_request(url, method, headers, body, http_callback);
+  http_request(url, method, headers, body, http_callback, 0);
   return 0;
 }
 

docs/en/modules/http.md

@@ -13,6 +13,8 @@ Each request method takes a callback which is invoked when the response has been
 
 For each operation it is possible to provide custom HTTP headers or override standard headers. By default the `Host` header is deduced from the URL and `User-Agent` is `ESP8266`. Note, however, that the `Connection` header *can not* be overridden! It is always set to `close`.
 
+HTTP redirects (HTTP status 300-308) are followed automatically up to a limit of 20 to avoid the dreaded redirect loops.
+
 **SSL/TLS support**
 
 Take note of constraints documented in the [net module](net.md).