ESP32: fix memory leak in encoder.fromHex and avoid use luaM_free (#2610)

components/modules/encoder.c

@@ -18,7 +18,8 @@ static uint8_t *toBase64 ( lua_State* L, const uint8_t *msg, size_t *len){
   if (!n)  // handle empty string case 
     return NULL;
 
-  uint8_t * q, *out = (uint8_t *)luaM_malloc(L, (n + 2) / 3 * 4);
+  int buf_size=(n + 2) / 3 * 4; // estimated encoded size
+  uint8_t * q, *out = (uint8_t *)luaM_malloc(L, buf_size);
   uint8_t bytes64[sizeof(b64)];
   memcpy(bytes64, b64, sizeof(b64));   //Avoid lots of flash unaligned fetches
   
@@ -32,6 +33,7 @@ static uint8_t *toBase64 ( lua_State* L, const uint8_t *msg, size_t *len){
     *q++ = (i + 2 < n) ? bytes64[(c & 63)] : BASE64_PADDING;
   }
   *len = q - out;
+  out = luaM_realloc_(L, out, buf_size, *len); //reallocate to actual encoded length
   return out;
 }
 
@@ -57,7 +59,8 @@ static uint8_t *fromBase64 ( lua_State* L, const uint8_t *enc_msg, size_t *len){
  for (i = 0; i < n - pad; i++) if (!ISBASE64(enc_msg[i])) luaL_error (L, "Invalid base64 string");
   unbytes64[BASE64_PADDING] = 0;
 
-  msg = q = (uint8_t *) luaM_malloc(L, 1+ (3 * n / 4)); 
+  int buf_size=1+ (3 * n / 4); // estimate decoded length
+  msg = q = (uint8_t *) luaM_malloc(L, buf_size); 
   for (i = 0, p = enc_msg; i<blocks; i++)  {
     uint8_t a = unbytes64[*p++]; 
     uint8_t b = unbytes64[*p++]; 
@@ -75,6 +78,7 @@ static uint8_t *fromBase64 ( lua_State* L, const uint8_t *enc_msg, size_t *len){
     if (pad == 1) *q++ = (b << 4) | (unbytes64[*p] >> 2);
   }
   *len = q - msg;
+  msg = luaM_realloc_(L, msg, buf_size, *len); //reallocate to actual decoded length
   return msg;
 }
 
@@ -84,24 +88,26 @@ static inline uint8_t to_hex_nibble(uint8_t b) {
     
 static uint8_t *toHex ( lua_State* L, const uint8_t *msg, size_t *len){
   int i, n = *len;
-  uint8_t *q, *out = (uint8_t *)luaM_malloc(L, n * 2);
+  *len <<= 1;
+  uint8_t *q, *out = (uint8_t *)luaM_malloc(L, *len);
   for (i = 0, q = out; i < n; i++) {
     *q++ = to_hex_nibble(msg[i] >> 4);
     *q++ = to_hex_nibble(msg[i] & 0xf);
   }
-  *len = 2*n; 
   return out;
 }
 
 static uint8_t *fromHex ( lua_State* L, const uint8_t *msg, size_t *len){
   int i, n = *len;
   const uint8_t *p;
-  uint8_t b, *q, *out = (uint8_t *)luaM_malloc(L, n * 2);
-  uint8_t c = 0;
   
   if (n &1)
     luaL_error (L, "Invalid hex string");
 
+  *len >>= 1;
+  uint8_t b, *q, *out = (uint8_t *)luaM_malloc(L, *len);
+  uint8_t c = 0;
+ 
   for (i = 0, p = msg, q = out; i < n; i++) {
      if (*p >= '0' && *p <= '9') {
        b = *p++ - '0';
@@ -110,6 +116,7 @@ static uint8_t *fromHex ( lua_State* L, const uint8_t *msg, size_t *len){
      } else if (*p >= 'A' && *p <= 'F') {
        b = *p++ - ('A' - 10);
      } else {
+       luaM_freearray(L, out, *len, uint8_t);
        luaL_error (L, "Invalid hex string");
        __builtin_unreachable ();
      }
@@ -119,7 +126,6 @@ static uint8_t *fromHex ( lua_State* L, const uint8_t *msg, size_t *len){
        *q++ = c+ b;
      }    
   }
-  *len = n>>1; 
   return out;
 }
 
@@ -128,14 +134,13 @@ static uint8_t *fromHex ( lua_State* L, const uint8_t *msg, size_t *len){
 // Where input string maybe empty, but not nil
 // Hence these all call the do_func wrapper
 static int do_func (lua_State *L, uint8_t * (*conv_func)(lua_State *, const uint8_t *, size_t *)) {
-  size_t l;
-  const uint8_t *input = (const uint8_t *)luaL_checklstring(L, 1, &l);
-//  luaL_argcheck(L, l>0, 1, "input string empty");
-  uint8_t *output = conv_func(L, input, &l);
+  size_t len;
+  const uint8_t *input = (const uint8_t *)luaL_checklstring(L, 1, &len);
+  uint8_t *output = conv_func(L, input, &len);
   
   if (output) {
-    lua_pushlstring(L, (char *)output, l);
-    luaM_free(L, output);
+    lua_pushlstring(L, (char *)output, len);
+    luaM_freearray(L, output, len, uint8_t);
   } else {
     lua_pushstring(L, "");
   }