From e7f063950bf6f0bd267a0e9b491cc4d15ada2731 Mon Sep 17 00:00:00 2001 From: Philip Gladstone Date: Sun, 21 May 2017 10:16:39 -0400 Subject: [PATCH 01/27] Update to current version of SPIFFS (#1949) --- app/spiffs/spiffs.h | 124 ++++++- app/spiffs/spiffs_cache.c | 43 ++- app/spiffs/spiffs_check.c | 78 ++--- app/spiffs/spiffs_config.h | 95 ++++++ app/spiffs/spiffs_gc.c | 125 ++++--- app/spiffs/spiffs_hydrogen.c | 300 ++++++++++++++--- app/spiffs/spiffs_nucleus.c | 624 +++++++++++++++++++++++++---------- app/spiffs/spiffs_nucleus.h | 66 +++- 8 files changed, 1136 insertions(+), 319 deletions(-) diff --git a/app/spiffs/spiffs.h b/app/spiffs/spiffs.h index 6711cd06..d87422d5 100644 --- a/app/spiffs/spiffs.h +++ b/app/spiffs/spiffs.h @@ -56,6 +56,10 @@ extern "C" { #define SPIFFS_ERR_PROBE_NOT_A_FS -10035 #define SPIFFS_ERR_NAME_TOO_LONG -10036 +#define SPIFFS_ERR_IX_MAP_UNMAPPED -10037 +#define SPIFFS_ERR_IX_MAP_MAPPED -10038 +#define SPIFFS_ERR_IX_MAP_BAD_RANGE -10039 + #define SPIFFS_ERR_INTERNAL -10050 #define SPIFFS_ERR_TEST -10100 @@ -133,7 +137,7 @@ typedef void (*spiffs_file_callback)(struct spiffs_t *fs, spiffs_fileop_type op, #ifndef SPIFFS_DBG #define SPIFFS_DBG(...) \ - print(__VA_ARGS__) + printf(__VA_ARGS__) #endif #ifndef SPIFFS_GC_DBG #define SPIFFS_GC_DBG(...) printf(__VA_ARGS__) @@ -293,6 +297,9 @@ typedef struct { spiffs_obj_type type; spiffs_page_ix pix; u8_t name[SPIFFS_OBJ_NAME_LEN]; +#if SPIFFS_OBJ_META_LEN + u8_t meta[SPIFFS_OBJ_META_LEN]; +#endif } spiffs_stat; struct spiffs_dirent { @@ -301,6 +308,9 @@ struct spiffs_dirent { spiffs_obj_type type; u32_t size; spiffs_page_ix pix; +#if SPIFFS_OBJ_META_LEN + u8_t meta[SPIFFS_OBJ_META_LEN]; +#endif }; typedef struct { @@ -309,6 +319,21 @@ typedef struct { int entry; } spiffs_DIR; +#if SPIFFS_IX_MAP + +typedef struct { + // buffer with looked up data pixes + spiffs_page_ix *map_buf; + // precise file byte offset + u32_t offset; + // start data span index of lookup buffer + spiffs_span_ix start_spix; + // end data span index of lookup buffer + spiffs_span_ix end_spix; +} spiffs_ix_map; + +#endif + // functions #if SPIFFS_USE_MAGIC && SPIFFS_USE_MAGIC_LENGTH && SPIFFS_SINGLETON==0 @@ -506,6 +531,24 @@ s32_t SPIFFS_close(spiffs *fs, spiffs_file fh); */ s32_t SPIFFS_rename(spiffs *fs, const char *old, const char *newPath); +#if SPIFFS_OBJ_META_LEN +/** + * Updates file's metadata + * @param fs the file system struct + * @param path path to the file + * @param meta new metadata. must be SPIFFS_OBJ_META_LEN bytes long. + */ +s32_t SPIFFS_update_meta(spiffs *fs, const char *name, const void *meta); + +/** + * Updates file's metadata + * @param fs the file system struct + * @param fh file handle of the file + * @param meta new metadata. must be SPIFFS_OBJ_META_LEN bytes long. + */ +s32_t SPIFFS_fupdate_meta(spiffs *fs, spiffs_file fh, const void *meta); +#endif + /** * Returns last error of last file operation. * @param fs the file system struct @@ -658,6 +701,85 @@ s32_t SPIFFS_tell(spiffs *fs, spiffs_file fh); */ s32_t SPIFFS_set_file_callback_func(spiffs *fs, spiffs_file_callback cb_func); +#if SPIFFS_IX_MAP + +/** + * Maps the first level index lookup to a given memory map. + * This will make reading big files faster, as the memory map will be used for + * looking up data pages instead of searching for the indices on the physical + * medium. When mapping, all affected indicies are found and the information is + * copied to the array. + * Whole file or only parts of it may be mapped. The index map will cover file + * contents from argument offset until and including arguments (offset+len). + * It is valid to map a longer range than the current file size. The map will + * then be populated when the file grows. + * On garbage collections and file data page movements, the map array will be + * automatically updated. Do not tamper with the map array, as this contains + * the references to the data pages. Modifying it from outside will corrupt any + * future readings using this file descriptor. + * The map will no longer be used when the file descriptor closed or the file + * is unmapped. + * This can be useful to get faster and more deterministic timing when reading + * large files, or when seeking and reading a lot within a file. + * @param fs the file system struct + * @param fh the file handle of the file to map + * @param map a spiffs_ix_map struct, describing the index map + * @param offset absolute file offset where to start the index map + * @param len length of the mapping in actual file bytes + * @param map_buf the array buffer for the look up data - number of required + * elements in the array can be derived from function + * SPIFFS_bytes_to_ix_map_entries given the length + */ +s32_t SPIFFS_ix_map(spiffs *fs, spiffs_file fh, spiffs_ix_map *map, + u32_t offset, u32_t len, spiffs_page_ix *map_buf); + +/** + * Unmaps the index lookup from this filehandle. All future readings will + * proceed as normal, requiring reading of the first level indices from + * physical media. + * The map and map buffer given in function SPIFFS_ix_map will no longer be + * referenced by spiffs. + * It is not strictly necessary to unmap a file before closing it, as closing + * a file will automatically unmap it. + * @param fs the file system struct + * @param fh the file handle of the file to unmap + */ +s32_t SPIFFS_ix_unmap(spiffs *fs, spiffs_file fh); + +/** + * Moves the offset for the index map given in function SPIFFS_ix_map. Parts or + * all of the map buffer will repopulated. + * @param fs the file system struct + * @param fh the mapped file handle of the file to remap + * @param offset new absolute file offset where to start the index map + */ +s32_t SPIFFS_ix_remap(spiffs *fs, spiffs_file fh, u32_t offs); + +/** + * Utility function to get number of spiffs_page_ix entries a map buffer must + * contain on order to map given amount of file data in bytes. + * See function SPIFFS_ix_map and SPIFFS_ix_map_entries_to_bytes. + * @param fs the file system struct + * @param bytes number of file data bytes to map + * @return needed number of elements in a spiffs_page_ix array needed to + * map given amount of bytes in a file + */ +s32_t SPIFFS_bytes_to_ix_map_entries(spiffs *fs, u32_t bytes); + +/** + * Utility function to amount of file data bytes that can be mapped when + * mapping a file with buffer having given number of spiffs_page_ix entries. + * See function SPIFFS_ix_map and SPIFFS_bytes_to_ix_map_entries. + * @param fs the file system struct + * @param map_page_ix_entries number of entries in a spiffs_page_ix array + * @return amount of file data in bytes that can be mapped given a map + * buffer having given amount of spiffs_page_ix entries + */ +s32_t SPIFFS_ix_map_entries_to_bytes(spiffs *fs, u32_t map_page_ix_entries); + +#endif // SPIFFS_IX_MAP + + #if SPIFFS_TEST_VISUALISATION /** * Prints out a visualization of the filesystem. diff --git a/app/spiffs/spiffs_cache.c b/app/spiffs/spiffs_cache.c index ea9bc82c..018f7630 100644 --- a/app/spiffs/spiffs_cache.c +++ b/app/spiffs/spiffs_cache.c @@ -20,12 +20,12 @@ static spiffs_cache_page *spiffs_cache_page_get(spiffs *fs, spiffs_page_ix pix) if ((cache->cpage_use_map & (1<flags & SPIFFS_CACHE_FLAG_TYPE_WR) == 0 && cp->pix == pix ) { - SPIFFS_CACHE_DBG("CACHE_GET: have cache page %i for %04x\n", i, pix); + SPIFFS_CACHE_DBG("CACHE_GET: have cache page "_SPIPRIi" for "_SPIPRIpg"\n", i, pix); cp->last_access = cache->last_access; return cp; } } - //SPIFFS_CACHE_DBG("CACHE_GET: no cache for %04x\n", pix); + //SPIFFS_CACHE_DBG("CACHE_GET: no cache for "_SPIPRIpg"\n", pix); return 0; } @@ -46,9 +46,9 @@ static s32_t spiffs_cache_page_free(spiffs *fs, int ix, u8_t write_back) { cache->cpage_use_map &= ~(1 << ix); if (cp->flags & SPIFFS_CACHE_FLAG_TYPE_WR) { - SPIFFS_CACHE_DBG("CACHE_FREE: free cache page %i objid %04x\n", ix, cp->obj_id); + SPIFFS_CACHE_DBG("CACHE_FREE: free cache page "_SPIPRIi" objid "_SPIPRIid"\n", ix, cp->obj_id); } else { - SPIFFS_CACHE_DBG("CACHE_FREE: free cache page %i pix %04x\n", ix, cp->pix); + SPIFFS_CACHE_DBG("CACHE_FREE: free cache page "_SPIPRIi" pix "_SPIPRIpg"\n", ix, cp->pix); } } @@ -98,7 +98,7 @@ static spiffs_cache_page *spiffs_cache_page_allocate(spiffs *fs) { spiffs_cache_page *cp = spiffs_get_cache_page_hdr(fs, cache, i); cache->cpage_use_map |= (1<last_access = cache->last_access; - SPIFFS_CACHE_DBG("CACHE_ALLO: allocated cache page %i\n", i); + SPIFFS_CACHE_DBG("CACHE_ALLO: allocated cache page "_SPIPRIi"\n", i); return cp; } } @@ -130,10 +130,13 @@ s32_t spiffs_phys_rd( spiffs_cache_page *cp = spiffs_cache_page_get(fs, SPIFFS_PADDR_TO_PAGE(fs, addr)); cache->last_access++; if (cp) { + // we've already got one, you see #if SPIFFS_CACHE_STATS fs->cache_hits++; #endif cp->last_access = cache->last_access; + u8_t *mem = spiffs_get_cache_page(fs, cache, cp->ix); + memcpy(dst, &mem[SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr)], len); } else { if ((op & SPIFFS_OP_TYPE_MASK) == SPIFFS_OP_T_OBJ_LU2) { // for second layer lookup functions, we do not cache in order to prevent shredding @@ -142,22 +145,34 @@ s32_t spiffs_phys_rd( #if SPIFFS_CACHE_STATS fs->cache_misses++; #endif + // this operation will always free one cache page (unless all already free), + // the result code stems from the write operation of the possibly freed cache page res = spiffs_cache_page_remove_oldest(fs, SPIFFS_CACHE_FLAG_TYPE_WR, 0); + cp = spiffs_cache_page_allocate(fs); if (cp) { cp->flags = SPIFFS_CACHE_FLAG_WRTHRU; cp->pix = SPIFFS_PADDR_TO_PAGE(fs, addr); - } - s32_t res2 = SPIFFS_HAL_READ(fs, - addr - SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr), - SPIFFS_CFG_LOG_PAGE_SZ(fs), - spiffs_get_cache_page(fs, cache, cp->ix)); - if (res2 != SPIFFS_OK) { - res = res2; + + s32_t res2 = SPIFFS_HAL_READ(fs, + addr - SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr), + SPIFFS_CFG_LOG_PAGE_SZ(fs), + spiffs_get_cache_page(fs, cache, cp->ix)); + if (res2 != SPIFFS_OK) { + // honor read failure before possible write failure (bad idea?) + res = res2; + } + u8_t *mem = spiffs_get_cache_page(fs, cache, cp->ix); + memcpy(dst, &mem[SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr)], len); + } else { + // this will never happen, last resort for sake of symmetry + s32_t res2 = SPIFFS_HAL_READ(fs, addr, len, dst); + if (res2 != SPIFFS_OK) { + // honor read failure before possible write failure (bad idea?) + res = res2; + } } } - u8_t *mem = spiffs_get_cache_page(fs, cache, cp->ix); - memcpy(dst, &mem[SPIFFS_PADDR_TO_PAGE_OFFSET(fs, addr)], len); return res; } diff --git a/app/spiffs/spiffs_check.c b/app/spiffs/spiffs_check.c index 1222dea3..dde85eff 100644 --- a/app/spiffs/spiffs_check.c +++ b/app/spiffs/spiffs_check.c @@ -182,7 +182,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s if (((lu_obj_id == SPIFFS_OBJ_ID_DELETED) && (p_hdr->flags & SPIFFS_PH_FLAG_DELET)) || ((lu_obj_id == SPIFFS_OBJ_ID_FREE) && (p_hdr->flags & SPIFFS_PH_FLAG_USED) == 0)) { // look up entry deleted / free but used in page header - SPIFFS_CHECK_DBG("LU: pix %04x deleted/free in lu but not on page\n", cur_pix); + SPIFFS_CHECK_DBG("LU: pix "_SPIPRIpg" deleted/free in lu but not on page\n", cur_pix); *reload_lu = 1; delete_page = 1; if (p_hdr->flags & SPIFFS_PH_FLAG_INDEX) { @@ -199,14 +199,14 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s // copy page to new place and re-write the object index to new place spiffs_page_ix new_pix; res = spiffs_rewrite_page(fs, cur_pix, p_hdr, &new_pix); - SPIFFS_CHECK_DBG("LU: FIXUP: data page not found elsewhere, rewriting %04x to new page %04x\n", cur_pix, new_pix); + SPIFFS_CHECK_DBG("LU: FIXUP: data page not found elsewhere, rewriting "_SPIPRIpg" to new page "_SPIPRIpg"\n", cur_pix, new_pix); SPIFFS_CHECK_RES(res); *reload_lu = 1; - SPIFFS_CHECK_DBG("LU: FIXUP: %04x rewritten to %04x, affected objix_pix %04x\n", cur_pix, new_pix, objix_pix); + SPIFFS_CHECK_DBG("LU: FIXUP: "_SPIPRIpg" rewritten to "_SPIPRIpg", affected objix_pix "_SPIPRIpg"\n", cur_pix, new_pix, objix_pix); res = spiffs_rewrite_index(fs, p_hdr->obj_id, p_hdr->span_ix, new_pix, objix_pix); if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) { // index bad also, cannot mend this file - SPIFFS_CHECK_DBG("LU: FIXUP: index bad %i, cannot mend!\n", res); + SPIFFS_CHECK_DBG("LU: FIXUP: index bad "_SPIPRIi", cannot mend!\n", res); res = spiffs_page_delete(fs, new_pix); SPIFFS_CHECK_RES(res); res = spiffs_delete_obj_lazy(fs, p_hdr->obj_id); @@ -229,7 +229,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s // got a data page also, assume lu corruption only, rewrite to new page spiffs_page_ix new_pix; res = spiffs_rewrite_page(fs, cur_pix, p_hdr, &new_pix); - SPIFFS_CHECK_DBG("LU: FIXUP: ix page with data not found elsewhere, rewriting %04x to new page %04x\n", cur_pix, new_pix); + SPIFFS_CHECK_DBG("LU: FIXUP: ix page with data not found elsewhere, rewriting "_SPIPRIpg" to new page "_SPIPRIpg"\n", cur_pix, new_pix); SPIFFS_CHECK_RES(res); *reload_lu = 1; CHECK_CB(fs, SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix); @@ -242,7 +242,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s if (lu_obj_id != SPIFFS_OBJ_ID_FREE && lu_obj_id != SPIFFS_OBJ_ID_DELETED) { // look up entry used if ((p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG) != (lu_obj_id | SPIFFS_OBJ_ID_IX_FLAG)) { - SPIFFS_CHECK_DBG("LU: pix %04x differ in obj_id lu:%04x ph:%04x\n", cur_pix, lu_obj_id, p_hdr->obj_id); + SPIFFS_CHECK_DBG("LU: pix "_SPIPRIpg" differ in obj_id lu:"_SPIPRIid" ph:"_SPIPRIid"\n", cur_pix, lu_obj_id, p_hdr->obj_id); delete_page = 1; if ((p_hdr->flags & SPIFFS_PH_FLAG_DELET) == 0 || (p_hdr->flags & SPIFFS_PH_FLAG_FINAL) || @@ -265,7 +265,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s res = spiffs_rewrite_index(fs, p_hdr->obj_id, p_hdr->span_ix, new_pix, objix_pix); if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) { // index bad also, cannot mend this file - SPIFFS_CHECK_DBG("LU: FIXUP: index bad %i, cannot mend!\n", res); + SPIFFS_CHECK_DBG("LU: FIXUP: index bad "_SPIPRIi", cannot mend!\n", res); res = spiffs_page_delete(fs, new_pix); SPIFFS_CHECK_RES(res); res = spiffs_delete_obj_lazy(fs, p_hdr->obj_id); @@ -321,7 +321,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s // rewrite as obj_id_ph new_ph.obj_id = p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG; res = spiffs_rewrite_page(fs, cur_pix, &new_ph, &new_pix); - SPIFFS_CHECK_DBG("LU: FIXUP: rewrite page %04x as %04x to pix %04x\n", cur_pix, new_ph.obj_id, new_pix); + SPIFFS_CHECK_DBG("LU: FIXUP: rewrite page "_SPIPRIpg" as "_SPIPRIid" to pix "_SPIPRIpg"\n", cur_pix, new_ph.obj_id, new_pix); CHECK_CB(fs, SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix); SPIFFS_CHECK_RES(res); *reload_lu = 1; @@ -330,7 +330,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s // got a data page for look up obj id // rewrite as obj_id_lu new_ph.obj_id = lu_obj_id | SPIFFS_OBJ_ID_IX_FLAG; - SPIFFS_CHECK_DBG("LU: FIXUP: rewrite page %04x as %04x\n", cur_pix, new_ph.obj_id); + SPIFFS_CHECK_DBG("LU: FIXUP: rewrite page "_SPIPRIpg" as "_SPIPRIid"\n", cur_pix, new_ph.obj_id); CHECK_CB(fs, SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix); res = spiffs_rewrite_page(fs, cur_pix, &new_ph, &new_pix); SPIFFS_CHECK_RES(res); @@ -344,7 +344,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s } } else if (((lu_obj_id & SPIFFS_OBJ_ID_IX_FLAG) && (p_hdr->flags & SPIFFS_PH_FLAG_INDEX)) || ((lu_obj_id & SPIFFS_OBJ_ID_IX_FLAG) == 0 && (p_hdr->flags & SPIFFS_PH_FLAG_INDEX) == 0)) { - SPIFFS_CHECK_DBG("LU: %04x lu/page index marking differ\n", cur_pix); + SPIFFS_CHECK_DBG("LU: "_SPIPRIpg" lu/page index marking differ\n", cur_pix); spiffs_page_ix data_pix, objix_pix_d; // see if other data page exists for given obj id and span index res = spiffs_obj_lu_find_id_and_span(fs, lu_obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, p_hdr->span_ix, cur_pix, &data_pix); @@ -402,10 +402,10 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s } } else if ((p_hdr->flags & SPIFFS_PH_FLAG_DELET) == 0) { - SPIFFS_CHECK_DBG("LU: pix %04x busy in lu but deleted on page\n", cur_pix); + SPIFFS_CHECK_DBG("LU: pix "_SPIPRIpg" busy in lu but deleted on page\n", cur_pix); delete_page = 1; } else if ((p_hdr->flags & SPIFFS_PH_FLAG_FINAL)) { - SPIFFS_CHECK_DBG("LU: pix %04x busy but not final\n", cur_pix); + SPIFFS_CHECK_DBG("LU: pix "_SPIPRIpg" busy but not final\n", cur_pix); // page can be removed if not referenced by object index *reload_lu = 1; res = spiffs_object_get_data_page_index_reference(fs, lu_obj_id, p_hdr->span_ix, &ref_pix, &objix_pix); @@ -433,7 +433,7 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s } if (delete_page) { - SPIFFS_CHECK_DBG("LU: FIXUP: deleting page %04x\n", cur_pix); + SPIFFS_CHECK_DBG("LU: FIXUP: deleting page "_SPIPRIpg"\n", cur_pix); CHECK_CB(fs, SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_DELETE_PAGE, cur_pix, 0); res = spiffs_page_delete(fs, cur_pix); SPIFFS_CHECK_RES(res); @@ -530,7 +530,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { spiffs_page_ix cur_pix = SPIFFS_OBJ_LOOKUP_PAGES(fs) + SPIFFS_PAGES_PER_BLOCK(fs) * cur_block; while (!restart && cur_pix < SPIFFS_PAGES_PER_BLOCK(fs) * (cur_block+1)) { //if ((cur_pix & 0xff) == 0) - // SPIFFS_CHECK_DBG("PA: processing pix %08x, block %08x of pix %08x, block %08x\n", + // SPIFFS_CHECK_DBG("PA: processing pix "_SPIPRIpg", block "_SPIPRIbl" of pix "_SPIPRIpg", block "_SPIPRIbl"\n", // cur_pix, cur_block, SPIFFS_PAGES_PER_BLOCK(fs) * fs->block_count, fs->block_count); // read header @@ -589,7 +589,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { || (rpix_within_range && SPIFFS_IS_LOOKUP_PAGE(fs, rpix))) { // bad reference - SPIFFS_CHECK_DBG("PA: pix %04x bad pix / LU referenced from page %04x\n", + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg"x bad pix / LU referenced from page "_SPIPRIpg"\n", rpix, cur_pix); // check for data page elsewhere spiffs_page_ix data_pix; @@ -608,15 +608,15 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { new_ph.span_ix = data_spix_offset + i; res = spiffs_page_allocate_data(fs, new_ph.obj_id, &new_ph, 0, 0, 0, 1, &data_pix); SPIFFS_CHECK_RES(res); - SPIFFS_CHECK_DBG("PA: FIXUP: found no existing data page, created new @ %04x\n", data_pix); + SPIFFS_CHECK_DBG("PA: FIXUP: found no existing data page, created new @ "_SPIPRIpg"\n", data_pix); } // remap index - SPIFFS_CHECK_DBG("PA: FIXUP: rewriting index pix %04x\n", cur_pix); + SPIFFS_CHECK_DBG("PA: FIXUP: rewriting index pix "_SPIPRIpg"\n", cur_pix); res = spiffs_rewrite_index(fs, objix_p_hdr->obj_id | SPIFFS_OBJ_ID_IX_FLAG, data_spix_offset + i, data_pix, cur_pix); if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) { // index bad also, cannot mend this file - SPIFFS_CHECK_DBG("PA: FIXUP: index bad %i, cannot mend - delete object\n", res); + SPIFFS_CHECK_DBG("PA: FIXUP: index bad "_SPIPRIi", cannot mend - delete object\n", res); CHECK_CB(fs, SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, objix_p_hdr->obj_id, 0); // delete file res = spiffs_page_delete(fs, cur_pix); @@ -640,7 +640,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { rp_hdr.span_ix != data_spix_offset + i || (rp_hdr.flags & (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_USED)) != (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_INDEX)) { - SPIFFS_CHECK_DBG("PA: pix %04x has inconsistent page header ix id/span:%04x/%04x, ref id/span:%04x/%04x flags:%02x\n", + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg" has inconsistent page header ix id/span:"_SPIPRIid"/"_SPIPRIsp", ref id/span:"_SPIPRIid"/"_SPIPRIsp" flags:"_SPIPRIfl"\n", rpix, p_hdr.obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, data_spix_offset + i, rp_hdr.obj_id, rp_hdr.span_ix, rp_hdr.flags); // try finding correct page @@ -654,19 +654,19 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { SPIFFS_CHECK_RES(res); if (data_pix == 0) { // not found, this index is badly borked - SPIFFS_CHECK_DBG("PA: FIXUP: index bad, delete object id %04x\n", p_hdr.obj_id); + SPIFFS_CHECK_DBG("PA: FIXUP: index bad, delete object id "_SPIPRIid"\n", p_hdr.obj_id); CHECK_CB(fs, SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0); res = spiffs_delete_obj_lazy(fs, p_hdr.obj_id); SPIFFS_CHECK_RES(res); break; } else { // found it, so rewrite index - SPIFFS_CHECK_DBG("PA: FIXUP: found correct data pix %04x, rewrite ix pix %04x id %04x\n", + SPIFFS_CHECK_DBG("PA: FIXUP: found correct data pix "_SPIPRIpg", rewrite ix pix "_SPIPRIpg" id "_SPIPRIid"\n", data_pix, cur_pix, p_hdr.obj_id); res = spiffs_rewrite_index(fs, p_hdr.obj_id, data_spix_offset + i, data_pix, cur_pix); if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) { // index bad also, cannot mend this file - SPIFFS_CHECK_DBG("PA: FIXUP: index bad %i, cannot mend!\n", res); + SPIFFS_CHECK_DBG("PA: FIXUP: index bad "_SPIPRIi", cannot mend!\n", res); CHECK_CB(fs, SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0); res = spiffs_delete_obj_lazy(fs, p_hdr.obj_id); } else { @@ -681,12 +681,12 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { const u32_t rpix_byte_ix = (rpix - pix_offset) / (8/bits); const u8_t rpix_bit_ix = (rpix & ((8/bits)-1)) * bits; if (fs->work[rpix_byte_ix] & (1<<(rpix_bit_ix + 1))) { - SPIFFS_CHECK_DBG("PA: pix %04x multiple referenced from page %04x\n", + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg" multiple referenced from page "_SPIPRIpg"\n", rpix, cur_pix); // Here, we should have fixed all broken references - getting this means there // must be multiple files with same object id. Only solution is to delete // the object which is referring to this page - SPIFFS_CHECK_DBG("PA: FIXUP: removing object %04x and page %04x\n", + SPIFFS_CHECK_DBG("PA: FIXUP: removing object "_SPIPRIid" and page "_SPIPRIpg"\n", p_hdr.obj_id, cur_pix); CHECK_CB(fs, SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0); res = spiffs_delete_obj_lazy(fs, p_hdr.obj_id); @@ -725,7 +725,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { if (bitmask == 0x1) { // 001 - SPIFFS_CHECK_DBG("PA: pix %04x USED, UNREFERENCED, not index\n", cur_pix); + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg" USED, UNREFERENCED, not index\n", cur_pix); u8_t rewrite_ix_to_this = 0; u8_t delete_page = 0; @@ -741,7 +741,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { if (((rpix == (spiffs_page_ix)-1 || rpix > SPIFFS_MAX_PAGES(fs)) || (SPIFFS_IS_LOOKUP_PAGE(fs, rpix)))) { // pointing to a bad page altogether, rewrite index to this rewrite_ix_to_this = 1; - SPIFFS_CHECK_DBG("PA: corresponding ref is bad: %04x, rewrite to this %04x\n", rpix, cur_pix); + SPIFFS_CHECK_DBG("PA: corresponding ref is bad: "_SPIPRIpg", rewrite to this "_SPIPRIpg"\n", rpix, cur_pix); } else { // pointing to something else, check what spiffs_page_header rp_hdr; @@ -752,12 +752,12 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { ((rp_hdr.flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_FINAL)) == (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_DELET))) { // pointing to something else valid, just delete this page then - SPIFFS_CHECK_DBG("PA: corresponding ref is good but different: %04x, delete this %04x\n", rpix, cur_pix); + SPIFFS_CHECK_DBG("PA: corresponding ref is good but different: "_SPIPRIpg", delete this "_SPIPRIpg"\n", rpix, cur_pix); delete_page = 1; } else { // pointing to something weird, update index to point to this page instead if (rpix != cur_pix) { - SPIFFS_CHECK_DBG("PA: corresponding ref is weird: %04x %s%s%s%s, rewrite this %04x\n", rpix, + SPIFFS_CHECK_DBG("PA: corresponding ref is weird: "_SPIPRIpg" %s%s%s%s, rewrite this "_SPIPRIpg"\n", rpix, (rp_hdr.flags & SPIFFS_PH_FLAG_INDEX) ? "" : "INDEX ", (rp_hdr.flags & SPIFFS_PH_FLAG_DELET) ? "" : "DELETED ", (rp_hdr.flags & SPIFFS_PH_FLAG_USED) ? "NOTUSED " : "", @@ -770,19 +770,19 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { } } } else if (res == SPIFFS_ERR_NOT_FOUND) { - SPIFFS_CHECK_DBG("PA: corresponding ref not found, delete %04x\n", cur_pix); + SPIFFS_CHECK_DBG("PA: corresponding ref not found, delete "_SPIPRIpg"\n", cur_pix); delete_page = 1; res = SPIFFS_OK; } if (rewrite_ix_to_this) { // if pointing to invalid page, redirect index to this page - SPIFFS_CHECK_DBG("PA: FIXUP: rewrite index id %04x data spix %04x to point to this pix: %04x\n", + SPIFFS_CHECK_DBG("PA: FIXUP: rewrite index id "_SPIPRIid" data spix "_SPIPRIsp" to point to this pix: "_SPIPRIpg"\n", p_hdr.obj_id, p_hdr.span_ix, cur_pix); res = spiffs_rewrite_index(fs, p_hdr.obj_id, p_hdr.span_ix, cur_pix, objix_pix); if (res <= _SPIFFS_ERR_CHECK_FIRST && res > _SPIFFS_ERR_CHECK_LAST) { // index bad also, cannot mend this file - SPIFFS_CHECK_DBG("PA: FIXUP: index bad %i, cannot mend!\n", res); + SPIFFS_CHECK_DBG("PA: FIXUP: index bad "_SPIPRIi", cannot mend!\n", res); CHECK_CB(fs, SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_BAD_FILE, p_hdr.obj_id, 0); res = spiffs_page_delete(fs, cur_pix); SPIFFS_CHECK_RES(res); @@ -794,7 +794,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { restart = 1; continue; } else if (delete_page) { - SPIFFS_CHECK_DBG("PA: FIXUP: deleting page %04x\n", cur_pix); + SPIFFS_CHECK_DBG("PA: FIXUP: deleting page "_SPIPRIpg"\n", cur_pix); CHECK_CB(fs, SPIFFS_CHECK_PAGE, SPIFFS_CHECK_DELETE_PAGE, cur_pix, 0); res = spiffs_page_delete(fs, cur_pix); } @@ -803,7 +803,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { if (bitmask == 0x2) { // 010 - SPIFFS_CHECK_DBG("PA: pix %04x FREE, REFERENCED, not index\n", cur_pix); + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg" FREE, REFERENCED, not index\n", cur_pix); // no op, this should be taken care of when checking valid references } @@ -813,7 +813,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { if (bitmask == 0x4) { // 100 - SPIFFS_CHECK_DBG("PA: pix %04x FREE, unreferenced, INDEX\n", cur_pix); + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg" FREE, unreferenced, INDEX\n", cur_pix); // this should never happen, major fubar } @@ -823,14 +823,14 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { if (bitmask == 0x6) { // 110 - SPIFFS_CHECK_DBG("PA: pix %04x FREE, REFERENCED, INDEX\n", cur_pix); + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg" FREE, REFERENCED, INDEX\n", cur_pix); // no op, this should be taken care of when checking valid references } if (bitmask == 0x7) { // 111 - SPIFFS_CHECK_DBG("PA: pix %04x USED, REFERENCED, INDEX\n", cur_pix); + SPIFFS_CHECK_DBG("PA: pix "_SPIPRIpg" USED, REFERENCED, INDEX\n", cur_pix); // no op, this should be taken care of when checking valid references } @@ -838,7 +838,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { } } - SPIFFS_CHECK_DBG("PA: processed %04x, restart %i\n", pix_offset, restart); + SPIFFS_CHECK_DBG("PA: processed "_SPIPRIpg", restart "_SPIPRIi"\n", pix_offset, restart); // next page range if (!restart) { pix_offset += pages_per_scan; @@ -898,7 +898,7 @@ static s32_t spiffs_object_index_consistency_check_v(spiffs *fs, spiffs_obj_id o if (p_hdr.span_ix == 0 && (p_hdr.flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE)) == (SPIFFS_PH_FLAG_DELET)) { - SPIFFS_CHECK_DBG("IX: pix %04x, obj id:%04x spix:%04x header not fully deleted - deleting\n", + SPIFFS_CHECK_DBG("IX: pix "_SPIPRIpg", obj id:"_SPIPRIid" spix:"_SPIPRIsp" header not fully deleted - deleting\n", cur_pix, obj_id, p_hdr.span_ix); CHECK_CB(fs, SPIFFS_CHECK_INDEX, SPIFFS_CHECK_DELETE_PAGE, cur_pix, obj_id); res = spiffs_page_delete(fs, cur_pix); @@ -954,7 +954,7 @@ static s32_t spiffs_object_index_consistency_check_v(spiffs *fs, spiffs_obj_id o } if (delete) { - SPIFFS_CHECK_DBG("IX: FIXUP: pix %04x, obj id:%04x spix:%04x is orphan index - deleting\n", + SPIFFS_CHECK_DBG("IX: FIXUP: pix "_SPIPRIpg", obj id:"_SPIPRIid" spix:"_SPIPRIsp" is orphan index - deleting\n", cur_pix, obj_id, p_hdr.span_ix); CHECK_CB(fs, SPIFFS_CHECK_INDEX, SPIFFS_CHECK_DELETE_ORPHANED_INDEX, cur_pix, obj_id); res = spiffs_page_delete(fs, cur_pix); diff --git a/app/spiffs/spiffs_config.h b/app/spiffs/spiffs_config.h index 40394a3c..c6f1b18f 100644 --- a/app/spiffs/spiffs_config.h +++ b/app/spiffs/spiffs_config.h @@ -35,6 +35,40 @@ #define SPIFFS_CHECK_DBG(...) //dbg_printf(__VA_ARGS__) #endif +// Defines spiffs debug print formatters +// some general signed number +#ifndef _SPIPRIi +#define _SPIPRIi "%d" +#endif +// address +#ifndef _SPIPRIad +#define _SPIPRIad "%08x" +#endif +// block +#ifndef _SPIPRIbl +#define _SPIPRIbl "%04x" +#endif +// page +#ifndef _SPIPRIpg +#define _SPIPRIpg "%04x" +#endif +// span index +#ifndef _SPIPRIsp +#define _SPIPRIsp "%04x" +#endif +// file descriptor +#ifndef _SPIPRIfd +#define _SPIPRIfd "%d" +#endif +// file object id +#ifndef _SPIPRIid +#define _SPIPRIid "%04x" +#endif +// file flags +#ifndef _SPIPRIfl +#define _SPIPRIfl "%02x" +#endif + // Enable/disable API functions to determine exact number of bytes // for filedescriptor and cache buffers. Once decided for a configuration, // this can be disabled to reduce flash. @@ -104,6 +138,20 @@ #define SPIFFS_OBJ_NAME_LEN (FS_OBJ_NAME_LEN+1) #endif +// Maximum length of the metadata associated with an object. +// Setting to non-zero value enables metadata-related API but also +// changes the on-disk format, so the change is not backward-compatible. +// +// Do note: the meta length must never exceed +// logical_page_size - (SPIFFS_OBJ_NAME_LEN + 64) +// +// This is derived from following: +// logical_page_size - (SPIFFS_OBJ_NAME_LEN + sizeof(spiffs_page_header) + +// spiffs_object_ix_header fields + at least some LUT entries) +#ifndef SPIFFS_OBJ_META_LEN +#define SPIFFS_OBJ_META_LEN (0) +#endif + // Size of buffer allocated on stack used when copying data. // Lower value generates more read/writes. No meaning having it bigger // than logical page size. @@ -203,6 +251,53 @@ #define SPIFFS_READ_ONLY 0 #endif +// Enable this to add a temporal file cache using the fd buffer. +// The effects of the cache is that SPIFFS_open will find the file faster in +// certain cases. It will make it a lot easier for spiffs to find files +// opened frequently, reducing number of readings from the spi flash for +// finding those files. +// This will grow each fd by 6 bytes. If your files are opened in patterns +// with a degree of temporal locality, the system is optimized. +// Examples can be letting spiffs serve web content, where one file is the css. +// The css is accessed for each html file that is opened, meaning it is +// accessed almost every second time a file is opened. Another example could be +// a log file that is often opened, written, and closed. +// The size of the cache is number of given file descriptors, as it piggybacks +// on the fd update mechanism. The cache lives in the closed file descriptors. +// When closed, the fd know the whereabouts of the file. Instead of forgetting +// this, the temporal cache will keep handling updates to that file even if the +// fd is closed. If the file is opened again, the location of the file is found +// directly. If all available descriptors become opened, all cache memory is +// lost. +#ifndef SPIFFS_TEMPORAL_FD_CACHE +#define SPIFFS_TEMPORAL_FD_CACHE 1 +#endif + +// Temporal file cache hit score. Each time a file is opened, all cached files +// will lose one point. If the opened file is found in cache, that entry will +// gain SPIFFS_TEMPORAL_CACHE_HIT_SCORE points. One can experiment with this +// value for the specific access patterns of the application. However, it must +// be between 1 (no gain for hitting a cached entry often) and 255. +#ifndef SPIFFS_TEMPORAL_CACHE_HIT_SCORE +#define SPIFFS_TEMPORAL_CACHE_HIT_SCORE 4 +#endif + +// Enable to be able to map object indices to memory. +// This allows for faster and more deterministic reading if cases of reading +// large files and when changing file offset by seeking around a lot. +// When mapping a file's index, the file system will be scanned for index pages +// and the info will be put in memory provided by user. When reading, the +// memory map can be looked up instead of searching for index pages on the +// medium. This way, user can trade memory against performance. +// Whole, parts of, or future parts not being written yet can be mapped. The +// memory array will be owned by spiffs and updated accordingly during garbage +// collecting or when modifying the indices. The latter is invoked by when the +// file is modified in some way. The index buffer is tied to the file +// descriptor. +#ifndef SPIFFS_IX_MAP +#define SPIFFS_IX_MAP 0 +#endif + // Set SPIFFS_TEST_VISUALISATION to non-zero to enable SPIFFS_vis function // in the api. This function will visualize all filesystem using given printf // function. diff --git a/app/spiffs/spiffs_gc.c b/app/spiffs/spiffs_gc.c index 8264a68f..c8fb01a7 100644 --- a/app/spiffs/spiffs_gc.c +++ b/app/spiffs/spiffs_gc.c @@ -11,7 +11,7 @@ static s32_t spiffs_gc_erase_block( spiffs_block_ix bix) { s32_t res; - SPIFFS_GC_DBG("gc: erase block %i\n", bix); + SPIFFS_GC_DBG("gc: erase block "_SPIPRIbl"\n", bix); res = spiffs_erase_block(fs, bix); SPIFFS_CHECK_RES(res); @@ -122,19 +122,19 @@ s32_t spiffs_gc_check( u32_t needed_pages = (len + SPIFFS_DATA_PAGE_SIZE(fs) - 1) / SPIFFS_DATA_PAGE_SIZE(fs); // if (fs->free_blocks <= 2 && (s32_t)needed_pages > free_pages) { -// SPIFFS_GC_DBG("gc: full freeblk:%i needed:%i free:%i dele:%i\n", fs->free_blocks, needed_pages, free_pages, fs->stats_p_deleted); +// SPIFFS_GC_DBG("gc: full freeblk:"_SPIPRIi" needed:"_SPIPRIi" free:"_SPIPRIi" dele:"_SPIPRIi"\n", fs->free_blocks, needed_pages, free_pages, fs->stats_p_deleted); // return SPIFFS_ERR_FULL; // } if ((s32_t)needed_pages > (s32_t)(free_pages + fs->stats_p_deleted)) { - SPIFFS_GC_DBG("gc_check: full freeblk:%i needed:%i free:%i dele:%i\n", fs->free_blocks, needed_pages, free_pages, fs->stats_p_deleted); + SPIFFS_GC_DBG("gc_check: full freeblk:"_SPIPRIi" needed:"_SPIPRIi" free:"_SPIPRIi" dele:"_SPIPRIi"\n", fs->free_blocks, needed_pages, free_pages, fs->stats_p_deleted); return SPIFFS_ERR_FULL; } do { - SPIFFS_GC_DBG("\ngc_check #%i: run gc free_blocks:%i pfree:%i pallo:%i pdele:%i [%i] len:%i of %i\n", + SPIFFS_GC_DBG("\ngc_check #"_SPIPRIi": run gc free_blocks:"_SPIPRIi" pfree:"_SPIPRIi" pallo:"_SPIPRIi" pdele:"_SPIPRIi" ["_SPIPRIi"] len:"_SPIPRIi" of "_SPIPRIi"\n", tries, fs->free_blocks, free_pages, fs->stats_p_allocated, fs->stats_p_deleted, (free_pages+fs->stats_p_allocated+fs->stats_p_deleted), - len, free_pages*SPIFFS_DATA_PAGE_SIZE(fs)); + len, (u32_t)(free_pages*SPIFFS_DATA_PAGE_SIZE(fs))); spiffs_block_ix *cands; int count; @@ -152,13 +152,13 @@ s32_t spiffs_gc_check( #endif cand = cands[0]; fs->cleaning = 1; - //printf("gcing: cleaning block %i\n", cand); + //SPIFFS_GC_DBG("gcing: cleaning block "_SPIPRIi"\n", cand); res = spiffs_gc_clean(fs, cand); fs->cleaning = 0; if (res < 0) { - SPIFFS_GC_DBG("gc_check: cleaning block %i, result %i\n", cand, res); + SPIFFS_GC_DBG("gc_check: cleaning block "_SPIPRIi", result "_SPIPRIi"\n", cand, res); } else { - SPIFFS_GC_DBG("gc_check: cleaning block %i, result %i\n", cand, res); + SPIFFS_GC_DBG("gc_check: cleaning block "_SPIPRIi", result "_SPIPRIi"\n", cand, res); } SPIFFS_CHECK_RES(res); @@ -188,7 +188,7 @@ s32_t spiffs_gc_check( res = SPIFFS_ERR_FULL; } - SPIFFS_GC_DBG("gc_check: finished, %i dirty, blocks %i free, %i pages free, %i tries, res %i\n", + SPIFFS_GC_DBG("gc_check: finished, "_SPIPRIi" dirty, blocks "_SPIPRIi" free, "_SPIPRIi" pages free, "_SPIPRIi" tries, res "_SPIPRIi"\n", fs->stats_p_allocated + fs->stats_p_deleted, fs->free_blocks, free_pages, tries, res); @@ -226,7 +226,7 @@ s32_t spiffs_gc_erase_page_stats( } // per entry obj_lookup_page++; } // per object lookup page - SPIFFS_GC_DBG("gc_check: wipe pallo:%i pdele:%i\n", allo, dele); + SPIFFS_GC_DBG("gc_check: wipe pallo:"_SPIPRIi" pdele:"_SPIPRIi"\n", allo, dele); fs->stats_p_allocated -= allo; fs->stats_p_deleted -= dele; return res; @@ -255,7 +255,7 @@ s32_t spiffs_gc_find_candidate( s32_t *cand_scores = (s32_t *)(fs->work + max_candidates * sizeof(spiffs_block_ix)); // align cand_scores on s32_t boundary - cand_scores = (s32_t*)(((ptrdiff_t)cand_scores + sizeof(ptrdiff_t) - 1) & ~(sizeof(ptrdiff_t) - 1)); + cand_scores = (s32_t*)(((intptr_t)cand_scores + sizeof(intptr_t) - 1) & ~(sizeof(intptr_t) - 1)); *block_candidates = cand_blocks; @@ -314,7 +314,7 @@ s32_t spiffs_gc_find_candidate( used_pages_in_block * SPIFFS_GC_HEUR_W_USED + erase_age * (fs_crammed ? 0 : SPIFFS_GC_HEUR_W_ERASE_AGE); int cand_ix = 0; - SPIFFS_GC_DBG("gc_check: bix:%i del:%i use:%i score:%i\n", cur_block, deleted_pages_in_block, used_pages_in_block, score); + SPIFFS_GC_DBG("gc_check: bix:"_SPIPRIbl" del:"_SPIPRIi" use:"_SPIPRIi" score:"_SPIPRIi"\n", cur_block, deleted_pages_in_block, used_pages_in_block, score); while (cand_ix < max_candidates) { if (cand_blocks[cand_ix] == (spiffs_block_ix)-1) { cand_blocks[cand_ix] = cur_block; @@ -356,6 +356,7 @@ typedef struct { spiffs_obj_id cur_obj_id; spiffs_span_ix cur_objix_spix; spiffs_page_ix cur_objix_pix; + spiffs_page_ix cur_data_pix; int stored_scan_entry_index; u8_t obj_id_found; } spiffs_gc; @@ -375,15 +376,16 @@ typedef struct { // s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { s32_t res = SPIFFS_OK; - int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id)); + const int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id)); + // this is the global localizer being pushed and popped int cur_entry = 0; spiffs_obj_id *obj_lu_buf = (spiffs_obj_id *)fs->lu_work; - spiffs_gc gc; + spiffs_gc gc; // our stack frame/state spiffs_page_ix cur_pix = 0; spiffs_page_object_ix_header *objix_hdr = (spiffs_page_object_ix_header *)fs->work; spiffs_page_object_ix *objix = (spiffs_page_object_ix *)fs->work; - SPIFFS_GC_DBG("gc_clean: cleaning block %i\n", bix); + SPIFFS_GC_DBG("gc_clean: cleaning block "_SPIPRIbl"\n", bix); memset(&gc, 0, sizeof(spiffs_gc)); gc.state = FIND_OBJ_DATA; @@ -392,12 +394,12 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { // move free cursor to next block, cannot use free pages from the block we want to clean fs->free_cursor_block_ix = (bix+1)%fs->block_count; fs->free_cursor_obj_lu_entry = 0; - SPIFFS_GC_DBG("gc_clean: move free cursor to block %i\n", fs->free_cursor_block_ix); + SPIFFS_GC_DBG("gc_clean: move free cursor to block "_SPIPRIbl"\n", fs->free_cursor_block_ix); } while (res == SPIFFS_OK && gc.state != FINISHED) { - SPIFFS_GC_DBG("gc_clean: state = %i entry:%i\n", gc.state, cur_entry); - gc.obj_id_found = 0; + SPIFFS_GC_DBG("gc_clean: state = "_SPIPRIi" entry:"_SPIPRIi"\n", gc.state, cur_entry); + gc.obj_id_found = 0; // reset (to no found data page) // scan through lookup pages int obj_lookup_page = cur_entry / entries_per_page; @@ -408,7 +410,7 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ, 0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work); - // check each entry + // check each object lookup entry while (scan && res == SPIFFS_OK && cur_entry - entry_offset < entries_per_page && cur_entry < (int)(SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))) { spiffs_obj_id obj_id = obj_lu_buf[cur_entry-entry_offset]; @@ -417,21 +419,26 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { // act upon object id depending on gc state switch (gc.state) { case FIND_OBJ_DATA: + // find a data page if (obj_id != SPIFFS_OBJ_ID_DELETED && obj_id != SPIFFS_OBJ_ID_FREE && ((obj_id & SPIFFS_OBJ_ID_IX_FLAG) == 0)) { - SPIFFS_GC_DBG("gc_clean: FIND_DATA state:%i - found obj id %04x\n", gc.state, obj_id); + // found a data page, stop scanning and handle in switch case below + SPIFFS_GC_DBG("gc_clean: FIND_DATA state:"_SPIPRIi" - found obj id "_SPIPRIid"\n", gc.state, obj_id); gc.obj_id_found = 1; gc.cur_obj_id = obj_id; + gc.cur_data_pix = cur_pix; scan = 0; } break; case MOVE_OBJ_DATA: + // evacuate found data pages for corresponding object index we have in memory, + // update memory representation if (obj_id == gc.cur_obj_id) { spiffs_page_header p_hdr; res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, 0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr); SPIFFS_CHECK_RES(res); - SPIFFS_GC_DBG("gc_clean: MOVE_DATA found data page %04x:%04x @ %04x\n", gc.cur_obj_id, p_hdr.span_ix, cur_pix); + SPIFFS_GC_DBG("gc_clean: MOVE_DATA found data page "_SPIPRIid":"_SPIPRIsp" @ "_SPIPRIpg"\n", gc.cur_obj_id, p_hdr.span_ix, cur_pix); if (SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, p_hdr.span_ix) != gc.cur_objix_spix) { SPIFFS_GC_DBG("gc_clean: MOVE_DATA no objix spix match, take in another run\n"); } else { @@ -439,7 +446,7 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { if (p_hdr.flags & SPIFFS_PH_FLAG_DELET) { // move page res = spiffs_page_move(fs, 0, 0, obj_id, &p_hdr, cur_pix, &new_data_pix); - SPIFFS_GC_DBG("gc_clean: MOVE_DATA move objix %04x:%04x page %04x to %04x\n", gc.cur_obj_id, p_hdr.span_ix, cur_pix, new_data_pix); + SPIFFS_GC_DBG("gc_clean: MOVE_DATA move objix "_SPIPRIid":"_SPIPRIsp" page "_SPIPRIpg" to "_SPIPRIpg"\n", gc.cur_obj_id, p_hdr.span_ix, cur_pix, new_data_pix); SPIFFS_CHECK_RES(res); // move wipes obj_lu, reload it res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ, @@ -447,8 +454,10 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work); SPIFFS_CHECK_RES(res); } else { - // page is deleted but not deleted in lookup, scrap it - SPIFFS_GC_DBG("gc_clean: MOVE_DATA wipe objix %04x:%04x page %04x\n", obj_id, p_hdr.span_ix, cur_pix); + // page is deleted but not deleted in lookup, scrap it - + // might seem unnecessary as we will erase this block, but + // we might get aborted + SPIFFS_GC_DBG("gc_clean: MOVE_DATA wipe objix "_SPIPRIid":"_SPIPRIsp" page "_SPIPRIpg"\n", obj_id, p_hdr.span_ix, cur_pix); res = spiffs_page_delete(fs, cur_pix); SPIFFS_CHECK_RES(res); new_data_pix = SPIFFS_OBJ_ID_FREE; @@ -457,16 +466,17 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { if (gc.cur_objix_spix == 0) { // update object index header page ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[p_hdr.span_ix] = new_data_pix; - SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page %04x to objix_hdr entry %02x in mem\n", new_data_pix, SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix)); + SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page "_SPIPRIpg" to objix_hdr entry "_SPIPRIsp" in mem\n", new_data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix)); } else { // update object index page ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix)] = new_data_pix; - SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page %04x to objix entry %02x in mem\n", new_data_pix, SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix)); + SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page "_SPIPRIpg" to objix entry "_SPIPRIsp" in mem\n", new_data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix)); } } } break; case MOVE_OBJ_IX: + // find and evacuate object index pages if (obj_id != SPIFFS_OBJ_ID_DELETED && obj_id != SPIFFS_OBJ_ID_FREE && (obj_id & SPIFFS_OBJ_ID_IX_FLAG)) { // found an index object id @@ -479,20 +489,24 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { if (p_hdr.flags & SPIFFS_PH_FLAG_DELET) { // move page res = spiffs_page_move(fs, 0, 0, obj_id, &p_hdr, cur_pix, &new_pix); - SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX move objix %04x:%04x page %04x to %04x\n", obj_id, p_hdr.span_ix, cur_pix, new_pix); + SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX move objix "_SPIPRIid":"_SPIPRIsp" page "_SPIPRIpg" to "_SPIPRIpg"\n", obj_id, p_hdr.span_ix, cur_pix, new_pix); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_UPD, obj_id, p_hdr.span_ix, new_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)&p_hdr, + SPIFFS_EV_IX_MOV, obj_id, p_hdr.span_ix, new_pix, 0); // move wipes obj_lu, reload it res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ, 0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->lu_work); SPIFFS_CHECK_RES(res); } else { - // page is deleted but not deleted in lookup, scrap it - SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX wipe objix %04x:%04x page %04x\n", obj_id, p_hdr.span_ix, cur_pix); + // page is deleted but not deleted in lookup, scrap it - + // might seem unnecessary as we will erase this block, but + // we might get aborted + SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX wipe objix "_SPIPRIid":"_SPIPRIsp" page "_SPIPRIpg"\n", obj_id, p_hdr.span_ix, cur_pix); res = spiffs_page_delete(fs, cur_pix); if (res == SPIFFS_OK) { - spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_DEL, obj_id, p_hdr.span_ix, cur_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)0, + SPIFFS_EV_IX_DEL, obj_id, p_hdr.span_ix, cur_pix, 0); } } SPIFFS_CHECK_RES(res); @@ -501,69 +515,88 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { default: scan = 0; break; - } + } // switch gc state cur_entry++; } // per entry - obj_lookup_page++; + obj_lookup_page++; // no need to check scan variable here, obj_lookup_page is set in start of loop } // per object lookup page - if (res != SPIFFS_OK) break; // state finalization and switch switch (gc.state) { case FIND_OBJ_DATA: if (gc.obj_id_found) { + // handle found data page - // find out corresponding obj ix page and load it to memory spiffs_page_header p_hdr; spiffs_page_ix objix_pix; - gc.stored_scan_entry_index = cur_entry; - cur_entry = 0; + gc.stored_scan_entry_index = cur_entry; // push cursor + cur_entry = 0; // restart scan from start gc.state = MOVE_OBJ_DATA; res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, 0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix), sizeof(spiffs_page_header), (u8_t*)&p_hdr); SPIFFS_CHECK_RES(res); gc.cur_objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, p_hdr.span_ix); - SPIFFS_GC_DBG("gc_clean: FIND_DATA find objix span_ix:%04x\n", gc.cur_objix_spix); + SPIFFS_GC_DBG("gc_clean: FIND_DATA find objix span_ix:"_SPIPRIsp"\n", gc.cur_objix_spix); res = spiffs_obj_lu_find_id_and_span(fs, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, gc.cur_objix_spix, 0, &objix_pix); + if (res == SPIFFS_ERR_NOT_FOUND) { + // on borked systems we might get an ERR_NOT_FOUND here - + // this is handled by simply deleting the page as it is not referenced + // from anywhere + SPIFFS_GC_DBG("gc_clean: FIND_OBJ_DATA objix not found! Wipe page "_SPIPRIpg"\n", gc.cur_data_pix); + res = spiffs_page_delete(fs, gc.cur_data_pix); + SPIFFS_CHECK_RES(res); + // then we restore states and continue scanning for data pages + cur_entry = gc.stored_scan_entry_index; // pop cursor + gc.state = FIND_OBJ_DATA; + break; // done + } SPIFFS_CHECK_RES(res); - SPIFFS_GC_DBG("gc_clean: FIND_DATA found object index at page %04x\n", objix_pix); + SPIFFS_GC_DBG("gc_clean: FIND_DATA found object index at page "_SPIPRIpg"\n", objix_pix); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, 0, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); + // cannot allow a gc if the presumed index in fact is no index, a + // check must run or lot of data may be lost SPIFFS_VALIDATE_OBJIX(objix->p_hdr, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, gc.cur_objix_spix); gc.cur_objix_pix = objix_pix; } else { + // no more data pages found, passed thru all block, start evacuating object indices gc.state = MOVE_OBJ_IX; cur_entry = 0; // restart entry scan index } break; case MOVE_OBJ_DATA: { - // store modified objix (hdr) page + // store modified objix (hdr) page residing in memory now that all + // data pages belonging to this object index and residing in the block + // we want to evacuate spiffs_page_ix new_objix_pix; gc.state = FIND_OBJ_DATA; - cur_entry = gc.stored_scan_entry_index; + cur_entry = gc.stored_scan_entry_index; // pop cursor if (gc.cur_objix_spix == 0) { // store object index header page - res = spiffs_object_update_index_hdr(fs, 0, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, gc.cur_objix_pix, fs->work, 0, 0, &new_objix_pix); - SPIFFS_GC_DBG("gc_clean: MOVE_DATA store modified objix_hdr page, %04x:%04x\n", new_objix_pix, 0); + res = spiffs_object_update_index_hdr(fs, 0, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, gc.cur_objix_pix, fs->work, 0, 0, 0, &new_objix_pix); + SPIFFS_GC_DBG("gc_clean: MOVE_DATA store modified objix_hdr page, "_SPIPRIpg":"_SPIPRIsp"\n", new_objix_pix, 0); SPIFFS_CHECK_RES(res); } else { // store object index page res = spiffs_page_move(fs, 0, fs->work, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, 0, gc.cur_objix_pix, &new_objix_pix); - SPIFFS_GC_DBG("gc_clean: MOVE_DATA store modified objix page, %04x:%04x\n", new_objix_pix, objix->p_hdr.span_ix); + SPIFFS_GC_DBG("gc_clean: MOVE_DATA store modified objix page, "_SPIPRIpg":"_SPIPRIsp"\n", new_objix_pix, objix->p_hdr.span_ix); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_UPD, gc.cur_obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work, + SPIFFS_EV_IX_UPD, gc.cur_obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); } } break; case MOVE_OBJ_IX: + // scanned thru all block, no more object indices found - our work here is done gc.state = FINISHED; break; default: cur_entry = 0; break; - } - SPIFFS_GC_DBG("gc_clean: state-> %i\n", gc.state); + } // switch gc.state + SPIFFS_GC_DBG("gc_clean: state-> "_SPIPRIi"\n", gc.state); } // while state != FINISHED diff --git a/app/spiffs/spiffs_hydrogen.c b/app/spiffs/spiffs_hydrogen.c index b97bb21f..9ff3e7a7 100644 --- a/app/spiffs/spiffs_hydrogen.c +++ b/app/spiffs/spiffs_hydrogen.c @@ -129,14 +129,14 @@ s32_t SPIFFS_mount(spiffs *fs, spiffs_config *config, u8_t *work, res = spiffs_obj_lu_scan(fs); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); - SPIFFS_DBG("page index byte len: %i\n", SPIFFS_CFG_LOG_PAGE_SZ(fs)); - SPIFFS_DBG("object lookup pages: %i\n", SPIFFS_OBJ_LOOKUP_PAGES(fs)); - SPIFFS_DBG("page pages per block: %i\n", SPIFFS_PAGES_PER_BLOCK(fs)); - SPIFFS_DBG("page header length: %i\n", sizeof(spiffs_page_header)); - SPIFFS_DBG("object header index entries: %i\n", SPIFFS_OBJ_HDR_IX_LEN(fs)); - SPIFFS_DBG("object index entries: %i\n", SPIFFS_OBJ_IX_LEN(fs)); - SPIFFS_DBG("available file descriptors: %i\n", fs->fd_count); - SPIFFS_DBG("free blocks: %i\n", fs->free_blocks); + SPIFFS_DBG("page index byte len: "_SPIPRIi"\n", (u32_t)SPIFFS_CFG_LOG_PAGE_SZ(fs)); + SPIFFS_DBG("object lookup pages: "_SPIPRIi"\n", (u32_t)SPIFFS_OBJ_LOOKUP_PAGES(fs)); + SPIFFS_DBG("page pages per block: "_SPIPRIi"\n", (u32_t)SPIFFS_PAGES_PER_BLOCK(fs)); + SPIFFS_DBG("page header length: "_SPIPRIi"\n", (u32_t)sizeof(spiffs_page_header)); + SPIFFS_DBG("object header index entries: "_SPIPRIi"\n", (u32_t)SPIFFS_OBJ_HDR_IX_LEN(fs)); + SPIFFS_DBG("object index entries: "_SPIPRIi"\n", (u32_t)SPIFFS_OBJ_IX_LEN(fs)); + SPIFFS_DBG("available file descriptors: "_SPIPRIi"\n", (u32_t)fs->fd_count); + SPIFFS_DBG("free blocks: "_SPIPRIi"\n", (u32_t)fs->free_blocks); fs->check_cb_f = check_cb_f; @@ -191,7 +191,7 @@ s32_t SPIFFS_creat(spiffs *fs, const char *path, spiffs_mode mode) { res = spiffs_obj_lu_find_free_obj_id(fs, &obj_id, (const u8_t*)path); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); - res = spiffs_object_create(fs, obj_id, (const u8_t*)path, SPIFFS_TYPE_FILE, 0); + res = spiffs_object_create(fs, obj_id, (const u8_t*)path, 0, SPIFFS_TYPE_FILE, 0); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); SPIFFS_UNLOCK(fs); return 0; @@ -212,10 +212,10 @@ spiffs_file SPIFFS_open(spiffs *fs, const char *path, spiffs_flags flags, spiffs #if SPIFFS_READ_ONLY // not valid flags in read only mode - flags &= ~SPIFFS_WRONLY | SPIFFS_CREAT | SPIFFS_TRUNC; + flags &= ~(SPIFFS_WRONLY | SPIFFS_CREAT | SPIFFS_TRUNC); #endif // SPIFFS_READ_ONLY - s32_t res = spiffs_fd_find_new(fs, &fd); + s32_t res = spiffs_fd_find_new(fs, &fd, path); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); res = spiffs_object_find_object_index_header_by_name(fs, (const u8_t*)path, &pix); @@ -243,7 +243,7 @@ spiffs_file SPIFFS_open(spiffs *fs, const char *path, spiffs_flags flags, spiffs spiffs_fd_return(fs, fd->file_nbr); } SPIFFS_API_CHECK_RES_UNLOCK(fs, res); - res = spiffs_object_create(fs, obj_id, (const u8_t*)path, SPIFFS_TYPE_FILE, &pix); + res = spiffs_object_create(fs, obj_id, (const u8_t*)path, 0, SPIFFS_TYPE_FILE, &pix); if (res < SPIFFS_OK) { spiffs_fd_return(fs, fd->file_nbr); } @@ -285,7 +285,7 @@ spiffs_file SPIFFS_open_by_dirent(spiffs *fs, struct spiffs_dirent *e, spiffs_fl spiffs_fd *fd; - s32_t res = spiffs_fd_find_new(fs, &fd); + s32_t res = spiffs_fd_find_new(fs, &fd, 0); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); res = spiffs_object_open_by_page(fs, e->pix, fd, flags, mode); @@ -317,7 +317,7 @@ spiffs_file SPIFFS_open_by_page(spiffs *fs, spiffs_page_ix page_ix, spiffs_flags spiffs_fd *fd; - s32_t res = spiffs_fd_find_new(fs, &fd); + s32_t res = spiffs_fd_find_new(fs, &fd, 0); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); if (SPIFFS_IS_LOOKUP_PAGE(fs, page_ix)) { @@ -356,7 +356,7 @@ spiffs_file SPIFFS_open_by_page(spiffs *fs, spiffs_page_ix page_ix, spiffs_flags return SPIFFS_FH_OFFS(fs, fd->file_nbr); } -s32_t SPIFFS_read(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { +static s32_t spiffs_hydro_read(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { SPIFFS_API_CHECK_CFG(fs); SPIFFS_API_CHECK_MOUNT(fs); SPIFFS_LOCK(fs); @@ -410,6 +410,15 @@ s32_t SPIFFS_read(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { return len; } +s32_t SPIFFS_read(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { + s32_t res = spiffs_hydro_read(fs, fh, buf, len); + if (res == SPIFFS_ERR_END_OF_OBJECT) { + res = 0; + } + return res; +} + + #if !SPIFFS_READ_ONLY static s32_t spiffs_hydro_write(spiffs *fs, spiffs_fd *fd, void *buf, u32_t offset, s32_t len) { (void)fs; @@ -493,7 +502,7 @@ s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { offset + len > fd->cache_page->offset + SPIFFS_CFG_LOG_PAGE_SZ(fs)) // writing beyond cache page { // boundary violation, write back cache first and allocate new - SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page %i for fd %i:%04x, boundary viol, offs:%i size:%i\n", + SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page "_SPIPRIpg" for fd "_SPIPRIfd":"_SPIPRIid", boundary viol, offs:"_SPIPRIi" size:"_SPIPRIi"\n", fd->cache_page->ix, fd->file_nbr, fd->obj_id, fd->cache_page->offset, fd->cache_page->size); res = spiffs_hydro_write(fs, fd, spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix), @@ -511,14 +520,14 @@ s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { if (fd->cache_page) { fd->cache_page->offset = offset; fd->cache_page->size = 0; - SPIFFS_CACHE_DBG("CACHE_WR_ALLO: allocating cache page %i for fd %i:%04x\n", + SPIFFS_CACHE_DBG("CACHE_WR_ALLO: allocating cache page "_SPIPRIpg" for fd "_SPIPRIfd":"_SPIPRIid"\n", fd->cache_page->ix, fd->file_nbr, fd->obj_id); } } if (fd->cache_page) { u32_t offset_in_cpage = offset - fd->cache_page->offset; - SPIFFS_CACHE_DBG("CACHE_WR_WRITE: storing to cache page %i for fd %i:%04x, offs %i:%i len %i\n", + SPIFFS_CACHE_DBG("CACHE_WR_WRITE: storing to cache page "_SPIPRIpg" for fd "_SPIPRIfd":"_SPIPRIid", offs "_SPIPRIi":"_SPIPRIi" len "_SPIPRIi"\n", fd->cache_page->ix, fd->file_nbr, fd->obj_id, offset, offset_in_cpage, len); spiffs_cache *cache = spiffs_get_cache(fs); @@ -539,15 +548,14 @@ s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { // big write, no need to cache it - but first check if there is a cached write already if (fd->cache_page) { // write back cache first - SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page %i for fd %i:%04x, big write, offs:%i size:%i\n", + SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page "_SPIPRIpg" for fd "_SPIPRIfd":"_SPIPRIid", big write, offs:"_SPIPRIi" size:"_SPIPRIi"\n", fd->cache_page->ix, fd->file_nbr, fd->obj_id, fd->cache_page->offset, fd->cache_page->size); res = spiffs_hydro_write(fs, fd, spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix), fd->cache_page->offset, fd->cache_page->size); spiffs_cache_fd_release(fs, fd->cache_page); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); - res = spiffs_hydro_write(fs, fd, buf, offset, len); - SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + // data written below } } } @@ -578,16 +586,19 @@ s32_t SPIFFS_lseek(spiffs *fs, spiffs_file fh, s32_t offs, int whence) { spiffs_fflush_cache(fs, fh); #endif + s32_t fileSize = fd->size == SPIFFS_UNDEFINED_LEN ? 0 : fd->size; + switch (whence) { case SPIFFS_SEEK_CUR: offs = fd->fdoffset+offs; break; case SPIFFS_SEEK_END: - offs = (fd->size == SPIFFS_UNDEFINED_LEN ? 0 : fd->size) + offs; + offs = fileSize + offs; break; } - if ((offs > (s32_t)fd->size) && (SPIFFS_UNDEFINED_LEN != fd->size)) { + if ((offs > fileSize)) { + fd->fdoffset = fileSize; res = SPIFFS_ERR_END_OF_OBJECT; } SPIFFS_API_CHECK_RES_UNLOCK(fs, res); @@ -625,7 +636,7 @@ s32_t SPIFFS_remove(spiffs *fs, const char *path) { spiffs_page_ix pix; s32_t res; - res = spiffs_fd_find_new(fs, &fd); + res = spiffs_fd_find_new(fs, &fd, 0); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); res = spiffs_object_find_object_index_header_by_name(fs, (const u8_t*)path, &pix); @@ -704,6 +715,9 @@ static s32_t spiffs_stat_pix(spiffs *fs, spiffs_page_ix pix, spiffs_file fh, spi s->size = objix_hdr.size == SPIFFS_UNDEFINED_LEN ? 0 : objix_hdr.size; s->pix = pix; strncpy((char *)s->name, (char *)objix_hdr.name, SPIFFS_OBJ_NAME_LEN); +#if SPIFFS_OBJ_META_LEN + memcpy(s->meta, objix_hdr.meta, SPIFFS_OBJ_META_LEN); +#endif return res; } @@ -771,7 +785,7 @@ static s32_t spiffs_fflush_cache(spiffs *fs, spiffs_file fh) { fd->cache_page = spiffs_cache_page_get_by_fd(fs, fd); } if (fd->cache_page) { - SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page %i for fd %i:%04x, flush, offs:%i size:%i\n", + SPIFFS_CACHE_DBG("CACHE_WR_DUMP: dumping cache page "_SPIPRIpg" for fd "_SPIPRIfd":"_SPIPRIid", flush, offs:"_SPIPRIi" size:"_SPIPRIi"\n", fd->cache_page->ix, fd->file_nbr, fd->obj_id, fd->cache_page->offset, fd->cache_page->size); res = spiffs_hydro_write(fs, fd, spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix), @@ -851,7 +865,7 @@ s32_t SPIFFS_rename(spiffs *fs, const char *old_path, const char *new_path) { } SPIFFS_API_CHECK_RES_UNLOCK(fs, res); - res = spiffs_fd_find_new(fs, &fd); + res = spiffs_fd_find_new(fs, &fd, 0); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); res = spiffs_object_open_by_page(fs, pix_old, fd, 0, 0); @@ -861,6 +875,49 @@ s32_t SPIFFS_rename(spiffs *fs, const char *old_path, const char *new_path) { SPIFFS_API_CHECK_RES_UNLOCK(fs, res); res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, (const u8_t*)new_path, + 0, 0, &pix_dummy); +#if SPIFFS_TEMPORAL_FD_CACHE + if (res == SPIFFS_OK) { + spiffs_fd_temporal_cache_rehash(fs, old_path, new_path); + } +#endif + + spiffs_fd_return(fs, fd->file_nbr); + + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + SPIFFS_UNLOCK(fs); + + return res; +#endif // SPIFFS_READ_ONLY +} + +#if SPIFFS_OBJ_META_LEN +s32_t SPIFFS_update_meta(spiffs *fs, const char *name, const void *meta) { +#if SPIFFS_READ_ONLY + (void)fs; (void)name; (void)meta; + return SPIFFS_ERR_RO_NOT_IMPL; +#else + SPIFFS_API_CHECK_CFG(fs); + SPIFFS_API_CHECK_MOUNT(fs); + SPIFFS_LOCK(fs); + + spiffs_page_ix pix, pix_dummy; + spiffs_fd *fd; + + s32_t res = spiffs_object_find_object_index_header_by_name(fs, (const u8_t*)name, &pix); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + res = spiffs_fd_find_new(fs, &fd, 0); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + res = spiffs_object_open_by_page(fs, pix, fd, 0, 0); + if (res != SPIFFS_OK) { + spiffs_fd_return(fs, fd->file_nbr); + } + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, 0, meta, 0, &pix_dummy); spiffs_fd_return(fs, fd->file_nbr); @@ -873,6 +930,40 @@ s32_t SPIFFS_rename(spiffs *fs, const char *old_path, const char *new_path) { #endif // SPIFFS_READ_ONLY } +s32_t SPIFFS_fupdate_meta(spiffs *fs, spiffs_file fh, const void *meta) { +#if SPIFFS_READ_ONLY + (void)fs; (void)fh; (void)meta; + return SPIFFS_ERR_RO_NOT_IMPL; +#else + SPIFFS_API_CHECK_CFG(fs); + SPIFFS_API_CHECK_MOUNT(fs); + SPIFFS_LOCK(fs); + + s32_t res; + spiffs_fd *fd; + spiffs_page_ix pix_dummy; + + fh = SPIFFS_FH_UNOFFS(fs, fh); + res = spiffs_fd_get(fs, fh, &fd); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + if ((fd->flags & SPIFFS_O_WRONLY) == 0) { + res = SPIFFS_ERR_NOT_WRITABLE; + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + } + + res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, 0, meta, + 0, &pix_dummy); + + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + SPIFFS_UNLOCK(fs); + + return res; +#endif // SPIFFS_READ_ONLY +} +#endif // SPIFFS_OBJ_META_LEN + spiffs_DIR *SPIFFS_opendir(spiffs *fs, const char *name, spiffs_DIR *d) { (void)name; @@ -913,7 +1004,7 @@ static s32_t spiffs_read_dir_v( if (res != SPIFFS_OK) return res; if ((obj_id & SPIFFS_OBJ_ID_IX_FLAG) && objix_hdr.p_hdr.span_ix == 0 && - (objix_hdr.p_hdr.flags& (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE)) == + (objix_hdr.p_hdr.flags & (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE)) == (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE)) { struct spiffs_dirent *e = (struct spiffs_dirent*)user_var_p; e->obj_id = obj_id; @@ -921,9 +1012,11 @@ static s32_t spiffs_read_dir_v( e->type = objix_hdr.type; e->size = objix_hdr.size == SPIFFS_UNDEFINED_LEN ? 0 : objix_hdr.size; e->pix = pix; +#if SPIFFS_OBJ_META_LEN + memcpy(e->meta, objix_hdr.meta, SPIFFS_OBJ_META_LEN); +#endif return SPIFFS_OK; } - return SPIFFS_VIS_COUNTINUE; } @@ -952,6 +1045,7 @@ struct spiffs_dirent *SPIFFS_readdir(spiffs_DIR *d, struct spiffs_dirent *e) { if (res == SPIFFS_OK) { d->block = bix; d->entry = entry + 1; + e->obj_id &= ~SPIFFS_OBJ_ID_IX_FLAG; ret = e; } else { d->fs->err_code = res; @@ -1103,6 +1197,138 @@ s32_t SPIFFS_set_file_callback_func(spiffs *fs, spiffs_file_callback cb_func) { return 0; } +#if SPIFFS_IX_MAP + +s32_t SPIFFS_ix_map(spiffs *fs, spiffs_file fh, spiffs_ix_map *map, + u32_t offset, u32_t len, spiffs_page_ix *map_buf) { + s32_t res; + SPIFFS_API_CHECK_CFG(fs); + SPIFFS_API_CHECK_MOUNT(fs); + SPIFFS_LOCK(fs); + + fh = SPIFFS_FH_UNOFFS(fs, fh); + + spiffs_fd *fd; + res = spiffs_fd_get(fs, fh, &fd); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + if (fd->ix_map) { + SPIFFS_API_CHECK_RES_UNLOCK(fs, SPIFFS_ERR_IX_MAP_MAPPED); + } + + map->map_buf = map_buf; + map->offset = offset; + // nb: spix range includes last + map->start_spix = offset / SPIFFS_DATA_PAGE_SIZE(fs); + map->end_spix = (offset + len) / SPIFFS_DATA_PAGE_SIZE(fs); + memset(map_buf, 0, sizeof(spiffs_page_ix) * (map->end_spix - map->start_spix + 1)); + fd->ix_map = map; + + // scan for pixes + res = spiffs_populate_ix_map(fs, fd, 0, map->end_spix - map->start_spix + 1); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + SPIFFS_UNLOCK(fs); + return res; +} + +s32_t SPIFFS_ix_unmap(spiffs *fs, spiffs_file fh) { + s32_t res; + SPIFFS_API_CHECK_CFG(fs); + SPIFFS_API_CHECK_MOUNT(fs); + SPIFFS_LOCK(fs); + + fh = SPIFFS_FH_UNOFFS(fs, fh); + + spiffs_fd *fd; + res = spiffs_fd_get(fs, fh, &fd); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + if (fd->ix_map == 0) { + SPIFFS_API_CHECK_RES_UNLOCK(fs, SPIFFS_ERR_IX_MAP_UNMAPPED); + } + + fd->ix_map = 0; + + SPIFFS_UNLOCK(fs); + return res; +} + +s32_t SPIFFS_ix_remap(spiffs *fs, spiffs_file fh, u32_t offset) { + s32_t res = SPIFFS_OK; + SPIFFS_API_CHECK_CFG(fs); + SPIFFS_API_CHECK_MOUNT(fs); + SPIFFS_LOCK(fs); + + fh = SPIFFS_FH_UNOFFS(fs, fh); + + spiffs_fd *fd; + res = spiffs_fd_get(fs, fh, &fd); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + + if (fd->ix_map == 0) { + SPIFFS_API_CHECK_RES_UNLOCK(fs, SPIFFS_ERR_IX_MAP_UNMAPPED); + } + + spiffs_ix_map *map = fd->ix_map; + + s32_t spix_diff = offset / SPIFFS_DATA_PAGE_SIZE(fs) - map->start_spix; + map->offset = offset; + + // move existing pixes if within map offs + if (spix_diff != 0) { + // move vector + int i; + const s32_t vec_len = map->end_spix - map->start_spix + 1; // spix range includes last + map->start_spix += spix_diff; + map->end_spix += spix_diff; + if (spix_diff >= vec_len) { + // moving beyond range + memset(&map->map_buf, 0, vec_len * sizeof(spiffs_page_ix)); + // populate_ix_map is inclusive + res = spiffs_populate_ix_map(fs, fd, 0, vec_len-1); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + } else if (spix_diff > 0) { + // diff positive + for (i = 0; i < vec_len - spix_diff; i++) { + map->map_buf[i] = map->map_buf[i + spix_diff]; + } + // memset is non-inclusive + memset(&map->map_buf[vec_len - spix_diff], 0, spix_diff * sizeof(spiffs_page_ix)); + // populate_ix_map is inclusive + res = spiffs_populate_ix_map(fs, fd, vec_len - spix_diff, vec_len-1); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + } else { + // diff negative + for (i = vec_len - 1; i >= -spix_diff; i--) { + map->map_buf[i] = map->map_buf[i + spix_diff]; + } + // memset is non-inclusive + memset(&map->map_buf[0], 0, -spix_diff * sizeof(spiffs_page_ix)); + // populate_ix_map is inclusive + res = spiffs_populate_ix_map(fs, fd, 0, -spix_diff - 1); + SPIFFS_API_CHECK_RES_UNLOCK(fs, res); + } + + } + + SPIFFS_UNLOCK(fs); + return res; +} + +s32_t SPIFFS_bytes_to_ix_map_entries(spiffs *fs, u32_t bytes) { + SPIFFS_API_CHECK_CFG(fs); + // always add one extra page, the offset might change to the middle of a page + return (bytes + SPIFFS_DATA_PAGE_SIZE(fs) ) / SPIFFS_DATA_PAGE_SIZE(fs); +} + +s32_t SPIFFS_ix_map_entries_to_bytes(spiffs *fs, u32_t map_page_ix_entries) { + SPIFFS_API_CHECK_CFG(fs); + return map_page_ix_entries * SPIFFS_DATA_PAGE_SIZE(fs); +} + +#endif // SPIFFS_IX_MAP + #if SPIFFS_TEST_VISUALISATION s32_t SPIFFS_vis(spiffs *fs) { s32_t res = SPIFFS_OK; @@ -1128,7 +1354,7 @@ s32_t SPIFFS_vis(spiffs *fs) { cur_entry - entry_offset < entries_per_page && cur_entry < (int)(SPIFFS_PAGES_PER_BLOCK(fs)-SPIFFS_OBJ_LOOKUP_PAGES(fs))) { spiffs_obj_id obj_id = obj_lu_buf[cur_entry-entry_offset]; if (cur_entry == 0) { - spiffs_printf("%4i ", bix); + spiffs_printf(_SPIPRIbl" ", bix); } else if ((cur_entry & 0x3f) == 0) { spiffs_printf(" "); } @@ -1156,7 +1382,7 @@ s32_t SPIFFS_vis(spiffs *fs) { SPIFFS_CHECK_RES(res); if (erase_count != (spiffs_obj_id)-1) { - spiffs_printf("\tera_cnt: %i\n", erase_count); + spiffs_printf("\tera_cnt: "_SPIPRIi"\n", erase_count); } else { spiffs_printf("\tera_cnt: N/A\n"); } @@ -1164,16 +1390,16 @@ s32_t SPIFFS_vis(spiffs *fs) { bix++; } // per block - spiffs_printf("era_cnt_max: %i\n", fs->max_erase_count); - spiffs_printf("last_errno: %i\n", fs->err_code); - spiffs_printf("blocks: %i\n", fs->block_count); - spiffs_printf("free_blocks: %i\n", fs->free_blocks); - spiffs_printf("page_alloc: %i\n", fs->stats_p_allocated); - spiffs_printf("page_delet: %i\n", fs->stats_p_deleted); + spiffs_printf("era_cnt_max: "_SPIPRIi"\n", fs->max_erase_count); + spiffs_printf("last_errno: "_SPIPRIi"\n", fs->err_code); + spiffs_printf("blocks: "_SPIPRIi"\n", fs->block_count); + spiffs_printf("free_blocks: "_SPIPRIi"\n", fs->free_blocks); + spiffs_printf("page_alloc: "_SPIPRIi"\n", fs->stats_p_allocated); + spiffs_printf("page_delet: "_SPIPRIi"\n", fs->stats_p_deleted); SPIFFS_UNLOCK(fs); u32_t total, used; SPIFFS_info(fs, &total, &used); - spiffs_printf("used: %i of %i\n", used, total); + spiffs_printf("used: "_SPIPRIi" of "_SPIPRIi"\n", used, total); return res; } #endif diff --git a/app/spiffs/spiffs_nucleus.c b/app/spiffs/spiffs_nucleus.c index 6c6b61d7..44ba7112 100644 --- a/app/spiffs/spiffs_nucleus.c +++ b/app/spiffs/spiffs_nucleus.c @@ -142,7 +142,7 @@ s32_t spiffs_obj_lu_find_entry_visitor( int entries_per_page = (SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(spiffs_obj_id)); // wrap initial - if (cur_entry >= (int)SPIFFS_OBJ_LOOKUP_MAX_ENTRIES(fs) - 1) { + if (cur_entry > (int)SPIFFS_OBJ_LOOKUP_MAX_ENTRIES(fs) - 1) { cur_entry = 0; cur_block++; cur_block_addr = cur_block * SPIFFS_CFG_LOG_BLOCK_SZ(fs); @@ -232,7 +232,7 @@ s32_t spiffs_erase_block( // here we ignore res, just try erasing the block while (size > 0) { - SPIFFS_DBG("erase %08x:%08x\n", addr, SPIFFS_CFG_PHYS_ERASE_SZ(fs)); + SPIFFS_DBG("erase "_SPIPRIad":"_SPIPRIi"\n", addr, SPIFFS_CFG_PHYS_ERASE_SZ(fs)); SPIFFS_HAL_ERASE(fs, addr, SPIFFS_CFG_PHYS_ERASE_SZ(fs)); addr += SPIFFS_CFG_PHYS_ERASE_SZ(fs); @@ -292,60 +292,27 @@ s32_t spiffs_probe( SPIFFS_CHECK_RES(res); } + // check that we have sane number of blocks + if (bix_count[0] < 3) return SPIFFS_ERR_PROBE_TOO_FEW_BLOCKS; // check that the order is correct, take aborted erases in calculation - // Note that bix_count[0] should be blockcnt, [1] should be blockcnt - 1 - // and [2] should be blockcnt - 3 // first block aborted erase - int fs_size; - if (magic[0] == (spiffs_obj_id)(-1) && bix_count[1] - bix_count[2] == 2) { - fs_size = bix_count[1]+1; - } else + if (magic[0] == (spiffs_obj_id)(-1) && bix_count[1] - bix_count[2] == 1) { + return (bix_count[1]+1) * cfg->log_block_size; + } // second block aborted erase - if (magic[1] == (spiffs_obj_id)(-1) && bix_count[0] - bix_count[2] == 3) { - fs_size = bix_count[0]; - } else + if (magic[1] == (spiffs_obj_id)(-1) && bix_count[0] - bix_count[2] == 2) { + return bix_count[0] * cfg->log_block_size; + } // third block aborted erase if (magic[2] == (spiffs_obj_id)(-1) && bix_count[0] - bix_count[1] == 1) { - fs_size = bix_count[0]; - } else + return bix_count[0] * cfg->log_block_size; + } // no block has aborted erase - if (bix_count[0] - bix_count[1] == 1 && bix_count[1] - bix_count[2] == 2) { - fs_size = bix_count[0]; - } else { - return SPIFFS_ERR_PROBE_NOT_A_FS; + if (bix_count[0] - bix_count[1] == 1 && bix_count[1] - bix_count[2] == 1) { + return bix_count[0] * cfg->log_block_size; } - // check that we have sane number of blocks - if (fs_size < 3) return SPIFFS_ERR_PROBE_TOO_FEW_BLOCKS; - - dummy_fs.block_count = fs_size; - - // Now verify that there is at least one good block at the end - for (bix = fs_size - 1; bix >= 3; bix--) { - spiffs_obj_id end_magic; - paddr = SPIFFS_MAGIC_PADDR(&dummy_fs, bix); -#if SPIFFS_HAL_CALLBACK_EXTRA - // not any proper fs to report here, so callback with null - // (cross fingers that no-one gets angry) - res = cfg->hal_read_f((void *)0, paddr, sizeof(spiffs_obj_id), (u8_t *)&end_magic); -#else - res = cfg->hal_read_f(paddr, sizeof(spiffs_obj_id), (u8_t *)&end_magic); -#endif - if (res < 0) { - return SPIFFS_ERR_PROBE_NOT_A_FS; - } - if (end_magic == (spiffs_obj_id)(-1)) { - if (bix < fs_size - 1) { - return SPIFFS_ERR_PROBE_NOT_A_FS; - } - } else if (end_magic != SPIFFS_MAGIC(&dummy_fs, bix)) { - return SPIFFS_ERR_PROBE_NOT_A_FS; - } else { - break; - } - } - - return fs_size * cfg->log_block_size; + return SPIFFS_ERR_PROBE_NOT_A_FS; } #endif // SPIFFS_USE_MAGIC && SPIFFS_USE_MAGIC_LENGTH && SPIFFS_SINGLETON==0 @@ -440,7 +407,7 @@ s32_t spiffs_obj_lu_scan( #if SPIFFS_USE_MAGIC if (unerased_bix != (spiffs_block_ix)-1) { // found one unerased block, remedy - SPIFFS_DBG("mount: erase block %i\n", bix); + SPIFFS_DBG("mount: erase block "_SPIPRIbl"\n", bix); #if SPIFFS_READ_ONLY res = SPIFFS_ERR_RO_ABORTED_OPERATION; #else @@ -500,7 +467,7 @@ s32_t spiffs_obj_lu_find_free( SPIFFS_OBJ_ID_FREE, block_ix, lu_entry); if (res == SPIFFS_OK) { fs->free_cursor_block_ix = *block_ix; - fs->free_cursor_obj_lu_entry = *lu_entry; + fs->free_cursor_obj_lu_entry = (*lu_entry) + 1; if (*lu_entry == 0) { fs->free_blocks--; } @@ -633,6 +600,152 @@ s32_t spiffs_obj_lu_find_id_and_span_by_phdr( return res; } +#if SPIFFS_IX_MAP + +// update index map of given fd with given object index data +static void spiffs_update_ix_map(spiffs *fs, + spiffs_fd *fd, spiffs_span_ix objix_spix, spiffs_page_object_ix *objix) { +#if SPIFFS_SINGLETON + (void)fs; +#endif + spiffs_ix_map *map = fd->ix_map; + spiffs_span_ix map_objix_start_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->start_spix); + spiffs_span_ix map_objix_end_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->end_spix); + + // check if updated ix is within map range + if (objix_spix < map_objix_start_spix || objix_spix > map_objix_end_spix) { + return; + } + + // update memory mapped page index buffer to new pages + + // get range of updated object index map data span indices + spiffs_span_ix objix_data_spix_start = + SPIFFS_DATA_SPAN_IX_FOR_OBJ_IX_SPAN_IX(fs, objix_spix); + spiffs_span_ix objix_data_spix_end = objix_data_spix_start + + (objix_spix == 0 ? SPIFFS_OBJ_HDR_IX_LEN(fs) : SPIFFS_OBJ_IX_LEN(fs)); + + // calc union of object index range and index map range array + spiffs_span_ix map_spix = MAX(map->start_spix, objix_data_spix_start); + spiffs_span_ix map_spix_end = MIN(map->end_spix + 1, objix_data_spix_end); + + while (map_spix < map_spix_end) { + spiffs_page_ix objix_data_pix; + if (objix_spix == 0) { + // get data page from object index header page + objix_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix_header)))[map_spix]; + } else { + // get data page from object index page + objix_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, map_spix)]; + } + + if (objix_data_pix == (spiffs_page_ix)-1) { + // reached end of object, abort + break; + } + + map->map_buf[map_spix - map->start_spix] = objix_data_pix; + SPIFFS_DBG("map "_SPIPRIid":"_SPIPRIsp" ("_SPIPRIsp"--"_SPIPRIsp") objix.spix:"_SPIPRIsp" to pix "_SPIPRIpg"\n", + fd->obj_id, map_spix - map->start_spix, + map->start_spix, map->end_spix, + objix->p_hdr.span_ix, + objix_data_pix); + + map_spix++; + } +} + +typedef struct { + spiffs_fd *fd; + u32_t remaining_objix_pages_to_visit; + spiffs_span_ix map_objix_start_spix; + spiffs_span_ix map_objix_end_spix; +} spiffs_ix_map_populate_state; + +static s32_t spiffs_populate_ix_map_v( + spiffs *fs, + spiffs_obj_id obj_id, + spiffs_block_ix bix, + int ix_entry, + const void *user_const_p, + void *user_var_p) { + (void)user_const_p; + s32_t res; + spiffs_ix_map_populate_state *state = (spiffs_ix_map_populate_state *)user_var_p; + spiffs_page_ix pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, ix_entry); + + // load header to check it + spiffs_page_object_ix *objix = (spiffs_page_object_ix *)fs->work; + res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, + 0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix), (u8_t *)objix); + SPIFFS_CHECK_RES(res); + SPIFFS_VALIDATE_OBJIX(objix->p_hdr, obj_id, objix->p_hdr.span_ix); + + // check if hdr is ok, and if objix range overlap with ix map range + if ((objix->p_hdr.flags & (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE)) == + (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE) && + objix->p_hdr.span_ix >= state->map_objix_start_spix && + objix->p_hdr.span_ix <= state->map_objix_end_spix) { + // ok, load rest of object index + res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, + 0, SPIFFS_PAGE_TO_PADDR(fs, pix) + sizeof(spiffs_page_object_ix), + SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix), + (u8_t *)objix + sizeof(spiffs_page_object_ix)); + SPIFFS_CHECK_RES(res); + + spiffs_update_ix_map(fs, state->fd, objix->p_hdr.span_ix, objix); + + state->remaining_objix_pages_to_visit--; + SPIFFS_DBG("map "_SPIPRIid" ("_SPIPRIsp"--"_SPIPRIsp") remaining objix pages "_SPIPRIi"\n", + state->fd->obj_id, + state->fd->ix_map->start_spix, state->fd->ix_map->end_spix, + state->remaining_objix_pages_to_visit); + } + + if (res == SPIFFS_OK) { + res = state->remaining_objix_pages_to_visit ? SPIFFS_VIS_COUNTINUE : SPIFFS_VIS_END; + } + return res; +} + +// populates index map, from vector entry start to vector entry end, inclusive +s32_t spiffs_populate_ix_map(spiffs *fs, spiffs_fd *fd, u32_t vec_entry_start, u32_t vec_entry_end) { + s32_t res; + spiffs_ix_map *map = fd->ix_map; + spiffs_ix_map_populate_state state; + vec_entry_start = MIN((map->end_spix - map->start_spix + 1) - 1, (s32_t)vec_entry_start); + vec_entry_end = MAX((map->end_spix - map->start_spix + 1) - 1, (s32_t)vec_entry_end); + if (vec_entry_start > vec_entry_end) { + return SPIFFS_ERR_IX_MAP_BAD_RANGE; + } + state.map_objix_start_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->start_spix + vec_entry_start); + state.map_objix_end_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->start_spix + vec_entry_end); + state.remaining_objix_pages_to_visit = + state.map_objix_end_spix - state.map_objix_start_spix + 1; + state.fd = fd; + + res = spiffs_obj_lu_find_entry_visitor( + fs, + SPIFFS_BLOCK_FOR_PAGE(fs, fd->objix_hdr_pix), + SPIFFS_OBJ_LOOKUP_ENTRY_FOR_PAGE(fs, fd->objix_hdr_pix), + SPIFFS_VIS_CHECK_ID, + fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, + spiffs_populate_ix_map_v, + 0, + &state, + 0, + 0); + + if (res == SPIFFS_VIS_END) { + res = SPIFFS_OK; + } + + return res; +} + +#endif + + #if !SPIFFS_READ_ONLY // Allocates a free defined page with given obj_id // Occupies object lookup entry and page @@ -794,7 +907,8 @@ s32_t spiffs_page_delete( s32_t spiffs_object_create( spiffs *fs, spiffs_obj_id obj_id, - const u8_t name[SPIFFS_OBJ_NAME_LEN], + const u8_t name[], + const u8_t meta[], spiffs_obj_type type, spiffs_page_ix *objix_hdr_pix) { s32_t res = SPIFFS_OK; @@ -810,7 +924,7 @@ s32_t spiffs_object_create( // find free entry res = spiffs_obj_lu_find_free(fs, fs->free_cursor_block_ix, fs->free_cursor_obj_lu_entry, &bix, &entry); SPIFFS_CHECK_RES(res); - SPIFFS_DBG("create: found free page @ %04x bix:%i entry:%i\n", SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry), bix, entry); + SPIFFS_DBG("create: found free page @ "_SPIPRIpg" bix:"_SPIPRIbl" entry:"_SPIPRIsp"\n", (spiffs_page_ix)SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry), bix, entry); // occupy page in object lookup res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_UPDT, @@ -825,15 +939,24 @@ s32_t spiffs_object_create( oix_hdr.p_hdr.flags = 0xff & ~(SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_USED); oix_hdr.type = type; oix_hdr.size = SPIFFS_UNDEFINED_LEN; // keep ones so we can update later without wasting this page - strncpy((char*)&oix_hdr.name, (const char*)name, SPIFFS_OBJ_NAME_LEN); - + strncpy((char*)oix_hdr.name, (const char*)name, SPIFFS_OBJ_NAME_LEN); +#if SPIFFS_OBJ_META_LEN + if (meta) { + memcpy(oix_hdr.meta, meta, SPIFFS_OBJ_META_LEN); + } else { + memset(oix_hdr.meta, 0xff, SPIFFS_OBJ_META_LEN); + } +#else + (void) meta; +#endif // update page res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_UPDT, 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PADDR(fs, bix, entry), sizeof(spiffs_page_object_ix_header), (u8_t*)&oix_hdr); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_NEW, obj_id, 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry), SPIFFS_UNDEFINED_LEN); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)&oix_hdr, + SPIFFS_EV_IX_NEW, obj_id, 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry), SPIFFS_UNDEFINED_LEN); if (objix_hdr_pix) { *objix_hdr_pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry); @@ -854,7 +977,8 @@ s32_t spiffs_object_update_index_hdr( spiffs_obj_id obj_id, spiffs_page_ix objix_hdr_pix, u8_t *new_objix_hdr_data, - const u8_t name[SPIFFS_OBJ_NAME_LEN], + const u8_t name[], + const u8_t meta[], u32_t size, spiffs_page_ix *new_pix) { s32_t res = SPIFFS_OK; @@ -880,6 +1004,13 @@ s32_t spiffs_object_update_index_hdr( if (name) { strncpy((char*)objix_hdr->name, (const char*)name, SPIFFS_OBJ_NAME_LEN); } +#if SPIFFS_OBJ_META_LEN + if (meta) { + memcpy(objix_hdr->meta, meta, SPIFFS_OBJ_META_LEN); + } +#else + (void) meta; +#endif if (size) { objix_hdr->size = size; } @@ -892,7 +1023,9 @@ s32_t spiffs_object_update_index_hdr( *new_pix = new_objix_hdr_pix; } // callback on object index update - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, obj_id, objix_hdr->p_hdr.span_ix, new_objix_hdr_pix, objix_hdr->size); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix_hdr, + new_objix_hdr_data ? SPIFFS_EV_IX_UPD : SPIFFS_EV_IX_UPD_HDR, + obj_id, objix_hdr->p_hdr.span_ix, new_objix_hdr_pix, objix_hdr->size); if (fd) fd->objix_hdr_pix = new_objix_hdr_pix; // if this is not in the registered cluster } @@ -902,35 +1035,41 @@ s32_t spiffs_object_update_index_hdr( void spiffs_cb_object_event( spiffs *fs, - spiffs_fd *fd, + spiffs_page_object_ix *objix, int ev, spiffs_obj_id obj_id_raw, spiffs_span_ix spix, spiffs_page_ix new_pix, u32_t new_size) { - (void)fd; +#if SPIFFS_IX_MAP == 0 + (void)objix; +#endif // update index caches in all file descriptors spiffs_obj_id obj_id = obj_id_raw & ~SPIFFS_OBJ_ID_IX_FLAG; u32_t i; spiffs_fd *fds = (spiffs_fd *)fs->fd_space; for (i = 0; i < fs->fd_count; i++) { spiffs_fd *cur_fd = &fds[i]; +#if SPIFFS_TEMPORAL_FD_CACHE + if (cur_fd->score == 0 || (cur_fd->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG) != obj_id) continue; +#else if (cur_fd->file_nbr == 0 || (cur_fd->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG) != obj_id) continue; +#endif if (spix == 0) { - if (ev == SPIFFS_EV_IX_NEW || ev == SPIFFS_EV_IX_UPD) { - SPIFFS_DBG(" callback: setting fd %i:%04x objix_hdr_pix to %04x, size:%i\n", cur_fd->file_nbr, cur_fd->obj_id, new_pix, new_size); + if (ev != SPIFFS_EV_IX_DEL) { + SPIFFS_DBG(" callback: setting fd "_SPIPRIfd":"_SPIPRIid" objix_hdr_pix to "_SPIPRIpg", size:"_SPIPRIi"\n", cur_fd->file_nbr, cur_fd->obj_id, new_pix, new_size); cur_fd->objix_hdr_pix = new_pix; if (new_size != 0) { cur_fd->size = new_size; } - } else if (ev == SPIFFS_EV_IX_DEL) { + } else { cur_fd->file_nbr = 0; cur_fd->obj_id = SPIFFS_OBJ_ID_DELETED; } } if (cur_fd->cursor_objix_spix == spix) { - if (ev == SPIFFS_EV_IX_NEW || ev == SPIFFS_EV_IX_UPD) { - SPIFFS_DBG(" callback: setting fd %i:%04x span:%04x objix_pix to %04x\n", cur_fd->file_nbr, cur_fd->obj_id, spix, new_pix); + if (ev != SPIFFS_EV_IX_DEL) { + SPIFFS_DBG(" callback: setting fd "_SPIPRIfd":"_SPIPRIid" span:"_SPIPRIsp" objix_pix to "_SPIPRIpg"\n", cur_fd->file_nbr, cur_fd->obj_id, spix, new_pix); cur_fd->cursor_objix_pix = new_pix; } else { cur_fd->cursor_objix_pix = 0; @@ -938,17 +1077,36 @@ void spiffs_cb_object_event( } } +#if SPIFFS_IX_MAP + + // update index maps + if (ev == SPIFFS_EV_IX_UPD || ev == SPIFFS_EV_IX_NEW) { + for (i = 0; i < fs->fd_count; i++) { + spiffs_fd *cur_fd = &fds[i]; + // check fd opened, having ix map, match obj id + if (cur_fd->file_nbr == 0 || + cur_fd->ix_map == 0 || + (cur_fd->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG) != obj_id) continue; + SPIFFS_DBG(" callback: map ix update fd "_SPIPRIfd":"_SPIPRIid" span:"_SPIPRIsp"\n", cur_fd->file_nbr, cur_fd->obj_id, spix); + spiffs_update_ix_map(fs, cur_fd, spix, objix); + } + } + +#endif + // callback to user if object index header if (fs->file_cb_f && spix == 0 && (obj_id_raw & SPIFFS_OBJ_ID_IX_FLAG)) { spiffs_fileop_type op; if (ev == SPIFFS_EV_IX_NEW) { op = SPIFFS_CB_CREATED; - } else if (ev == SPIFFS_EV_IX_UPD) { + } else if (ev == SPIFFS_EV_IX_UPD || + ev == SPIFFS_EV_IX_MOV || + ev == SPIFFS_EV_IX_UPD_HDR) { op = SPIFFS_CB_UPDATED; } else if (ev == SPIFFS_EV_IX_DEL) { op = SPIFFS_CB_DELETED; } else { - SPIFFS_DBG(" callback: WARNING unknown callback event %02x\n", ev); + SPIFFS_DBG(" callback: WARNING unknown callback event "_SPIPRIi"\n", ev); return; // bail out } fs->file_cb_f(fs, op, obj_id, new_pix); @@ -1006,7 +1164,7 @@ s32_t spiffs_object_open_by_page( SPIFFS_VALIDATE_OBJIX(oix_hdr.p_hdr, fd->obj_id, 0); - SPIFFS_DBG("open: fd %i is obj id %04x\n", fd->file_nbr, fd->obj_id); + SPIFFS_DBG("open: fd "_SPIPRIfd" is obj id "_SPIPRIid"\n", fd->file_nbr, fd->obj_id); return res; } @@ -1019,7 +1177,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { s32_t res = SPIFFS_OK; u32_t written = 0; - SPIFFS_DBG("append: %i bytes @ offs %i of size %i\n", len, offset, fd->size); + SPIFFS_DBG("append: "_SPIPRIi" bytes @ offs "_SPIPRIi" of size "_SPIPRIi"\n", len, offset, fd->size); if (offset > fd->size) { SPIFFS_DBG("append: offset reversed to size\n"); @@ -1028,7 +1186,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { res = spiffs_gc_check(fs, len + SPIFFS_DATA_PAGE_SIZE(fs)); // add an extra page of data worth for meta if (res != SPIFFS_OK) { - SPIFFS_DBG("append: gc check fail %i\n", res); + SPIFFS_DBG("append: gc check fail "_SPIPRIi"\n", res); } SPIFFS_CHECK_RES(res); @@ -1056,7 +1214,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { // within this clause we return directly if something fails, object index mess-up if (written > 0) { // store previous object index page, unless first pass - SPIFFS_DBG("append: %04x store objix %04x:%04x, written %i\n", fd->obj_id, + SPIFFS_DBG("append: "_SPIPRIid" store objix "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", fd->obj_id, cur_objix_pix, prev_objix_spix, written); if (prev_objix_spix == 0) { // this is an update to object index header page @@ -1071,9 +1229,9 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { } else { // was a nonempty object, update to new page res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, fs->work, 0, offset+written, &new_objix_hdr_page); + fd->objix_hdr_pix, fs->work, 0, 0, offset+written, &new_objix_hdr_page); SPIFFS_CHECK_RES(res); - SPIFFS_DBG("append: %04x store new objix_hdr, %04x:%04x, written %i\n", fd->obj_id, + SPIFFS_DBG("append: "_SPIPRIid" store new objix_hdr, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", fd->obj_id, new_objix_hdr_page, 0, written); } } else { @@ -1084,12 +1242,13 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_UPDT, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD,fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work, + SPIFFS_EV_IX_UPD,fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0); // update length in object index header page res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, 0, 0, offset+written, &new_objix_hdr_page); + fd->objix_hdr_pix, 0, 0, 0, offset+written, &new_objix_hdr_page); SPIFFS_CHECK_RES(res); - SPIFFS_DBG("append: %04x store new size I %i in objix_hdr, %04x:%04x, written %i\n", fd->obj_id, + SPIFFS_DBG("append: "_SPIPRIid" store new size I "_SPIPRIi" in objix_hdr, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", fd->obj_id, offset+written, new_objix_hdr_page, 0, written); } fd->size = offset+written; @@ -1099,7 +1258,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { // create or load new object index page if (cur_objix_spix == 0) { // load object index header page, must always exist - SPIFFS_DBG("append: %04x load objixhdr page %04x:%04x\n", fd->obj_id, cur_objix_pix, cur_objix_spix); + SPIFFS_DBG("append: "_SPIPRIid" load objixhdr page "_SPIPRIpg":"_SPIPRIsp"\n", fd->obj_id, cur_objix_pix, cur_objix_spix); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); @@ -1114,23 +1273,24 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { res = spiffs_page_allocate_data(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, &p_hdr, 0, 0, 0, 1, &cur_objix_pix); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_NEW, fd->obj_id, cur_objix_spix, cur_objix_pix, 0); // quick "load" of new object index page memset(fs->work, 0xff, SPIFFS_CFG_LOG_PAGE_SZ(fs)); memcpy(fs->work, &p_hdr, sizeof(spiffs_page_header)); - SPIFFS_DBG("append: %04x create objix page, %04x:%04x, written %i\n", fd->obj_id + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work, + SPIFFS_EV_IX_NEW, fd->obj_id, cur_objix_spix, cur_objix_pix, 0); + SPIFFS_DBG("append: "_SPIPRIid" create objix page, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", fd->obj_id , cur_objix_pix, cur_objix_spix, written); } else { // on first pass, we load existing object index page spiffs_page_ix pix; - SPIFFS_DBG("append: %04x find objix span_ix:%04x\n", fd->obj_id, cur_objix_spix); + SPIFFS_DBG("append: "_SPIPRIid" find objix span_ix:"_SPIPRIsp"\n", fd->obj_id, cur_objix_spix); if (fd->cursor_objix_spix == cur_objix_spix) { pix = fd->cursor_objix_pix; } else { res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &pix); SPIFFS_CHECK_RES(res); } - SPIFFS_DBG("append: %04x found object index at page %04x [fd size %i]\n", fd->obj_id, pix, fd->size); + SPIFFS_DBG("append: "_SPIPRIid" found object index at page "_SPIPRIpg" [fd size "_SPIPRIi"]\n", fd->obj_id, pix, fd->size); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); @@ -1154,7 +1314,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { p_hdr.flags = 0xff & ~(SPIFFS_PH_FLAG_FINAL); // finalize immediately res = spiffs_page_allocate_data(fs, fd->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, &p_hdr, &data[written], to_write, page_offs, 1, &data_page); - SPIFFS_DBG("append: %04x store new data page, %04x:%04x offset:%i, len %i, written %i\n", fd->obj_id, + SPIFFS_DBG("append: "_SPIPRIid" store new data page, "_SPIPRIpg":"_SPIPRIsp" offset:"_SPIPRIi", len "_SPIPRIi", written "_SPIPRIi"\n", fd->obj_id, data_page, data_spix, page_offs, to_write, written); } else { // append to existing page, fill out free data in existing page @@ -1171,7 +1331,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_UPDT, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, data_page) + sizeof(spiffs_page_header) + page_offs, to_write, &data[written]); - SPIFFS_DBG("append: %04x store to existing data page, %04x:%04x offset:%i, len %i, written %i\n", fd->obj_id + SPIFFS_DBG("append: "_SPIPRIid" store to existing data page, "_SPIPRIpg":"_SPIPRIsp" offset:"_SPIPRIi", len "_SPIPRIi", written "_SPIPRIi"\n", fd->obj_id , data_page, data_spix, page_offs, to_write, written); } @@ -1181,14 +1341,14 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { if (cur_objix_spix == 0) { // update object index header page ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix] = data_page; - SPIFFS_DBG("append: %04x wrote page %04x to objix_hdr entry %02x in mem\n", fd->obj_id + SPIFFS_DBG("append: "_SPIPRIid" wrote page "_SPIPRIpg" to objix_hdr entry "_SPIPRIsp" in mem\n", fd->obj_id , data_page, data_spix); objix_hdr->size = offset+written; } else { // update object index page ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = data_page; - SPIFFS_DBG("append: %04x wrote page %04x to objix entry %02x in mem\n", fd->obj_id - , data_page, SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); + SPIFFS_DBG("append: "_SPIPRIid" wrote page "_SPIPRIpg" to objix entry "_SPIPRIsp" in mem\n", fd->obj_id + , data_page, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); } // update internals @@ -1207,7 +1367,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { if (cur_objix_spix != 0) { // wrote beyond object index header page // write last modified object index page, unless object header index page - SPIFFS_DBG("append: %04x store objix page, %04x:%04x, written %i\n", fd->obj_id, + SPIFFS_DBG("append: "_SPIPRIid" store objix page, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", fd->obj_id, cur_objix_pix, cur_objix_spix, written); res2 = spiffs_page_index_check(fs, fd, cur_objix_pix, cur_objix_spix); @@ -1216,12 +1376,13 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { res2 = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_UPDT, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res2); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work, + SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0); // update size in object header index page res2 = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, 0, 0, offset+written, &new_objix_hdr_page); - SPIFFS_DBG("append: %04x store new size II %i in objix_hdr, %04x:%04x, written %i, res %i\n", fd->obj_id + fd->objix_hdr_pix, 0, 0, 0, offset+written, &new_objix_hdr_page); + SPIFFS_DBG("append: "_SPIPRIid" store new size II "_SPIPRIi" in objix_hdr, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi", res "_SPIPRIi"\n", fd->obj_id , offset+written, new_objix_hdr_page, 0, written, res2); SPIFFS_CHECK_RES(res2); } else { @@ -1229,7 +1390,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { if (offset == 0) { // wrote to empty object - simply update size and write whole page objix_hdr->size = offset+written; - SPIFFS_DBG("append: %04x store fresh objix_hdr page, %04x:%04x, written %i\n", fd->obj_id + SPIFFS_DBG("append: "_SPIPRIid" store fresh objix_hdr page, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", fd->obj_id , cur_objix_pix, cur_objix_spix, written); res2 = spiffs_page_index_check(fs, fd, cur_objix_pix, cur_objix_spix); @@ -1239,12 +1400,13 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res2); // callback on object index update - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix_hdr->p_hdr.span_ix, cur_objix_pix, objix_hdr->size); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work, + SPIFFS_EV_IX_UPD_HDR, fd->obj_id, objix_hdr->p_hdr.span_ix, cur_objix_pix, objix_hdr->size); } else { // modifying object index header page, update size and make new copy res2 = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, fs->work, 0, offset+written, &new_objix_hdr_page); - SPIFFS_DBG("append: %04x store modified objix_hdr page, %04x:%04x, written %i\n", fd->obj_id + fd->objix_hdr_pix, fs->work, 0, 0, offset+written, &new_objix_hdr_page); + SPIFFS_DBG("append: "_SPIPRIid" store modified objix_hdr page, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", fd->obj_id , new_objix_hdr_page, 0, written); SPIFFS_CHECK_RES(res2); } @@ -1293,8 +1455,8 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { if (prev_objix_spix == 0) { // store previous object index header page res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, fs->work, 0, 0, &new_objix_hdr_pix); - SPIFFS_DBG("modify: store modified objix_hdr page, %04x:%04x, written %i\n", new_objix_hdr_pix, 0, written); + fd->objix_hdr_pix, fs->work, 0, 0, 0, &new_objix_hdr_pix); + SPIFFS_DBG("modify: store modified objix_hdr page, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", new_objix_hdr_pix, 0, written); SPIFFS_CHECK_RES(res); } else { // store new version of previous object index page @@ -1304,16 +1466,17 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { SPIFFS_CHECK_RES(res); res = spiffs_page_move(fs, fd->file_nbr, (u8_t*)objix, fd->obj_id, 0, cur_objix_pix, &new_objix_pix); - SPIFFS_DBG("modify: store previous modified objix page, %04x:%04x, written %i\n", new_objix_pix, objix->p_hdr.span_ix, written); + SPIFFS_DBG("modify: store previous modified objix page, "_SPIPRIid":"_SPIPRIsp", written "_SPIPRIi"\n", new_objix_pix, objix->p_hdr.span_ix, written); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix, + SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); } } // load next object index page if (cur_objix_spix == 0) { // load object index header page, must exist - SPIFFS_DBG("modify: load objixhdr page %04x:%04x\n", cur_objix_pix, cur_objix_spix); + SPIFFS_DBG("modify: load objixhdr page "_SPIPRIpg":"_SPIPRIsp"\n", cur_objix_pix, cur_objix_spix); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); @@ -1321,14 +1484,14 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { } else { // load existing object index page on first pass spiffs_page_ix pix; - SPIFFS_DBG("modify: find objix span_ix:%04x\n", cur_objix_spix); + SPIFFS_DBG("modify: find objix span_ix:"_SPIPRIsp"\n", cur_objix_spix); if (fd->cursor_objix_spix == cur_objix_spix) { pix = fd->cursor_objix_pix; } else { res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &pix); SPIFFS_CHECK_RES(res); } - SPIFFS_DBG("modify: found object index at page %04x\n", pix); + SPIFFS_DBG("modify: found object index at page "_SPIPRIpg"\n", pix); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); @@ -1359,7 +1522,7 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { // a full page, allocate and write a new page of data res = spiffs_page_allocate_data(fs, fd->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG, &p_hdr, &data[written], to_write, page_offs, 1, &data_pix); - SPIFFS_DBG("modify: store new data page, %04x:%04x offset:%i, len %i, written %i\n", data_pix, data_spix, page_offs, to_write, written); + SPIFFS_DBG("modify: store new data page, "_SPIPRIpg":"_SPIPRIsp" offset:"_SPIPRIi", len "_SPIPRIi", written "_SPIPRIi"\n", data_pix, data_spix, page_offs, to_write, written); } else { // write to existing page, allocate new and copy unmodified data @@ -1400,7 +1563,7 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { (u8_t *)&p_hdr.flags); if (res != SPIFFS_OK) break; - SPIFFS_DBG("modify: store to existing data page, src:%04x, dst:%04x:%04x offset:%i, len %i, written %i\n", orig_data_pix, data_pix, data_spix, page_offs, to_write, written); + SPIFFS_DBG("modify: store to existing data page, src:"_SPIPRIpg", dst:"_SPIPRIpg":"_SPIPRIsp" offset:"_SPIPRIi", len "_SPIPRIi", written "_SPIPRIi"\n", orig_data_pix, data_pix, data_spix, page_offs, to_write, written); } // delete original data page @@ -1410,11 +1573,11 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { if (cur_objix_spix == 0) { // update object index header page ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix] = data_pix; - SPIFFS_DBG("modify: wrote page %04x to objix_hdr entry %02x in mem\n", data_pix, data_spix); + SPIFFS_DBG("modify: wrote page "_SPIPRIpg" to objix_hdr entry "_SPIPRIsp" in mem\n", data_pix, data_spix); } else { // update object index page ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = data_pix; - SPIFFS_DBG("modify: wrote page %04x to objix entry %02x in mem\n", data_pix, SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); + SPIFFS_DBG("modify: wrote page "_SPIPRIpg" to objix entry "_SPIPRIsp" in mem\n", data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); } // update internals @@ -1439,17 +1602,18 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { SPIFFS_CHECK_RES(res2); res2 = spiffs_page_move(fs, fd->file_nbr, (u8_t*)objix, fd->obj_id, 0, cur_objix_pix, &new_objix_pix); - SPIFFS_DBG("modify: store modified objix page, %04x:%04x, written %i\n", new_objix_pix, cur_objix_spix, written); + SPIFFS_DBG("modify: store modified objix page, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", new_objix_pix, cur_objix_spix, written); fd->cursor_objix_pix = new_objix_pix; fd->cursor_objix_spix = cur_objix_spix; SPIFFS_CHECK_RES(res2); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix, + SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); } else { // wrote within object index header page res2 = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, fs->work, 0, 0, &new_objix_hdr_pix); - SPIFFS_DBG("modify: store modified objix_hdr page, %04x:%04x, written %i\n", new_objix_hdr_pix, 0, written); + fd->objix_hdr_pix, fs->work, 0, 0, 0, &new_objix_hdr_pix); + SPIFFS_DBG("modify: store modified objix_hdr page, "_SPIPRIpg":"_SPIPRIsp", written "_SPIPRIi"\n", new_objix_hdr_pix, 0, written); SPIFFS_CHECK_RES(res2); } @@ -1526,17 +1690,17 @@ s32_t spiffs_object_find_object_index_header_by_name( s32_t spiffs_object_truncate( spiffs_fd *fd, u32_t new_size, - u8_t remove) { + u8_t remove_full) { s32_t res = SPIFFS_OK; spiffs *fs = fd->fs; - if ((fd->size == SPIFFS_UNDEFINED_LEN || fd->size == 0) && !remove) { + if ((fd->size == SPIFFS_UNDEFINED_LEN || fd->size == 0) && !remove_full) { // no op return res; } // need 2 pages if not removing: object index page + possibly chopped data page - if (remove == 0) { + if (remove_full == 0) { res = spiffs_gc_check(fs, SPIFFS_DATA_PAGE_SIZE(fs) * 2); SPIFFS_CHECK_RES(res); } @@ -1552,7 +1716,7 @@ s32_t spiffs_object_truncate( spiffs_page_ix new_objix_hdr_pix; // before truncating, check if object is to be fully removed and mark this - if (remove && new_size == 0) { + if (remove_full && new_size == 0) { u8_t flags = ~( SPIFFS_PH_FLAG_USED | SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE); res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_UPDT, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, fd->objix_hdr_pix) + offsetof(spiffs_page_header, flags), @@ -1569,14 +1733,15 @@ s32_t spiffs_object_truncate( if (prev_objix_spix != cur_objix_spix) { if (prev_objix_spix != (spiffs_span_ix)-1) { // remove previous object index page - SPIFFS_DBG("truncate: delete objix page %04x:%04x\n", objix_pix, prev_objix_spix); + SPIFFS_DBG("truncate: delete objix page "_SPIPRIpg":"_SPIPRIsp"\n", objix_pix, prev_objix_spix); res = spiffs_page_index_check(fs, fd, objix_pix, prev_objix_spix); SPIFFS_CHECK_RES(res); res = spiffs_page_delete(fs, objix_pix); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_DEL, fd->obj_id, objix->p_hdr.span_ix, objix_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)0, + SPIFFS_EV_IX_DEL, fd->obj_id, objix->p_hdr.span_ix, objix_pix, 0); if (prev_objix_spix > 0) { // Update object index header page, unless we totally want to remove the file. // If fully removing, we're not keeping consistency as good as when storing the header between chunks, @@ -1584,10 +1749,10 @@ s32_t spiffs_object_truncate( // report ERR_FULL a la windows. We cannot have that. // Hence, take the risk - if aborted, a file check would free the lost pages and mend things // as the file is marked as fully deleted in the beginning. - if (remove == 0) { - SPIFFS_DBG("truncate: update objix hdr page %04x:%04x to size %i\n", fd->objix_hdr_pix, prev_objix_spix, cur_size); + if (remove_full == 0) { + SPIFFS_DBG("truncate: update objix hdr page "_SPIPRIpg":"_SPIPRIsp" to size "_SPIPRIi"\n", fd->objix_hdr_pix, prev_objix_spix, cur_size); res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, 0, 0, cur_size, &new_objix_hdr_pix); + fd->objix_hdr_pix, 0, 0, 0, cur_size, &new_objix_hdr_pix); SPIFFS_CHECK_RES(res); } fd->size = cur_size; @@ -1601,7 +1766,7 @@ s32_t spiffs_object_truncate( SPIFFS_CHECK_RES(res); } - SPIFFS_DBG("truncate: load objix page %04x:%04x for data spix:%04x\n", objix_pix, cur_objix_spix, data_spix); + SPIFFS_DBG("truncate: load objix page "_SPIPRIpg":"_SPIPRIsp" for data spix:"_SPIPRIsp"\n", objix_pix, cur_objix_spix, data_spix); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); @@ -1623,20 +1788,20 @@ s32_t spiffs_object_truncate( ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = SPIFFS_OBJ_ID_FREE; } - SPIFFS_DBG("truncate: got data pix %04x\n", data_pix); + SPIFFS_DBG("truncate: got data pix "_SPIPRIpg"\n", data_pix); - if (new_size == 0 || remove || cur_size - new_size >= SPIFFS_DATA_PAGE_SIZE(fs)) { + if (new_size == 0 || remove_full || cur_size - new_size >= SPIFFS_DATA_PAGE_SIZE(fs)) { // delete full data page res = spiffs_page_data_check(fs, fd, data_pix, data_spix); if (res != SPIFFS_ERR_DELETED && res != SPIFFS_OK && res != SPIFFS_ERR_INDEX_REF_FREE) { - SPIFFS_DBG("truncate: err validating data pix %i\n", res); + SPIFFS_DBG("truncate: err validating data pix "_SPIPRIi"\n", res); break; } if (res == SPIFFS_OK) { res = spiffs_page_delete(fs, data_pix); if (res != SPIFFS_OK) { - SPIFFS_DBG("truncate: err deleting data pix %i\n", res); + SPIFFS_DBG("truncate: err deleting data pix "_SPIPRIi"\n", res); break; } } else if (res == SPIFFS_ERR_DELETED || res == SPIFFS_ERR_INDEX_REF_FREE) { @@ -1651,13 +1816,13 @@ s32_t spiffs_object_truncate( } fd->size = cur_size; fd->offset = cur_size; - SPIFFS_DBG("truncate: delete data page %04x for data spix:%04x, cur_size:%i\n", data_pix, data_spix, cur_size); + SPIFFS_DBG("truncate: delete data page "_SPIPRIpg" for data spix:"_SPIPRIsp", cur_size:"_SPIPRIi"\n", data_pix, data_spix, cur_size); } else { // delete last page, partially spiffs_page_header p_hdr; spiffs_page_ix new_data_pix; u32_t bytes_to_remove = SPIFFS_DATA_PAGE_SIZE(fs) - (new_size % SPIFFS_DATA_PAGE_SIZE(fs)); - SPIFFS_DBG("truncate: delete %i bytes from data page %04x for data spix:%04x, cur_size:%i\n", bytes_to_remove, data_pix, data_spix, cur_size); + SPIFFS_DBG("truncate: delete "_SPIPRIi" bytes from data page "_SPIPRIpg" for data spix:"_SPIPRIsp", cur_size:"_SPIPRIi"\n", bytes_to_remove, data_pix, data_spix, cur_size); res = spiffs_page_data_check(fs, fd, data_pix, data_spix); if (res != SPIFFS_OK) break; @@ -1689,11 +1854,11 @@ s32_t spiffs_object_truncate( if (cur_objix_spix == 0) { // update object index header page ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix] = new_data_pix; - SPIFFS_DBG("truncate: wrote page %04x to objix_hdr entry %02x in mem\n", new_data_pix, SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); + SPIFFS_DBG("truncate: wrote page "_SPIPRIpg" to objix_hdr entry "_SPIPRIsp" in mem\n", new_data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); } else { // update object index page ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = new_data_pix; - SPIFFS_DBG("truncate: wrote page %04x to objix entry %02x in mem\n", new_data_pix, SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); + SPIFFS_DBG("truncate: wrote page "_SPIPRIpg" to objix entry "_SPIPRIsp" in mem\n", new_data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); } cur_size = new_size; fd->size = new_size; @@ -1707,30 +1872,31 @@ s32_t spiffs_object_truncate( if (cur_objix_spix == 0) { // update object index header page if (cur_size == 0) { - if (remove) { + if (remove_full) { // remove object altogether - SPIFFS_DBG("truncate: remove object index header page %04x\n", objix_pix); + SPIFFS_DBG("truncate: remove object index header page "_SPIPRIpg"\n", objix_pix); res = spiffs_page_index_check(fs, fd, objix_pix, 0); SPIFFS_CHECK_RES(res); res = spiffs_page_delete(fs, objix_pix); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_DEL, fd->obj_id, 0, objix_pix, 0); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)0, + SPIFFS_EV_IX_DEL, fd->obj_id, 0, objix_pix, 0); } else { // make uninitialized object - SPIFFS_DBG("truncate: reset objix_hdr page %04x\n", objix_pix); + SPIFFS_DBG("truncate: reset objix_hdr page "_SPIPRIpg"\n", objix_pix); memset(fs->work + sizeof(spiffs_page_object_ix_header), 0xff, SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix_header)); res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - objix_pix, fs->work, 0, SPIFFS_UNDEFINED_LEN, &new_objix_hdr_pix); + objix_pix, fs->work, 0, 0, SPIFFS_UNDEFINED_LEN, &new_objix_hdr_pix); SPIFFS_CHECK_RES(res); } } else { // update object index header page SPIFFS_DBG("truncate: update object index header page with indices and size\n"); res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - objix_pix, fs->work, 0, cur_size, &new_objix_hdr_pix); + objix_pix, fs->work, 0, 0, cur_size, &new_objix_hdr_pix); SPIFFS_CHECK_RES(res); } } else { @@ -1743,14 +1909,15 @@ s32_t spiffs_object_truncate( // move and update object index page res = spiffs_page_move(fs, fd->file_nbr, (u8_t*)objix_hdr, fd->obj_id, 0, objix_pix, &new_objix_pix); SPIFFS_CHECK_RES(res); - spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); - SPIFFS_DBG("truncate: store modified objix page, %04x:%04x\n", new_objix_pix, cur_objix_spix); + spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix_hdr, + SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0); + SPIFFS_DBG("truncate: store modified objix page, "_SPIPRIpg":"_SPIPRIsp"\n", new_objix_pix, cur_objix_spix); fd->cursor_objix_pix = new_objix_pix; fd->cursor_objix_spix = cur_objix_spix; fd->offset = cur_size; // update object index header page with new size res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, - fd->objix_hdr_pix, 0, 0, cur_size, &new_objix_hdr_pix); + fd->objix_hdr_pix, 0, 0, 0, cur_size, &new_objix_hdr_pix); SPIFFS_CHECK_RES(res); } fd->size = cur_size; @@ -1776,45 +1943,59 @@ s32_t spiffs_object_read( spiffs_page_object_ix *objix = (spiffs_page_object_ix *)fs->work; while (cur_offset < offset + len) { - cur_objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, data_spix); - if (prev_objix_spix != cur_objix_spix) { - // load current object index (header) page - if (cur_objix_spix == 0) { - objix_pix = fd->objix_hdr_pix; - } else { - SPIFFS_DBG("read: find objix %04x:%04x\n", fd->obj_id, cur_objix_spix); - res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &objix_pix); - SPIFFS_CHECK_RES(res); - } - SPIFFS_DBG("read: load objix page %04x:%04x for data spix:%04x\n", objix_pix, cur_objix_spix, data_spix); - res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, - fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); - SPIFFS_CHECK_RES(res); - SPIFFS_VALIDATE_OBJIX(objix->p_hdr, fd->obj_id, cur_objix_spix); - - fd->offset = cur_offset; - fd->cursor_objix_pix = objix_pix; - fd->cursor_objix_spix = cur_objix_spix; - - prev_objix_spix = cur_objix_spix; - } - - if (cur_objix_spix == 0) { - // get data page from object index header page - data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix]; +#if SPIFFS_IX_MAP + // check if we have a memory, index map and if so, if we're within index map's range + // and if so, if the entry is populated + if (fd->ix_map && data_spix >= fd->ix_map->start_spix && data_spix <= fd->ix_map->end_spix + && fd->ix_map->map_buf[data_spix - fd->ix_map->start_spix]) { + data_pix = fd->ix_map->map_buf[data_spix - fd->ix_map->start_spix]; } else { - // get data page from object index page - data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)]; - } +#endif + cur_objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, data_spix); + if (prev_objix_spix != cur_objix_spix) { + // load current object index (header) page + if (cur_objix_spix == 0) { + objix_pix = fd->objix_hdr_pix; + } else { + SPIFFS_DBG("read: find objix "_SPIPRIid":"_SPIPRIsp"\n", fd->obj_id, cur_objix_spix); + if (fd->cursor_objix_spix == cur_objix_spix) { + objix_pix = fd->cursor_objix_pix; + } else { + res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &objix_pix); + SPIFFS_CHECK_RES(res); + } + } + SPIFFS_DBG("read: load objix page "_SPIPRIpg":"_SPIPRIsp" for data spix:"_SPIPRIsp"\n", objix_pix, cur_objix_spix, data_spix); + res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, + fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); + SPIFFS_CHECK_RES(res); + SPIFFS_VALIDATE_OBJIX(objix->p_hdr, fd->obj_id, cur_objix_spix); + fd->offset = cur_offset; + fd->cursor_objix_pix = objix_pix; + fd->cursor_objix_spix = cur_objix_spix; + + prev_objix_spix = cur_objix_spix; + } + + if (cur_objix_spix == 0) { + // get data page from object index header page + data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix]; + } else { + // get data page from object index page + data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)]; + } +#if SPIFFS_IX_MAP + } +#endif // all remaining data u32_t len_to_read = offset + len - cur_offset; // remaining data in page len_to_read = MIN(len_to_read, SPIFFS_DATA_PAGE_SIZE(fs) - (cur_offset % SPIFFS_DATA_PAGE_SIZE(fs))); // remaining data in file len_to_read = MIN(len_to_read, fd->size); - SPIFFS_DBG("read: offset:%i rd:%i data spix:%04x is data_pix:%04x addr:%08x\n", cur_offset, len_to_read, data_spix, data_pix, - SPIFFS_PAGE_TO_PADDR(fs, data_pix) + sizeof(spiffs_page_header) + (cur_offset % SPIFFS_DATA_PAGE_SIZE(fs))); + SPIFFS_DBG("read: offset:"_SPIPRIi" rd:"_SPIPRIi" data spix:"_SPIPRIsp" is data_pix:"_SPIPRIpg" addr:"_SPIPRIad"\n", cur_offset, len_to_read, data_spix, data_pix, + (u32_t)(SPIFFS_PAGE_TO_PADDR(fs, data_pix) + sizeof(spiffs_page_header) + (cur_offset % SPIFFS_DATA_PAGE_SIZE(fs)))); if (len_to_read <= 0) { res = SPIFFS_ERR_END_OF_OBJECT; break; @@ -1900,7 +2081,7 @@ static s32_t spiffs_obj_lu_find_free_obj_id_compact_v(spiffs *fs, spiffs_obj_id if (id >= state->min_obj_id && id <= state->max_obj_id) { u8_t *map = (u8_t *)fs->work; int ix = (id - state->min_obj_id) / state->compaction; - //SPIFFS_DBG("free_obj_id: add ix %i for id %04x min:%04x max%04x comp:%i\n", ix, id, state->min_obj_id, state->max_obj_id, state->compaction); + //SPIFFS_DBG("free_obj_id: add ix "_SPIPRIi" for id "_SPIPRIid" min"_SPIPRIid" max"_SPIPRIid" comp:"_SPIPRIi"\n", ix, id, state->min_obj_id, state->max_obj_id, state->compaction); map[ix]++; } } @@ -1928,7 +2109,7 @@ s32_t spiffs_obj_lu_find_free_obj_id(spiffs *fs, spiffs_obj_id *obj_id, const u8 if (state.max_obj_id - state.min_obj_id <= (spiffs_obj_id)SPIFFS_CFG_LOG_PAGE_SZ(fs)*8) { // possible to represent in bitmap u32_t i, j; - SPIFFS_DBG("free_obj_id: BITM min:%04x max:%04x\n", state.min_obj_id, state.max_obj_id); + SPIFFS_DBG("free_obj_id: BITM min:"_SPIPRIid" max:"_SPIPRIid"\n", state.min_obj_id, state.max_obj_id); memset(fs->work, 0, SPIFFS_CFG_LOG_PAGE_SZ(fs)); res = spiffs_obj_lu_find_entry_visitor(fs, 0, 0, 0, 0, spiffs_obj_lu_find_free_obj_id_bitmap_v, @@ -1973,14 +2154,14 @@ s32_t spiffs_obj_lu_find_free_obj_id(spiffs *fs, spiffs_obj_id *obj_id, const u8 return SPIFFS_ERR_FULL; } - SPIFFS_DBG("free_obj_id: COMP select index:%i min_count:%i min:%04x max:%04x compact:%i\n", min_i, min_count, state.min_obj_id, state.max_obj_id, state.compaction); + SPIFFS_DBG("free_obj_id: COMP select index:"_SPIPRIi" min_count:"_SPIPRIi" min:"_SPIPRIid" max:"_SPIPRIid" compact:"_SPIPRIi"\n", min_i, min_count, state.min_obj_id, state.max_obj_id, state.compaction); if (min_count == 0) { // no id in this range, skip compacting and use directly *obj_id = min_i * state.compaction + state.min_obj_id; return SPIFFS_OK; } else { - SPIFFS_DBG("free_obj_id: COMP SEL chunk:%04x min:%04x -> %04x\n", state.compaction, state.min_obj_id, state.min_obj_id + min_i * state.compaction); + SPIFFS_DBG("free_obj_id: COMP SEL chunk:"_SPIPRIi" min:"_SPIPRIid" -> "_SPIPRIid"\n", state.compaction, state.min_obj_id, state.min_obj_id + min_i * state.compaction); state.min_obj_id += min_i * state.compaction; state.max_obj_id = state.min_obj_id + state.compaction; // decrease compaction @@ -1993,7 +2174,7 @@ s32_t spiffs_obj_lu_find_free_obj_id(spiffs *fs, spiffs_obj_id *obj_id, const u8 // in a work memory of log_page_size bytes, we may fit in log_page_size ids // todo what if compaction is > 255 - then we cannot fit it in a byte state.compaction = (state.max_obj_id-state.min_obj_id) / ((SPIFFS_CFG_LOG_PAGE_SZ(fs) / sizeof(u8_t))); - SPIFFS_DBG("free_obj_id: COMP min:%04x max:%04x compact:%i\n", state.min_obj_id, state.max_obj_id, state.compaction); + SPIFFS_DBG("free_obj_id: COMP min:"_SPIPRIid" max:"_SPIPRIid" compact:"_SPIPRIi"\n", state.min_obj_id, state.max_obj_id, state.compaction); memset(fs->work, 0, SPIFFS_CFG_LOG_PAGE_SZ(fs)); res = spiffs_obj_lu_find_entry_visitor(fs, 0, 0, 0, 0, spiffs_obj_lu_find_free_obj_id_compact_v, &state, 0, 0, 0); @@ -2007,7 +2188,84 @@ s32_t spiffs_obj_lu_find_free_obj_id(spiffs *fs, spiffs_obj_id *obj_id, const u8 } #endif // !SPIFFS_READ_ONLY -s32_t spiffs_fd_find_new(spiffs *fs, spiffs_fd **fd) { +#if SPIFFS_TEMPORAL_FD_CACHE +// djb2 hash +static u32_t spiffs_hash(spiffs *fs, const u8_t *name) { + (void)fs; + u32_t hash = 5381; + u8_t c; + int i = 0; + while ((c = name[i++]) && i < SPIFFS_OBJ_NAME_LEN) { + hash = (hash * 33) ^ c; + } + return hash; +} +#endif + +s32_t spiffs_fd_find_new(spiffs *fs, spiffs_fd **fd, const char *name) { +#if SPIFFS_TEMPORAL_FD_CACHE + u32_t i; + u16_t min_score = 0xffff; + u32_t cand_ix = (u32_t)-1; + u32_t name_hash = name ? spiffs_hash(fs, (const u8_t *)name) : 0; + spiffs_fd *fds = (spiffs_fd *)fs->fd_space; + + if (name) { + // first, decrease score of all closed descriptors + for (i = 0; i < fs->fd_count; i++) { + spiffs_fd *cur_fd = &fds[i]; + if (cur_fd->file_nbr == 0) { + if (cur_fd->score > 1) { // score == 0 indicates never used fd + cur_fd->score--; + } + } + } + } + + // find the free fd with least score + for (i = 0; i < fs->fd_count; i++) { + spiffs_fd *cur_fd = &fds[i]; + if (cur_fd->file_nbr == 0) { + if (name && cur_fd->name_hash == name_hash) { + cand_ix = i; + break; + } + if (cur_fd->score < min_score) { + min_score = cur_fd->score; + cand_ix = i; + } + } + } + + if (cand_ix != (u32_t)-1) { + spiffs_fd *cur_fd = &fds[cand_ix]; + if (name) { + if (cur_fd->name_hash == name_hash && cur_fd->score > 0) { + // opened an fd with same name hash, assume same file + // set search point to saved obj index page and hope we have a correct match directly + // when start searching - if not, we will just keep searching until it is found + fs->cursor_block_ix = SPIFFS_BLOCK_FOR_PAGE(fs, cur_fd->objix_hdr_pix); + fs->cursor_obj_lu_entry = SPIFFS_OBJ_LOOKUP_ENTRY_FOR_PAGE(fs, cur_fd->objix_hdr_pix); + // update score + if (cur_fd->score < 0xffff-SPIFFS_TEMPORAL_CACHE_HIT_SCORE) { + cur_fd->score += SPIFFS_TEMPORAL_CACHE_HIT_SCORE; + } else { + cur_fd->score = 0xffff; + } + } else { + // no hash hit, restore this fd to initial state + cur_fd->score = SPIFFS_TEMPORAL_CACHE_HIT_SCORE; + cur_fd->name_hash = name_hash; + } + } + cur_fd->file_nbr = cand_ix+1; + *fd = cur_fd; + return SPIFFS_OK; + } else { + return SPIFFS_ERR_OUT_OF_FILE_DESCS; + } +#else + (void)name; u32_t i; spiffs_fd *fds = (spiffs_fd *)fs->fd_space; for (i = 0; i < fs->fd_count; i++) { @@ -2019,6 +2277,7 @@ s32_t spiffs_fd_find_new(spiffs *fs, spiffs_fd **fd) { } } return SPIFFS_ERR_OUT_OF_FILE_DESCS; +#endif } s32_t spiffs_fd_return(spiffs *fs, spiffs_file f) { @@ -2031,6 +2290,9 @@ s32_t spiffs_fd_return(spiffs *fs, spiffs_file f) { return SPIFFS_ERR_FILE_CLOSED; } fd->file_nbr = 0; +#if SPIFFS_IX_MAP + fd->ix_map = 0; +#endif return SPIFFS_OK; } @@ -2045,3 +2307,21 @@ s32_t spiffs_fd_get(spiffs *fs, spiffs_file f, spiffs_fd **fd) { } return SPIFFS_OK; } + +#if SPIFFS_TEMPORAL_FD_CACHE +void spiffs_fd_temporal_cache_rehash( + spiffs *fs, + const char *old_path, + const char *new_path) { + u32_t i; + u32_t old_hash = spiffs_hash(fs, (const u8_t *)old_path); + u32_t new_hash = spiffs_hash(fs, (const u8_t *)new_path); + spiffs_fd *fds = (spiffs_fd *)fs->fd_space; + for (i = 0; i < fs->fd_count; i++) { + spiffs_fd *cur_fd = &fds[i]; + if (cur_fd->score > 0 && cur_fd->name_hash == old_hash) { + cur_fd->name_hash = new_hash; + } + } +} +#endif diff --git a/app/spiffs/spiffs_nucleus.h b/app/spiffs/spiffs_nucleus.h index f6acaacf..7d676ee2 100644 --- a/app/spiffs/spiffs_nucleus.h +++ b/app/spiffs/spiffs_nucleus.h @@ -116,13 +116,23 @@ #define SPIFFS_ERR_CHECK_FLAGS_BAD (SPIFFS_ERR_INTERNAL - 3) #define _SPIFFS_ERR_CHECK_LAST (SPIFFS_ERR_INTERNAL - 4) +// visitor result, continue searching #define SPIFFS_VIS_COUNTINUE (SPIFFS_ERR_INTERNAL - 20) +// visitor result, continue searching after reloading lu buffer #define SPIFFS_VIS_COUNTINUE_RELOAD (SPIFFS_ERR_INTERNAL - 21) +// visitor result, stop searching #define SPIFFS_VIS_END (SPIFFS_ERR_INTERNAL - 22) -#define SPIFFS_EV_IX_UPD 0 -#define SPIFFS_EV_IX_NEW 1 -#define SPIFFS_EV_IX_DEL 2 +// updating an object index contents +#define SPIFFS_EV_IX_UPD (0) +// creating a new object index +#define SPIFFS_EV_IX_NEW (1) +// deleting an object index +#define SPIFFS_EV_IX_DEL (2) +// moving an object index without updating contents +#define SPIFFS_EV_IX_MOV (3) +// updating an object index header data only, not the table itself +#define SPIFFS_EV_IX_UPD_HDR (4) #define SPIFFS_OBJ_ID_IX_FLAG ((spiffs_obj_id)(1<<(8*sizeof(spiffs_obj_id)-1))) @@ -137,7 +147,7 @@ ((spiffs_obj_id)(0x20140529 ^ SPIFFS_CFG_LOG_PAGE_SZ(fs))) #else // SPIFFS_USE_MAGIC_LENGTH #define SPIFFS_MAGIC(fs, bix) \ - ((spiffs_obj_id)(0x20140529 ^ SPIFFS_CFG_LOG_PAGE_SZ(fs) ^ ((fs)->block_count - ((bix) < 3 ? (1<<(bix)) - 1 : (bix)<<2)))) + ((spiffs_obj_id)(0x20140529 ^ SPIFFS_CFG_LOG_PAGE_SZ(fs) ^ ((fs)->block_count - (bix)))) #endif // SPIFFS_USE_MAGIC_LENGTH #endif // SPIFFS_USE_MAGIC @@ -228,7 +238,9 @@ // object index span index number for given data span index or entry #define SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, spix) \ ((spix) < SPIFFS_OBJ_HDR_IX_LEN(fs) ? 0 : (1+((spix)-SPIFFS_OBJ_HDR_IX_LEN(fs))/SPIFFS_OBJ_IX_LEN(fs))) - +// get data span index for object index span index +#define SPIFFS_DATA_SPAN_IX_FOR_OBJ_IX_SPAN_IX(fs, spix) \ + ( (spix) == 0 ? 0 : (SPIFFS_OBJ_HDR_IX_LEN(fs) + (((spix)-1) * SPIFFS_OBJ_IX_LEN(fs))) ) #define SPIFFS_OP_T_OBJ_LU (0<<0) #define SPIFFS_OP_T_OBJ_LU2 (1<<0) @@ -312,7 +324,7 @@ if ((ph).span_ix != (spix)) return SPIFFS_ERR_DATA_SPAN_MISMATCH; -// check id +// check id, only visit matching objec ids #define SPIFFS_VIS_CHECK_ID (1<<0) // report argument object id to visitor - else object lookup id is reported #define SPIFFS_VIS_CHECK_PH (1<<1) @@ -425,6 +437,16 @@ typedef struct { #if SPIFFS_CACHE_WR spiffs_cache_page *cache_page; #endif +#if SPIFFS_TEMPORAL_FD_CACHE + // djb2 hash of filename + u32_t name_hash; + // hit score (score == 0 indicates never used fd) + u16_t score; +#endif +#if SPIFFS_IX_MAP + // spiffs index map, if 0 it means unmapped + spiffs_ix_map *ix_map; +#endif } spiffs_fd; @@ -458,6 +480,10 @@ typedef struct __attribute(( packed )) spiffs_obj_type type; // name of object u8_t name[SPIFFS_OBJ_NAME_LEN]; +#if SPIFFS_OBJ_META_LEN + // metadata. not interpreted by SPIFFS in any way. + u8_t meta[SPIFFS_OBJ_META_LEN]; +#endif } spiffs_page_object_ix_header; // object index page header @@ -612,7 +638,8 @@ s32_t spiffs_page_delete( s32_t spiffs_object_create( spiffs *fs, spiffs_obj_id obj_id, - const u8_t name[SPIFFS_OBJ_NAME_LEN], + const u8_t name[], + const u8_t meta[], spiffs_obj_type type, spiffs_page_ix *objix_hdr_pix); @@ -622,13 +649,24 @@ s32_t spiffs_object_update_index_hdr( spiffs_obj_id obj_id, spiffs_page_ix objix_hdr_pix, u8_t *new_objix_hdr_data, - const u8_t name[SPIFFS_OBJ_NAME_LEN], + const u8_t name[], + const u8_t meta[], u32_t size, spiffs_page_ix *new_pix); -void spiffs_cb_object_event( +#if SPIFFS_IX_MAP + +s32_t spiffs_populate_ix_map( spiffs *fs, spiffs_fd *fd, + u32_t vec_entry_start, + u32_t vec_entry_end); + +#endif + +void spiffs_cb_object_event( + spiffs *fs, + spiffs_page_object_ix *objix, int ev, spiffs_obj_id obj_id, spiffs_span_ix spix, @@ -704,7 +742,8 @@ s32_t spiffs_gc_quick( s32_t spiffs_fd_find_new( spiffs *fs, - spiffs_fd **fd); + spiffs_fd **fd, + const char *name); s32_t spiffs_fd_return( spiffs *fs, @@ -715,6 +754,13 @@ s32_t spiffs_fd_get( spiffs_file f, spiffs_fd **fd); +#if SPIFFS_TEMPORAL_FD_CACHE +void spiffs_fd_temporal_cache_rehash( + spiffs *fs, + const char *old_path, + const char *new_path); +#endif + #if SPIFFS_CACHE void spiffs_cache_init( spiffs *fs); From 7dae5236e6377cc4e347a02ad7c2895d00f8e562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnim=20L=C3=A4uger?= Date: Sun, 21 May 2017 16:17:29 +0200 Subject: [PATCH 02/27] Deprecate xyz.init() in favor of xyz.setup(), removing inherent i2c configuration (#1887) --- app/modules/adxl345.c | 35 ++++++---- app/modules/am2320.c | 38 +++++++---- app/modules/bme280.c | 54 ++++++++++------ app/modules/bmp085.c | 58 ++++++++++------- app/modules/hmc5883l.c | 37 +++++++---- app/modules/l3g4200d.c | 37 +++++++---- docs/en/modules/adxl345.md | 20 +++++- docs/en/modules/am2320.md | 23 ++++++- docs/en/modules/bme280.md | 125 +++++++++++++++++++++--------------- docs/en/modules/bmp085.md | 24 ++++++- docs/en/modules/hmc5883l.md | 20 +++++- docs/en/modules/l3g4200d.md | 20 +++++- 12 files changed, 334 insertions(+), 157 deletions(-) diff --git a/app/modules/adxl345.c b/app/modules/adxl345.c index f248bf77..b7c9d3eb 100644 --- a/app/modules/adxl345.c +++ b/app/modules/adxl345.c @@ -12,7 +12,7 @@ static const uint32_t adxl345_i2c_id = 0; static const uint8_t adxl345_i2c_addr = 0x53; -static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { +static uint8_t r8u(uint32_t id, uint8_t reg) { uint8_t ret; platform_i2c_send_start(id); @@ -26,19 +26,9 @@ static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { return ret; } -static int ICACHE_FLASH_ATTR adxl345_init(lua_State* L) { - - uint32_t sda; - uint32_t scl; +static int adxl345_setup(lua_State* L) { uint8_t devid; - sda = luaL_checkinteger(L, 1); - scl = luaL_checkinteger(L, 2); - - luaL_argcheck(L, sda > 0 && scl > 0, 1, "no i2c for D0"); - - platform_i2c_setup(adxl345_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); - devid = r8u(adxl345_i2c_id, 0x00); if (devid != 229) { @@ -55,7 +45,24 @@ static int ICACHE_FLASH_ATTR adxl345_init(lua_State* L) { return 0; } -static int ICACHE_FLASH_ATTR adxl345_read(lua_State* L) { +static int adxl345_init(lua_State* L) { + + uint32_t sda; + uint32_t scl; + + platform_print_deprecation_note("adxl345.init() is replaced by adxl345.setup()", "in the next version"); + + sda = luaL_checkinteger(L, 1); + scl = luaL_checkinteger(L, 2); + + luaL_argcheck(L, sda > 0 && scl > 0, 1, "no i2c for D0"); + + platform_i2c_setup(adxl345_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); + + return adxl345_setup(L); +} + +static int adxl345_read(lua_State* L) { uint8_t data[6]; int x,y,z; @@ -88,6 +95,8 @@ static int ICACHE_FLASH_ATTR adxl345_read(lua_State* L) { static const LUA_REG_TYPE adxl345_map[] = { { LSTRKEY( "read" ), LFUNCVAL( adxl345_read )}, + { LSTRKEY( "setup" ), LFUNCVAL( adxl345_setup )}, + /// init() is deprecated { LSTRKEY( "init" ), LFUNCVAL( adxl345_init )}, { LNILKEY, LNILVAL} }; diff --git a/app/modules/am2320.c b/app/modules/am2320.c index a0d94608..a3e6736b 100644 --- a/app/modules/am2320.c +++ b/app/modules/am2320.c @@ -84,10 +84,8 @@ static int _read(uint32_t id, void *buf, uint8_t len, uint8_t off) return 0; } -static int am2320_init(lua_State* L) +static int am2320_setup(lua_State* L) { - uint32_t sda; - uint32_t scl; int ret; struct { uint8_t cmd; @@ -97,6 +95,24 @@ static int am2320_init(lua_State* L) uint32_t id; } nfo; + os_delay_us(1500); // give some time to settle things down + ret = _read(am2320_i2c_id, &nfo, sizeof(nfo)-2, 0x08); + if(ret) + return luaL_error(L, "transmission error"); + + lua_pushinteger(L, ntohs(nfo.model)); + lua_pushinteger(L, nfo.version); + lua_pushinteger(L, ntohl(nfo.id)); + return 3; +} + +static int am2320_init(lua_State* L) +{ + uint32_t sda; + uint32_t scl; + + platform_print_deprecation_note("am2320.init() is replaced by am2320.setup()", "in the next version"); + if (!lua_isnumber(L, 1) || !lua_isnumber(L, 2)) { return luaL_error(L, "wrong arg range"); } @@ -110,15 +126,7 @@ static int am2320_init(lua_State* L) platform_i2c_setup(am2320_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); - os_delay_us(1500); // give some time to settle things down - ret = _read(am2320_i2c_id, &nfo, sizeof(nfo)-2, 0x08); - if(ret) - return luaL_error(L, "transmission error"); - - lua_pushinteger(L, ntohs(nfo.model)); - lua_pushinteger(L, nfo.version); - lua_pushinteger(L, ntohl(nfo.id)); - return 3; + return am2320_setup(L); } static int am2320_read(lua_State* L) @@ -145,8 +153,10 @@ static int am2320_read(lua_State* L) } static const LUA_REG_TYPE am2320_map[] = { - { LSTRKEY( "read" ), LFUNCVAL( am2320_read )}, - { LSTRKEY( "init" ), LFUNCVAL( am2320_init )}, + { LSTRKEY( "read" ), LFUNCVAL( am2320_read )}, + { LSTRKEY( "setup" ), LFUNCVAL( am2320_setup )}, + // init() is deprecated + { LSTRKEY( "init" ), LFUNCVAL( am2320_init )}, { LNILKEY, LNILVAL} }; diff --git a/app/modules/bme280.c b/app/modules/bme280.c index 24503386..a1ef58da 100644 --- a/app/modules/bme280.c +++ b/app/modules/bme280.c @@ -227,35 +227,25 @@ static double bme280_qfe2qnh(int32_t qfe, int32_t h) { return qnh; } -static int bme280_lua_init(lua_State* L) { - uint8_t sda; - uint8_t scl; +static int bme280_lua_setup(lua_State* L) { uint8_t config; uint8_t ack; uint8_t full_init; - + uint8_t const bit3 = 0b111; uint8_t const bit2 = 0b11; - if (!lua_isnumber(L, 1) || !lua_isnumber(L, 2)) { - return luaL_error(L, "wrong arg range"); - } - sda = luaL_checkinteger(L, 1); - scl = luaL_checkinteger(L, 2); - - bme280_mode = (!lua_isnumber(L, 6)?BME280_NORMAL_MODE:(luaL_checkinteger(L, 6)&bit2)) // 6-th parameter: power mode - | ((!lua_isnumber(L, 4)?BME280_OVERSAMP_16X:(luaL_checkinteger(L, 4)&bit3)) << 2) // 4-th parameter: pressure oversampling - | ((!lua_isnumber(L, 3)?BME280_OVERSAMP_16X:(luaL_checkinteger(L, 3)&bit3)) << 5); // 3-rd parameter: temperature oversampling + bme280_mode = (!lua_isnumber(L, 4)?BME280_NORMAL_MODE:(luaL_checkinteger(L, 4)&bit2)) // 4-th parameter: power mode + | ((!lua_isnumber(L, 2)?BME280_OVERSAMP_16X:(luaL_checkinteger(L, 2)&bit3)) << 2) // 2-nd parameter: pressure oversampling + | ((!lua_isnumber(L, 1)?BME280_OVERSAMP_16X:(luaL_checkinteger(L, 1)&bit3)) << 5); // 1-st parameter: temperature oversampling - bme280_ossh = (!lua_isnumber(L, 5))?BME280_OVERSAMP_16X:(luaL_checkinteger(L, 5)&bit3); // 5-th parameter: humidity oversampling + bme280_ossh = (!lua_isnumber(L, 3))?BME280_OVERSAMP_16X:(luaL_checkinteger(L, 3)&bit3); // 3-rd parameter: humidity oversampling - config = ((!lua_isnumber(L, 7)?BME280_STANDBY_TIME_20_MS:(luaL_checkinteger(L, 7)&bit3))<< 5) // 7-th parameter: inactive duration in normal mode - | ((!lua_isnumber(L, 8)?BME280_FILTER_COEFF_16:(luaL_checkinteger(L, 8)&bit3)) << 2); // 8-th parameter: IIR filter - full_init = !lua_isnumber(L, 9)?1:lua_tointeger(L, 9); // 9-th parameter: init the chip too + config = ((!lua_isnumber(L, 5)?BME280_STANDBY_TIME_20_MS:(luaL_checkinteger(L, 5)&bit3))<< 5) // 5-th parameter: inactive duration in normal mode + | ((!lua_isnumber(L, 6)?BME280_FILTER_COEFF_16:(luaL_checkinteger(L, 6)&bit3)) << 2); // 6-th parameter: IIR filter + full_init = !lua_isnumber(L, 7)?1:lua_tointeger(L, 7); // 7-th parameter: init the chip too NODE_DBG("mode: %x\nhumidity oss: %x\nconfig: %x\n", bme280_mode, bme280_ossh, config); - platform_i2c_setup(bme280_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); - bme280_i2c_addr = BME280_I2C_ADDRESS1; platform_i2c_send_start(bme280_i2c_id); ack = platform_i2c_send_address(bme280_i2c_id, bme280_i2c_addr, PLATFORM_I2C_DIRECTION_TRANSMITTER); @@ -324,6 +314,30 @@ static int bme280_lua_init(lua_State* L) { return 1; } +static int bme280_lua_init(lua_State* L) { + uint8_t sda; + uint8_t scl; + uint8_t config; + uint8_t ack; + uint8_t full_init; + + platform_print_deprecation_note("bme280.init() is replaced by bme280.setup()", "in the next version"); + + if (!lua_isnumber(L, 1) || !lua_isnumber(L, 2)) { + return luaL_error(L, "wrong arg range"); + } + sda = luaL_checkinteger(L, 1); + scl = luaL_checkinteger(L, 2); + + platform_i2c_setup(bme280_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); + + // remove sda and scl parameters from stack + lua_remove(L, 1); + lua_remove(L, 1); + + return bme280_lua_setup(L); +} + static void bme280_readoutdone (void *arg) { NODE_DBG("timer out\n"); @@ -481,7 +495,9 @@ static int bme280_lua_dewpoint(lua_State* L) { } static const LUA_REG_TYPE bme280_map[] = { + // init() is deprecated { LSTRKEY( "init" ), LFUNCVAL(bme280_lua_init)}, + { LSTRKEY( "setup" ), LFUNCVAL(bme280_lua_setup)}, { LSTRKEY( "temp" ), LFUNCVAL(bme280_lua_temp)}, { LSTRKEY( "baro" ), LFUNCVAL(bme280_lua_baro)}, { LSTRKEY( "humi" ), LFUNCVAL(bme280_lua_humi)}, diff --git a/app/modules/bmp085.c b/app/modules/bmp085.c index a1d9b580..8e911687 100644 --- a/app/modules/bmp085.c +++ b/app/modules/bmp085.c @@ -21,7 +21,7 @@ static struct { int16_t MD; } bmp085_data; -static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { +static uint8_t r8u(uint32_t id, uint8_t reg) { uint8_t ret; platform_i2c_send_start(id); @@ -35,32 +35,18 @@ static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { return ret; } -static uint16_t ICACHE_FLASH_ATTR r16u(uint32_t id, uint8_t reg) { +static uint16_t r16u(uint32_t id, uint8_t reg) { uint8_t high = r8u(id, reg); uint8_t low = r8u(id, reg + 1); return (high << 8) | low; } -static int16_t ICACHE_FLASH_ATTR r16(uint32_t id, uint8_t reg) { +static int16_t r16(uint32_t id, uint8_t reg) { return (int16_t) r16u(id, reg); } -static int ICACHE_FLASH_ATTR bmp085_init(lua_State* L) { - uint32_t sda; - uint32_t scl; - - if (!lua_isnumber(L, 1) || !lua_isnumber(L, 2)) { - return luaL_error(L, "wrong arg range"); - } - - sda = luaL_checkinteger(L, 1); - scl = luaL_checkinteger(L, 2); - - if (scl == 0 || sda == 0) { - return luaL_error(L, "no i2c for D0"); - } - - platform_i2c_setup(bmp085_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); +static int bmp085_setup(lua_State* L) { + (void)L; bmp085_data.AC1 = r16(bmp085_i2c_id, 0xAA); bmp085_data.AC2 = r16(bmp085_i2c_id, 0xAC); @@ -77,6 +63,28 @@ static int ICACHE_FLASH_ATTR bmp085_init(lua_State* L) { return 0; } +static int bmp085_init(lua_State* L) { + uint32_t sda; + uint32_t scl; + + platform_print_deprecation_note("bmp085.init() is replaced by bmp085.setup()", "in the next version"); + + if (!lua_isnumber(L, 1) || !lua_isnumber(L, 2)) { + return luaL_error(L, "wrong arg range"); + } + + sda = luaL_checkinteger(L, 1); + scl = luaL_checkinteger(L, 2); + + if (scl == 0 || sda == 0) { + return luaL_error(L, "no i2c for D0"); + } + + platform_i2c_setup(bmp085_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); + + return bmp085_setup(L); +} + static uint32_t bmp085_temperature_raw_b5(void) { int16_t t, X1, X2; @@ -96,16 +104,16 @@ static uint32_t bmp085_temperature_raw_b5(void) { return X1 + X2; } -static int16_t ICACHE_FLASH_ATTR bmp085_temperature(void) { +static int16_t bmp085_temperature(void) { return (bmp085_temperature_raw_b5() + 8) >> 4; } -static int ICACHE_FLASH_ATTR bmp085_lua_temperature(lua_State* L) { +static int bmp085_lua_temperature(lua_State* L) { lua_pushinteger(L, bmp085_temperature()); return 1; } -static int32_t ICACHE_FLASH_ATTR bmp085_pressure_raw(int oss) { +static int32_t bmp085_pressure_raw(int oss) { int32_t p; int32_t p1, p2, p3; @@ -132,7 +140,7 @@ static int32_t ICACHE_FLASH_ATTR bmp085_pressure_raw(int oss) { return p; } -static int ICACHE_FLASH_ATTR bmp085_lua_pressure_raw(lua_State* L) { +static int bmp085_lua_pressure_raw(lua_State* L) { uint8_t oss = 0; int32_t p; @@ -148,7 +156,7 @@ static int ICACHE_FLASH_ATTR bmp085_lua_pressure_raw(lua_State* L) { return 1; } -static int ICACHE_FLASH_ATTR bmp085_lua_pressure(lua_State* L) { +static int bmp085_lua_pressure(lua_State* L) { uint8_t oss = 0; int32_t p; int32_t X1, X2, X3, B3, B4, B5, B6, B7; @@ -187,6 +195,8 @@ static const LUA_REG_TYPE bmp085_map[] = { { LSTRKEY( "temperature" ), LFUNCVAL( bmp085_lua_temperature )}, { LSTRKEY( "pressure" ), LFUNCVAL( bmp085_lua_pressure )}, { LSTRKEY( "pressure_raw" ), LFUNCVAL( bmp085_lua_pressure_raw )}, + { LSTRKEY( "setup" ), LFUNCVAL( bmp085_setup )}, + // init() is deprecated { LSTRKEY( "init" ), LFUNCVAL( bmp085_init )}, { LNILKEY, LNILVAL} }; diff --git a/app/modules/hmc5883l.c b/app/modules/hmc5883l.c index 4f7c4979..ec602966 100644 --- a/app/modules/hmc5883l.c +++ b/app/modules/hmc5883l.c @@ -12,7 +12,7 @@ static const uint32_t hmc5883_i2c_id = 0; static const uint8_t hmc5883_i2c_addr = 0x1E; -static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { +static uint8_t r8u(uint32_t id, uint8_t reg) { uint8_t ret; platform_i2c_send_start(id); @@ -26,7 +26,7 @@ static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { return ret; } -static void ICACHE_FLASH_ATTR w8u(uint32_t id, uint8_t reg, uint8_t val) { +static void w8u(uint32_t id, uint8_t reg, uint8_t val) { platform_i2c_send_start(hmc5883_i2c_id); platform_i2c_send_address(hmc5883_i2c_id, hmc5883_i2c_addr, PLATFORM_I2C_DIRECTION_TRANSMITTER); platform_i2c_send_byte(hmc5883_i2c_id, reg); @@ -34,19 +34,9 @@ static void ICACHE_FLASH_ATTR w8u(uint32_t id, uint8_t reg, uint8_t val) { platform_i2c_send_stop(hmc5883_i2c_id); } -static int ICACHE_FLASH_ATTR hmc5883_init(lua_State* L) { - - uint32_t sda; - uint32_t scl; +static int hmc5883_setup(lua_State* L) { uint8_t devid_a, devid_b, devid_c; - sda = luaL_checkinteger(L, 1); - scl = luaL_checkinteger(L, 2); - - luaL_argcheck(L, sda > 0 && scl > 0, 1, "no i2c for D0"); - - platform_i2c_setup(hmc5883_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); - devid_a = r8u(hmc5883_i2c_id, 10); devid_b = r8u(hmc5883_i2c_id, 11); devid_c = r8u(hmc5883_i2c_id, 12); @@ -67,7 +57,24 @@ static int ICACHE_FLASH_ATTR hmc5883_init(lua_State* L) { return 0; } -static int ICACHE_FLASH_ATTR hmc5883_read(lua_State* L) { +static int hmc5883_init(lua_State* L) { + + uint32_t sda; + uint32_t scl; + + platform_print_deprecation_note("hmc5883l.init() is replaced by hmc5883l.setup()", "in the next version"); + + sda = luaL_checkinteger(L, 1); + scl = luaL_checkinteger(L, 2); + + luaL_argcheck(L, sda > 0 && scl > 0, 1, "no i2c for D0"); + + platform_i2c_setup(hmc5883_i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); + + return hmc5883_setup(L); +} + +static int hmc5883_read(lua_State* L) { uint8_t data[6]; int x,y,z; @@ -101,6 +108,8 @@ static int ICACHE_FLASH_ATTR hmc5883_read(lua_State* L) { static const LUA_REG_TYPE hmc5883_map[] = { { LSTRKEY( "read" ), LFUNCVAL( hmc5883_read )}, + { LSTRKEY( "setup" ), LFUNCVAL( hmc5883_setup )}, + // init() is deprecated { LSTRKEY( "init" ), LFUNCVAL( hmc5883_init )}, { LNILKEY, LNILVAL} }; diff --git a/app/modules/l3g4200d.c b/app/modules/l3g4200d.c index 95ed90c5..6975fb80 100644 --- a/app/modules/l3g4200d.c +++ b/app/modules/l3g4200d.c @@ -12,7 +12,7 @@ static const uint32_t i2c_id = 0; static const uint8_t i2c_addr = 0x69; -static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { +static uint8_t r8u(uint32_t id, uint8_t reg) { uint8_t ret; platform_i2c_send_start(id); @@ -26,7 +26,7 @@ static uint8_t ICACHE_FLASH_ATTR r8u(uint32_t id, uint8_t reg) { return ret; } -static void ICACHE_FLASH_ATTR w8u(uint32_t id, uint8_t reg, uint8_t val) { +static void w8u(uint32_t id, uint8_t reg, uint8_t val) { platform_i2c_send_start(i2c_id); platform_i2c_send_address(i2c_id, i2c_addr, PLATFORM_I2C_DIRECTION_TRANSMITTER); platform_i2c_send_byte(i2c_id, reg); @@ -34,19 +34,9 @@ static void ICACHE_FLASH_ATTR w8u(uint32_t id, uint8_t reg, uint8_t val) { platform_i2c_send_stop(i2c_id); } -static int ICACHE_FLASH_ATTR l3g4200d_init(lua_State* L) { - - uint32_t sda; - uint32_t scl; +static int l3g4200d_setup(lua_State* L) { uint8_t devid; - sda = luaL_checkinteger(L, 1); - scl = luaL_checkinteger(L, 2); - - luaL_argcheck(L, sda > 0 && scl > 0, 1, "no i2c for D0"); - - platform_i2c_setup(i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); - devid = r8u(i2c_id, 0xF); if (devid != 0xD3) { @@ -58,7 +48,24 @@ static int ICACHE_FLASH_ATTR l3g4200d_init(lua_State* L) { return 0; } -static int ICACHE_FLASH_ATTR l3g4200d_read(lua_State* L) { +static int l3g4200d_init(lua_State* L) { + + uint32_t sda; + uint32_t scl; + + platform_print_deprecation_note("l3g4200d.init() is replaced by l3g4200d.setup()", "in the next version"); + + sda = luaL_checkinteger(L, 1); + scl = luaL_checkinteger(L, 2); + + luaL_argcheck(L, sda > 0 && scl > 0, 1, "no i2c for D0"); + + platform_i2c_setup(i2c_id, sda, scl, PLATFORM_I2C_SPEED_SLOW); + + return l3g4200d_setup(L); +} + +static int l3g4200d_read(lua_State* L) { uint8_t data[6]; int x,y,z; @@ -91,6 +98,8 @@ static int ICACHE_FLASH_ATTR l3g4200d_read(lua_State* L) { static const LUA_REG_TYPE l3g4200d_map[] = { { LSTRKEY( "read" ), LFUNCVAL( l3g4200d_read )}, + { LSTRKEY( "setup" ), LFUNCVAL( l3g4200d_setup )}, + // init() is deprecated { LSTRKEY( "init" ), LFUNCVAL( l3g4200d_init )}, { LNILKEY, LNILVAL} }; diff --git a/docs/en/modules/adxl345.md b/docs/en/modules/adxl345.md index 150e0524..ff9f94ac 100644 --- a/docs/en/modules/adxl345.md +++ b/docs/en/modules/adxl345.md @@ -17,7 +17,9 @@ X,Y,Z data (integers) #### Example ```lua -adxl345.init(1, 2) +local sda, scl = 1, 2 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +adxl345.setup() local x,y,z = adxl345.read() print(string.format("X = %d, Y = %d, Z = %d", x, y, z)) ``` @@ -25,6 +27,10 @@ print(string.format("X = %d, Y = %d, Z = %d", x, y, z)) ## adxl345.init() Initializes the module and sets the pin configuration. +!!! attention + + This function is deprecated and will be removed in upcoming releases. Use `adxl345.setup()` instead. + #### Syntax `adxl345.init(sda, scl)` @@ -34,3 +40,15 @@ Initializes the module and sets the pin configuration. #### Returns `nil` + +## adxl345.setup() +Initializes the module. + +#### Syntax +`adxl345.setup()` + +#### Parameters +None + +#### Returns +`nil` diff --git a/docs/en/modules/am2320.md b/docs/en/modules/am2320.md index 2db61809..7e9824d6 100644 --- a/docs/en/modules/am2320.md +++ b/docs/en/modules/am2320.md @@ -9,6 +9,10 @@ This module provides access to the [AM2320](https://akizukidenshi.com/download/d ## am2320.init() Initializes the module and sets the pin configuration. Returns model, version, serial but is seams these where all zero on my model. +!!! attention + + This function is deprecated and will be removed in upcoming releases. Use `am2320.setup()` instead. + #### Syntax `model, version, serial = am2320.init(sda, scl)` @@ -35,9 +39,26 @@ Samples the sensor and returns the relative humidity in % and temperature in cel #### Example ```lua -am2320.init(1, 2) +sda, scl = 1, 2 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +am2320.setup() rh, t = am2320.read() print(string.format("RH: %s%%", rh / 10)) print(string.format("Temperature: %s degrees C", t / 10)) ``` +## am2320.setup() +Initializes the module. Returns model, version, serial but is seams these where all zero on my model. + +#### Syntax +`model, version, serial = am2320.setup()` + +#### Parameters +None + +#### Returns +- `model` 16 bits number of model +- `version` 8 bits version number +- `serial` 32 bits serial number + + Note: I have only observed values of 0 for all of these, maybe other sensors return more sensible readings. diff --git a/docs/en/modules/bme280.md b/docs/en/modules/bme280.md index b5e67d57..2b71cfe4 100644 --- a/docs/en/modules/bme280.md +++ b/docs/en/modules/bme280.md @@ -5,7 +5,7 @@ This module provides a simple interface to [BME280/BMP280 temperature/air presssure/humidity sensors](http://www.bosch-sensortec.com/bst/products/all_products/bme280) (Bosch Sensortec). -Note that you must call [`init()`](#bme280init) before you can start reading values! +Note that you must call [`setup()`](#bme280setup) before you can start reading values! ## bme280.altitude() @@ -69,13 +69,72 @@ none Initializes module. Initialization is mandatory before read values. +!!! attention + + This function is deprecated and will be removed in upcoming releases. Use `bme280.setup()` instead. + #### Syntax `bme280.init(sda, scl, [temp_oss, press_oss, humi_oss, power_mode, inactive_duration, IIR_filter])` #### Parameters -- `sda` - SDA pin -- `scl` - SCL pin +See [`setup()`](#bme280setup). + +## bme280.qfe2qnh() + +For given altitude converts the air pressure to sea level air pressure. + +#### Syntax +`bme280.qfe2qnh(P, altitude)` + +#### Parameters +- `P` measured pressure +- `altitude` altitude in meters of measurement point + +#### Returns +sea level pressure + + +## bme280.read() + +Reads the sensor and returns the temperature, the air pressure, the air relative humidity and + +#### Syntax +`bme280.read([altitude])` + +#### Parameters +- (optional) `altitude`- altitude in meters of measurement point. If provided also the air pressure converted to sea level air pressure is returned. + +#### Returns +- `T` temperature in celsius as an integer multiplied with 100 +- `P` air pressure in hectopascals multiplied by 1000 +- `H` relative humidity in percent multiplied by 1000 +- `QNH` air pressure in hectopascals multiplied by 1000 converted to sea level + +Any of these variables is `nil` if the readout of given measure was not successful. + +## bme280.startreadout() +Starts readout (turns the sensor into forced mode). After the readout the sensor turns to sleep mode. + +#### Syntax +`bme280.startreadout(delay, callback)` + +#### Parameters +- `delay` sets sensor to forced mode and calls the `callback` (if provided) after given number of milliseconds. For 0 the default delay is set to 113ms (sufficient time to perform reading for oversampling settings 16x). For different oversampling setting please refer to [BME280 Final Datasheet - Appendix B: Measurement time and current calculation](http://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BME280_DS001-11.pdf#page=51). +- `callback` if provided it will be invoked after given `delay`. The sensor reading should be finalized by then so. + +#### Returns +`nil` + +## bme280.setup() + +Initializes module. Initialization is mandatory before read values. + +#### Syntax + +`bme280.setup([temp_oss, press_oss, humi_oss, power_mode, inactive_duration, IIR_filter])` + +#### Parameters - (optional) `temp_oss` - Controls oversampling of temperature data. Default oversampling is 16x. - (optional) `press_oss` - Controls oversampling of pressure data. Default oversampling is 16x. - (optional) `humi_oss` - Controls oversampling of humidity data. Default oversampling is 16x @@ -127,7 +186,9 @@ Using forced mode is recommended for applications which require low sampling rat ```lua alt=320 -- altitude of the measurement place -bme280.init(3, 4) +sda, scl = 3, 4 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +bme280.setup() P, T = bme280.baro() print(string.format("QFE=%d.%03d", P/1000, P%1000)) @@ -157,7 +218,9 @@ Or simpler and more efficient ```lua alt=320 -- altitude of the measurement place -bme280.init(3, 4) +sda, scl = 3, 4 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +bme280.setup() T, P, H, QNH = bme280.read(alt) local Tsgn = (T < 0 and -1 or 1); T = Tsgn*T @@ -176,11 +239,13 @@ local curAltsgn = (curAlt < 0 and -1 or 1); curAlt = curAltsgn*curAlt print(string.format("altitude=%s%d.%02d", curAltsgn<0 and "-" or "", curAlt/100, curAlt%100)) ``` -Use `bme280.init(sda, scl, 1, 3, 0, 3, 0, 4)` for "game mode" - Oversampling settings pressure ×4, temperature ×1, humidity ×0, sensor mode: normal mode, inactive duration = 0.5 ms, IIR filter settings filter coefficient 16. +Use `bme280.setup(1, 3, 0, 3, 0, 4)` for "game mode" - Oversampling settings pressure ×4, temperature ×1, humidity ×0, sensor mode: normal mode, inactive duration = 0.5 ms, IIR filter settings filter coefficient 16. Example of readout in forced mode (asynchronous) ```lua -bme280.init(3, 4, nil, nil, nil, 0) -- initialize to sleep mode +sda, scl = 3, 4 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +bme280.setup(nil, nil, nil, 0) -- initialize to sleep mode bme280.startreadout(0, function () T, P = bme280.read() local Tsgn = (T < 0 and -1 or 1); T = Tsgn*T @@ -188,52 +253,6 @@ bme280.startreadout(0, function () end) ``` -## bme280.qfe2qnh() - -For given altitude converts the air pressure to sea level air pressure. - -#### Syntax -`bme280.qfe2qnh(P, altitude)` - -#### Parameters -- `P` measured pressure -- `altitude` altitude in meters of measurement point - -#### Returns -sea level pressure - - -## bme280.read() - -Reads the sensor and returns the temperature, the air pressure, the air relative humidity and - -#### Syntax -`bme280.read([altitude])` - -#### Parameters -- (optional) `altitude`- altitude in meters of measurement point. If provided also the air pressure converted to sea level air pressure is returned. - -#### Returns -- `T` temperature in celsius as an integer multiplied with 100 -- `P` air pressure in hectopascals multiplied by 1000 -- `H` relative humidity in percent multiplied by 1000 -- `QNH` air pressure in hectopascals multiplied by 1000 converted to sea level - -Any of these variables is `nil` if the readout of given measure was not successful. - -## bme280.startreadout() -Starts readout (turns the sensor into forced mode). After the readout the sensor turns to sleep mode. - -#### Syntax -`bme280.startreadout(delay, callback)` - -#### Parameters -- `delay` sets sensor to forced mode and calls the `callback` (if provided) after given number of milliseconds. For 0 the default delay is set to 113ms (sufficient time to perform reading for oversampling settings 16x). For different oversampling setting please refer to [BME280 Final Datasheet - Appendix B: Measurement time and current calculation](http://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BME280_DS001-11.pdf#page=51). -- `callback` if provided it will be invoked after given `delay`. The sensor reading should be finalized by then so. - -#### Returns -`nil` - ## bme280.temp() Reads the sensor and returns the temperature in celsius as an integer multiplied with 100. diff --git a/docs/en/modules/bmp085.md b/docs/en/modules/bmp085.md index 1bb96aa2..5df899ab 100644 --- a/docs/en/modules/bmp085.md +++ b/docs/en/modules/bmp085.md @@ -9,6 +9,10 @@ This module provides access to the [BMP085](https://www.sparkfun.com/tutorials/2 ## bmp085.init() Initializes the module and sets the pin configuration. +!!! attention + + This function is deprecated and will be removed in upcoming releases. Use `bmp085.setup()` instead. + #### Syntax `bmp085.init(sda, scl)` @@ -19,6 +23,18 @@ Initializes the module and sets the pin configuration. #### Returns `nil` +## bmp085.setup() +Initializes the module. + +#### Syntax +`bmp085.setup()` + +#### Parameters +None + +#### Returns +`nil` + ## bmp085.temperature() Samples the sensor and returns the temperature in celsius as an integer multiplied with 10. @@ -30,7 +46,9 @@ temperature multiplied with 10 (integer) #### Example ```lua -bmp085.init(1, 2) +local sda, scl = 1, 2 +i2c.setup(0, sda, scl, i2c.SLOW) +bmp085.setup() local t = bmp085.temperature() print(string.format("Temperature: %s.%s degrees C", t / 10, t % 10)) ``` @@ -53,7 +71,9 @@ pressure in pascals (integer) #### Example ```lua -bmp085.init(1, 2) +local sda, scl = 1, 2 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +bmp085.setup() local p = bmp085.pressure() print(string.format("Pressure: %s.%s mbar", p / 100, p % 100)) ``` diff --git a/docs/en/modules/hmc5883l.md b/docs/en/modules/hmc5883l.md index 213dc0c2..43f535f8 100644 --- a/docs/en/modules/hmc5883l.md +++ b/docs/en/modules/hmc5883l.md @@ -18,7 +18,9 @@ temperature multiplied with 10 (integer) #### Example ```lua -hmc58831.init(1, 2) +local sda, scl = 1, 2 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +hmc58831.setup() local x,y,z = hmc5883l.read() print(string.format("x = %d, y = %d, z = %d", x, y, z)) ``` @@ -26,6 +28,10 @@ print(string.format("x = %d, y = %d, z = %d", x, y, z)) ## hmc5883l.init() Initializes the module and sets the pin configuration. +!!! attention + + This function is deprecated and will be removed in upcoming releases. Use `hmc5883l.setup()` instead. + #### Syntax `hmc5883l.init(sda, scl)` @@ -35,3 +41,15 @@ Initializes the module and sets the pin configuration. #### Returns `nil` + +## hmc5883l.setup() +Initializes the module. + +#### Syntax +`hmc5883l.setup()` + +#### Parameters +None + +#### Returns +`nil` diff --git a/docs/en/modules/l3g4200d.md b/docs/en/modules/l3g4200d.md index 9f47918b..5f667cf5 100644 --- a/docs/en/modules/l3g4200d.md +++ b/docs/en/modules/l3g4200d.md @@ -17,7 +17,9 @@ X,Y,Z gyroscope output #### Example ```lua -l3g4200d.init(1, 2) +local sda, scl = 1, 2 +i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once +l3g4200d.setup() local x,y,z = l3g4200d.read() print(string.format("X = %d, Y = %d, Z = %d", x, y, z) ``` @@ -25,6 +27,10 @@ print(string.format("X = %d, Y = %d, Z = %d", x, y, z) ## l3g4200d.init() Initializes the module and sets the pin configuration. +!!! attention + + This function is deprecated and will be removed in upcoming releases. Use `l3g4200d.setup()` instead. + #### Syntax `l3g4200d.init(sda, scl)` @@ -34,3 +40,15 @@ Initializes the module and sets the pin configuration. #### Returns `nil` + +## l3g4200d.setup() +Initializes the module. + +#### Syntax +`l3g4200d.setup()` + +#### Parameters +None + +#### Returns +`nil` From 7b1f0223ada692acfb847cb97951862d202d4fb9 Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Sun, 21 May 2017 07:17:54 -0700 Subject: [PATCH 03/27] Removed wifi station event monitor (#1900) The following functions were removed: - `wifi.sta.eventMonReg()` - `wifi.sta.eventMonStart()` - `wifi.sta.eventMonStop()` The corresponding documentation has also been removed This PR addresses issue #1893 --- app/include/user_config.h | 1 - app/modules/wifi.c | 5 -- app/modules/wifi_common.h | 5 -- app/modules/wifi_eventmon.c | 94 ----------------------------- docs/en/modules/wifi.md | 116 ------------------------------------ 5 files changed, 221 deletions(-) diff --git a/app/include/user_config.h b/app/include/user_config.h index 393da734..f7c2a467 100644 --- a/app/include/user_config.h +++ b/app/include/user_config.h @@ -111,7 +111,6 @@ extern void luaL_assertfail(const char *file, int line, const char *message); //#define WIFI_SMART_ENABLE -#define WIFI_STATION_STATUS_MONITOR_ENABLE #define WIFI_SDK_EVENT_MONITOR_ENABLE #define WIFI_EVENT_MONITOR_DISCONNECT_REASON_LIST_ENABLE diff --git a/app/modules/wifi.c b/app/modules/wifi.c index e1c8f94c..d30382c7 100644 --- a/app/modules/wifi.c +++ b/app/modules/wifi.c @@ -1629,11 +1629,6 @@ static const LUA_REG_TYPE wifi_station_map[] = { { LSTRKEY( "config" ), LFUNCVAL( wifi_station_config ) }, { LSTRKEY( "connect" ), LFUNCVAL( wifi_station_connect4lua ) }, { LSTRKEY( "disconnect" ), LFUNCVAL( wifi_station_disconnect4lua ) }, -#if defined(WIFI_STATION_STATUS_MONITOR_ENABLE) - { LSTRKEY( "eventMonReg" ), LFUNCVAL( wifi_station_event_mon_reg ) }, //defined in wifi_eventmon.c - { LSTRKEY( "eventMonStart" ), LFUNCVAL( wifi_station_event_mon_start ) }, //defined in wifi_eventmon.c - { LSTRKEY( "eventMonStop" ), LFUNCVAL( wifi_station_event_mon_stop ) }, //defined in wifi_eventmon.c -#endif { LSTRKEY( "getap" ), LFUNCVAL( wifi_station_listap ) }, { LSTRKEY( "getapindex" ), LFUNCVAL( wifi_station_get_ap_index ) }, { LSTRKEY( "getapinfo" ), LFUNCVAL( wifi_station_get_ap_info4lua ) }, diff --git a/app/modules/wifi_common.h b/app/modules/wifi_common.h index df8a77a7..c77d89cc 100644 --- a/app/modules/wifi_common.h +++ b/app/modules/wifi_common.h @@ -65,10 +65,5 @@ enum wifi_suspension_state extern const LUA_REG_TYPE wifi_event_monitor_map[]; void wifi_eventmon_init(); #endif -#ifdef WIFI_STATION_STATUS_MONITOR_ENABLE - int wifi_station_event_mon_start(lua_State* L); - int wifi_station_event_mon_reg(lua_State* L); - void wifi_station_event_mon_stop(lua_State* L); -#endif #endif /* APP_MODULES_WIFI_COMMON_H_ */ diff --git a/app/modules/wifi_eventmon.c b/app/modules/wifi_eventmon.c index 05af3c38..14f32043 100644 --- a/app/modules/wifi_eventmon.c +++ b/app/modules/wifi_eventmon.c @@ -17,100 +17,6 @@ #if defined(LUA_USE_MODULES_WIFI) -#ifdef WIFI_STATION_STATUS_MONITOR_ENABLE - -//variables for wifi event monitor -static int wifi_station_status_cb_ref[6] = {[0 ... 6-1] = LUA_NOREF}; -static os_timer_t wifi_sta_status_timer; -static uint8 prev_wifi_status=0; - -// wifi.sta.eventMonStop() -void wifi_station_event_mon_stop(lua_State* L) -{ - os_timer_disarm(&wifi_sta_status_timer); - if(lua_isstring(L,1)) - { - int i; - for (i=0; i<6; i++) - { - unregister_lua_cb(L, &wifi_station_status_cb_ref[i]); - } - } - return; -} - -static void wifi_status_cb(int arg) -{ - lua_State* L = lua_getstate(); - if (wifi_get_opmode() == SOFTAP_MODE) - { - os_timer_disarm(&wifi_sta_status_timer); - return; - } - int wifi_status = wifi_station_get_connect_status(); - if (wifi_status != prev_wifi_status) - { - if(wifi_station_status_cb_ref[wifi_status] != LUA_NOREF) - { - lua_rawgeti(L, LUA_REGISTRYINDEX, wifi_station_status_cb_ref[wifi_status]); - lua_pushnumber(L, prev_wifi_status); - lua_call(L, 1, 0); - } - } - prev_wifi_status = wifi_status; -} - -// wifi.sta.eventMonReg() -int wifi_station_event_mon_reg(lua_State* L) -{ - platform_print_deprecation_note("wifi.sta.eventmonreg() is replaced by wifi.eventmon.register()", "in the next version"); - - uint8 id=(uint8)luaL_checknumber(L, 1); - if ((id > 5)) // verify user specified a valid wifi status - { - return luaL_error( L, "valid wifi status:0-5" ); - } - - if (lua_type(L, 2) == LUA_TFUNCTION || lua_type(L, 2) == LUA_TLIGHTFUNCTION) //check if 2nd item on stack is a function - { - lua_pushvalue(L, 2); //push function to top of stack - register_lua_cb(L, &wifi_station_status_cb_ref[id]);//pop function from top of the stack, register it in the LUA_REGISTRY, then assign returned lua_ref to wifi_station_status_cb_ref[id] - } - else - { - unregister_lua_cb(L, &wifi_station_status_cb_ref[id]); // unregister user's callback - } - return 0; -} - - -// wifi.sta.eventMonStart() -int wifi_station_event_mon_start(lua_State* L) -{ - if(wifi_get_opmode() == SOFTAP_MODE) //Verify ESP is in either Station mode or StationAP mode - { - return luaL_error( L, "Can't monitor in SOFTAP mode" ); - } - if (wifi_station_status_cb_ref[0] == LUA_NOREF && wifi_station_status_cb_ref[1] == LUA_NOREF && - wifi_station_status_cb_ref[2] == LUA_NOREF && wifi_station_status_cb_ref[3] == LUA_NOREF && - wifi_station_status_cb_ref[4] == LUA_NOREF && wifi_station_status_cb_ref[5] == LUA_NOREF ) - { //verify user has registered callbacks - return luaL_error( L, "No callbacks defined" ); - } - uint32 ms = 150; //set default timer interval - if(lua_isnumber(L, 1)) // check if user has specified a different timer interval - { - ms=luaL_checknumber(L, 1); // retrieve user-defined interval - } - - os_timer_disarm(&wifi_sta_status_timer); - os_timer_setfn(&wifi_sta_status_timer, (os_timer_func_t *)wifi_status_cb, NULL); - os_timer_arm(&wifi_sta_status_timer, ms, 1); - return 0; -} - -#endif - #ifdef WIFI_SDK_EVENT_MONITOR_ENABLE //variables for wifi event monitor diff --git a/docs/en/modules/wifi.md b/docs/en/modules/wifi.md index 0f81d36d..6775dbbb 100644 --- a/docs/en/modules/wifi.md +++ b/docs/en/modules/wifi.md @@ -489,116 +489,6 @@ none - [`wifi.sta.config()`](#wifistaconfig) - [`wifi.sta.connect()`](#wifistaconnect) -## wifi.sta.eventMonReg() - -Registers callbacks for WiFi station status events. - -!!! note - Please update your program to use the [`wifi.eventmon`](#wifieventmon-module) API, as the `wifi.sta.eventmon___()` API is deprecated. - -#### Syntax -- `wifi.sta.eventMonReg(wifi_status[, function([previous_state])])` - -#### Parameters -- `wifi_status` WiFi status you would like to set a callback for: - - `wifi.STA_IDLE` - - `wifi.STA_CONNECTING` - - `wifi.STA_WRONGPWD` - - `wifi.STA_APNOTFOUND` - - `wifi.STA_FAIL` - - `wifi.STA_GOTIP` -- `function` callback function to perform when event occurs - - Note: leaving field blank unregisters callback. -- `previous_state` previous wifi_state(0 - 5) - -#### Returns -`nil` - -#### Example -```lua ---register callback -wifi.sta.eventMonReg(wifi.STA_IDLE, function() print("STATION_IDLE") end) -wifi.sta.eventMonReg(wifi.STA_CONNECTING, function() print("STATION_CONNECTING") end) -wifi.sta.eventMonReg(wifi.STA_WRONGPWD, function() print("STATION_WRONG_PASSWORD") end) -wifi.sta.eventMonReg(wifi.STA_APNOTFOUND, function() print("STATION_NO_AP_FOUND") end) -wifi.sta.eventMonReg(wifi.STA_FAIL, function() print("STATION_CONNECT_FAIL") end) -wifi.sta.eventMonReg(wifi.STA_GOTIP, function() print("STATION_GOT_IP") end) - ---register callback: use previous state -wifi.sta.eventMonReg(wifi.STA_CONNECTING, function(previous_State) - if(previous_State==wifi.STA_GOTIP) then - print("Station lost connection with access point\n\tAttempting to reconnect...") - else - print("STATION_CONNECTING") - end -end) - ---unregister callback -wifi.sta.eventMonReg(wifi.STA_IDLE) -``` -#### See also -- [`wifi.sta.eventMonStart()`](#wifistaeventmonstart) -- [`wifi.sta.eventMonStop()`](#wifistaeventmonstop) -- [`wifi.eventmon.register()`](#wifieventmonregister) -- [`wifi.eventmon.unregister()`](#wifieventmonunregister) - - -## wifi.sta.eventMonStart() - -Starts WiFi station event monitor. - -#### Syntax -`wifi.sta.eventMonStart([ms])` - -### Parameters -- `ms` interval between checks in milliseconds, defaults to 150ms if not provided. - -#### Returns -`nil` - -#### Example -```lua ---start WiFi event monitor with default interval -wifi.sta.eventMonStart() - ---start WiFi event monitor with 100ms interval -wifi.sta.eventMonStart(100) -``` - -#### See also -- [`wifi.sta.eventMonReg()`](#wifistaeventmonreg) -- [`wifi.sta.eventMonStop()`](#wifistaeventmonstop) -- [`wifi.eventmon.register()`](#wifieventmonregister) -- [`wifi.eventmon.unregister()`](#wifieventmonunregister) - -## wifi.sta.eventMonStop() - -Stops WiFi station event monitor. -#### Syntax -`wifi.sta.eventMonStop([unregister_all])` - -#### Parameters -- `unregister_all` enter 1 to unregister all previously registered functions. - - Note: leave blank to leave callbacks registered - -#### Returns -`nil` - -#### Example -```lua ---stop WiFi event monitor -wifi.sta.eventMonStop() - ---stop WiFi event monitor and unregister all callbacks -wifi.sta.eventMonStop(1) -``` - -#### See also -- [`wifi.sta.eventMonReg()`](#wifistaeventmonreg) -- [`wifi.sta.eventMonStart()`](#wifistaeventmonstart) -- [`wifi.eventmon.register()`](#wifieventmonregister) -- [`wifi.eventmon.unregister()`](#wifieventmonunregister) - ## wifi.sta.getap() Scans AP list as a Lua table into callback function. @@ -1464,7 +1354,6 @@ none boolean indicating success # wifi.eventmon Module -Note: The functions `wifi.sta.eventMon___()` and `wifi.eventmon.___()` are completely seperate and can be used independently of one another. ## wifi.eventmon.register() @@ -1573,9 +1462,6 @@ T: Table returned by event. ``` #### See also - [`wifi.eventmon.unregister()`](#wifieventmonunregister) -- [`wifi.sta.eventMonStart()`](#wifistaeventmonstart) -- [`wifi.sta.eventMonStop()`](#wifistaeventmonstop) -- [`wifi.sta.eventMonReg()`](#wifistaeventmonreg) ## wifi.eventmon.unregister() @@ -1608,8 +1494,6 @@ Event: WiFi event you would like to set a callback for. ``` #### See also - [`wifi.eventmon.register()`](#wifieventmonregister) -- [`wifi.sta.eventMonStart()`](#wifistaeventmonstart) -- [`wifi.sta.eventMonStop()`](#wifistaeventmonstop) ## wifi.eventmon.reason From 169cb69ee238a11f61379cd4ec4f16381049e2ba Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Sun, 21 May 2017 07:18:56 -0700 Subject: [PATCH 04/27] Add event callbacks to wifi.sta.config() and wifi.ap.config() and more (#1903) * Add event callbacks to wifi.sta.config() and wifi.ap.config() and more Added option to register event callbacks during configuration of both station and AP. Added option to register event callbacks to `wifi.sta.connect()` and `wifi.sta.disconnect()` * Add note about event registration to wifi module documentation Other minor changes to wifi documentation are also included * Add more detail to documentation for wifi.sta.config() --- app/modules/wifi.c | 169 ++++++++++++++++++++++++++++++++++++ app/modules/wifi_common.h | 1 + app/modules/wifi_eventmon.c | 2 +- docs/en/modules/wifi.md | 59 +++++++++++-- 4 files changed, 224 insertions(+), 7 deletions(-) diff --git a/app/modules/wifi.c b/app/modules/wifi.c index d30382c7..9bb24e71 100644 --- a/app/modules/wifi.c +++ b/app/modules/wifi.c @@ -833,6 +833,102 @@ static int wifi_station_config( lua_State* L ) } lua_pop(L, 1); +#ifdef WIFI_SDK_EVENT_MONITOR_ENABLE + + lua_State* L_temp = NULL; + + lua_getfield(L, 1, "connect_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_STAMODE_CONNECTED); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "connect_cb:not function"); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "disconnect_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_STAMODE_DISCONNECTED); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "disconnect_cb:not function"); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "authmode_change_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_STAMODE_AUTHMODE_CHANGE); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "authmode_change_cb:not function"); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "got_ip_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_STAMODE_GOT_IP); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "gotip_cb:not function"); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "dhcp_timeout_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_STAMODE_DHCP_TIMEOUT); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "dhcp_timeout_cb:not function"); + } + } + lua_pop(L, 1); + +#endif + } else //to be deprecated { @@ -930,6 +1026,13 @@ static int wifi_station_config( lua_State* L ) // Lua: wifi.sta.connect() static int wifi_station_connect4lua( lua_State* L ) { +#ifdef WIFI_SDK_EVENT_MONITOR_ENABLE + if(lua_isfunction(L, 1)){ + lua_pushnumber(L, EVENT_STAMODE_CONNECTED); + lua_pushvalue(L, 1); + wifi_event_monitor_register(L); + } +#endif wifi_station_connect(); return 0; } @@ -937,6 +1040,13 @@ static int wifi_station_connect4lua( lua_State* L ) // Lua: wifi.sta.disconnect() static int wifi_station_disconnect4lua( lua_State* L ) { +#ifdef WIFI_SDK_EVENT_MONITOR_ENABLE + if(lua_isfunction(L, 1)){ + lua_pushnumber(L, EVENT_STAMODE_DISCONNECTED); + lua_pushvalue(L, 1); + wifi_event_monitor_register(L); + } +#endif wifi_station_disconnect(); return 0; } @@ -1509,6 +1619,65 @@ static int wifi_ap_config( lua_State* L ) } lua_pop(L, 1); +#ifdef WIFI_SDK_EVENT_MONITOR_ENABLE + + lua_State* L_temp = NULL; + + lua_getfield(L, 1, "staconnected_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_SOFTAPMODE_STACONNECTED); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "staconnected_cb:not function"); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "stadisconnected_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_SOFTAPMODE_STADISCONNECTED); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "stadisconnected_cb:not function"); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "probereq_cb"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + L_temp = lua_newthread(L); + lua_pushnumber(L, EVENT_SOFTAPMODE_PROBEREQRECVED); + lua_pushvalue(L, -3); + lua_xmove(L, L_temp, 2); + wifi_event_monitor_register(L_temp); + } + else + { + return luaL_argerror(L, 1, "probereq_cb:not function"); + } + } + lua_pop(L, 1); + +#endif #if defined(WIFI_DEBUG) char debug_temp[sizeof(config.password)+1]; diff --git a/app/modules/wifi_common.h b/app/modules/wifi_common.h index c77d89cc..c88e91c0 100644 --- a/app/modules/wifi_common.h +++ b/app/modules/wifi_common.h @@ -64,6 +64,7 @@ enum wifi_suspension_state #ifdef WIFI_SDK_EVENT_MONITOR_ENABLE extern const LUA_REG_TYPE wifi_event_monitor_map[]; void wifi_eventmon_init(); + int wifi_event_monitor_register(lua_State* L); #endif #endif /* APP_MODULES_WIFI_COMMON_H_ */ diff --git a/app/modules/wifi_eventmon.c b/app/modules/wifi_eventmon.c index 14f32043..00277594 100644 --- a/app/modules/wifi_eventmon.c +++ b/app/modules/wifi_eventmon.c @@ -32,7 +32,7 @@ static evt_queue_t *wifi_event_queue_tail; //pointer to end of queue static int wifi_event_cb_ref[EVENT_MAX+1] = { [0 ... EVENT_MAX] = LUA_NOREF}; //holds references to registered Lua callbacks // wifi.eventmon.register() -static int wifi_event_monitor_register(lua_State* L) +int wifi_event_monitor_register(lua_State* L) { uint8 id = (uint8)luaL_checknumber(L, 1); if ( id > EVENT_MAX ) //Check if user is trying to register a callback for a valid event. diff --git a/docs/en/modules/wifi.md b/docs/en/modules/wifi.md index 6775dbbb..2339683b 100644 --- a/docs/en/modules/wifi.md +++ b/docs/en/modules/wifi.md @@ -389,6 +389,9 @@ none Sets the WiFi station configuration. +!!! note + It is not advised to assume that the WiFi is connected at any time during initialization start-up. WiFi connection status should be validated either by using a WiFi event callback or by polling the status on a timer. + #### Syntax `wifi.sta.config(station_config)` @@ -408,7 +411,30 @@ Sets the WiFi station configuration. - "DE AD BE EF 7A C0" - `save` Save station configuration to flash. - `true` configuration **will** be retained through power cycle. - - `false` configuration **will not** be retained through power cycle. (Default) + - `false` configuration **will not** be retained through power cycle. (Default). + - Event callbacks will only be available if `WIFI_SDK_EVENT_MONITOR_ENABLE` is uncommented in `user_config.h` + - Please note: To ensure all station events are handled at boot time, all relevant callbacks must be registered as early as possible in `init.lua` with either `wifi.sta.config()` or `wifi.eventmon.register()`. + - `connected_cb`: Callback to execute when station is connected to an access point. (Optional) + - Items returned in table : + - `SSID`: SSID of access point. (format: string) + - `BSSID`: BSSID of access point. (format: string) + - `channel`: The channel the access point is on. (format: number) + - `disconnected_cb`: Callback to execute when station is disconnected from an access point. (Optional) + - Items returned in table : + - `SSID`: SSID of access point. (format: string) + - `BSSID`: BSSID of access point. (format: string) + - `REASON`: See [wifi.eventmon.reason](#wifieventmonreason) below. (format: number) + - `authmode_change_cb`: Callback to execute when the access point has changed authorization mode. (Optional) + - Items returned in table : + - `old_auth_mode`: Old wifi authorization mode. (format: number) + - `new_auth_mode`: New wifi authorization mode. (format: number) + - `got_ip_cb`: Callback to execute when the station received an IP address from the access point. (Optional) + - Items returned in table : + - `IP`: The IP address assigned to the station. (format: string) + - `netmask`: Subnet mask. (format: string) + - `gateway`: The IP address of the access point the station is connected to. (format: string) + - `dhcp_timeout_cb`: Station DHCP request has timed out. (Optional) + - Blank table is returned. #### Returns - `true` Success @@ -457,10 +483,14 @@ wifi.sta.config(station_cfg) Connects to the configured AP in station mode. You only ever need to call this if auto-connect was disabled in [`wifi.sta.config()`](#wifistaconfig). #### Syntax -`wifi.sta.connect()` +`wifi.sta.connect([connected_cb])` #### Parameters -none +- `connected_cb`: Callback to execute when station is connected to an access point. (Optional) + - Items returned in table : + - `SSID`: SSID of access point. (format: string) + - `BSSID`: BSSID of access point. (format: string) + - `channel`: The channel the access point is on. (format: number) #### Returns `nil` @@ -477,10 +507,14 @@ Disconnects from AP in station mode. Please note that disconnecting from Access Point does not reduce power consumption. If power saving is your goal, please refer to the description for `wifi.NULLMODE` in the function [`wifi.setmode()`](#wifisetmode) for more details. #### Syntax -`wifi.sta.disconnect()` +`wifi.sta.disconnect([disconnected_cb])` #### Parameters -none +- `disconnected_cb`: Callback to execute when station is disconnected from an access point. (Optional) + - Items returned in table : + - `SSID`: SSID of access point. (format: string) + - `BSSID`: BSSID of access point. (format: string) + - `REASON`: See [wifi.eventmon.reason](#wifieventmonreason) below. (format: number) #### Returns `nil` @@ -1011,7 +1045,20 @@ Sets SSID and password in AP mode. Be sure to make the password at least 8 chara - `save` save configuration to flash. - `true` configuration **will** be retained through power cycle. (Default) - `false` configuration **will not** be retained through power cycle. - + - Event callbacks will only be available if `WIFI_SDK_EVENT_MONITOR_ENABLE` is uncommented in `user_config.h` + - Please note: To ensure all SoftAP events are handled at boot time, all relevant callbacks must be registered as early as possible in `init.lua` with either `wifi.ap.config()` or `wifi.eventmon.register()`. + - `staconnected_cb`: Callback executed when a new client has connected to the access point. (Optional) + - Items returned in table : + - `MAC`: MAC address of client that has connected. + - `AID`: SDK provides no details concerning this return value. + - `stadisconnected_cb`: Callback executed when a client has disconnected from the access point. (Optional) + - Items returned in table : + - `MAC`: MAC address of client that has disconnected. + - `AID`: SDK provides no details concerning this return value. + - `probereq_cb`: Callback executed when a probe request was received. (Optional) + - Items returned in table : + - `MAC`: MAC address of the client that is probing the access point. + - `RSSI`: Received Signal Strength Indicator of client. #### Returns - `true` Success From e90ffb42667da287537c269c967cb9d723edb376 Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Sun, 21 May 2017 07:30:26 -0700 Subject: [PATCH 05/27] Add mcp4725 module (#1966) --- app/include/user_modules.h | 1 + app/modules/mcp4725.c | 217 +++++++++++++++++++++++++++++++++++++ docs/en/modules/mcp4725.md | 146 +++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 365 insertions(+) create mode 100644 app/modules/mcp4725.c create mode 100644 docs/en/modules/mcp4725.md diff --git a/app/include/user_modules.h b/app/include/user_modules.h index 06714516..74bf1719 100644 --- a/app/include/user_modules.h +++ b/app/include/user_modules.h @@ -40,6 +40,7 @@ //#define LUA_USE_MODULES_HX711 #define LUA_USE_MODULES_I2C //#define LUA_USE_MODULES_L3G4200D +//#define LUA_USE_MODULES_MCP4725 //#define LUA_USE_MODULES_MDNS #define LUA_USE_MODULES_MQTT #define LUA_USE_MODULES_NET diff --git a/app/modules/mcp4725.c b/app/modules/mcp4725.c new file mode 100644 index 00000000..b9117746 --- /dev/null +++ b/app/modules/mcp4725.c @@ -0,0 +1,217 @@ +/* + * Driver for Microchip MCP4725 12-bit digital to analog converter. + */ + +#include "module.h" +#include "lauxlib.h" +#include "platform.h" +#include "osapi.h" + +#define MCP4725_I2C_ADDR_BASE (0x60) +#define MCP4725_I2C_ADDR_A0_MASK (0x01) // user configurable +#define MCP4725_I2C_ADDR_A1_MASK (0x02) // hard wired at factory +#define MCP4725_I2C_ADDR_A2_MASK (0x04) // hard wired at factory + +#define MCP4725_COMMAND_WRITE_DAC (0x40) +#define MCP4725_COMMAND_WRITE_DAC_EEPROM (0x60) + +#define MCP4725_POWER_DOWN_NORMAL (0x00) +#define MCP4725_POWER_DOWN_RES_1K (0x02) +#define MCP4725_POWER_DOWN_RES_100K (0x04) +#define MCP4725_POWER_DOWN_RES_500K (0x06) + +static const unsigned mcp4725_i2c_id = 0; + +static uint8 get_address(lua_State* L, uint8 i2c_address){ + uint8 addr_temp = i2c_address; + uint16 temp_var = 0; + lua_getfield(L, 1, "A2"); + if (!lua_isnil(L, -1)) + { + if( lua_isnumber(L, -1) ) + { + temp_var = lua_tonumber(L, -1); + if(temp_var < 2){ + temp_var = MCP4725_I2C_ADDR_A2_MASK & (temp_var << 2); + addr_temp|=temp_var; + } + else + return luaL_argerror( L, 1, "A2: Must be 0 or 1" ); + } + else + { + return luaL_argerror( L, 1, "A2: Must be number" ); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "A1"); + if (!lua_isnil(L, -1)) + { + if( lua_isnumber(L, -1) ) + { + temp_var = lua_tonumber(L, -1); + if(temp_var < 2){ + temp_var = MCP4725_I2C_ADDR_A1_MASK & (temp_var << 1); + addr_temp|=temp_var; + } + else + return luaL_argerror( L, 1, "A1: Must be 0 or 1" ); + } + else + { + return luaL_argerror( L, 1, "A1: Must be number" ); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "A0"); + if (!lua_isnil(L, -1)) + { + if( lua_isnumber(L, -1) ) + { + temp_var = lua_tonumber(L, -1); + if(temp_var<2){ + temp_var = MCP4725_I2C_ADDR_A0_MASK & (temp_var); + addr_temp|=temp_var; + } + else + return luaL_argerror( L, 1, "A0: Must be 0 or 1" ); + } + else + { + return luaL_argerror( L, 1, "A0: Must be number" ); + } + } + lua_pop(L, 1); + + return addr_temp; +} + +static int mcp4725_write(lua_State* L){ + + uint8 i2c_address = MCP4725_I2C_ADDR_BASE; + uint16 dac_value = 0; + uint8 cmd_byte = 0; + + if(lua_istable(L, 1)) + { + i2c_address = get_address(L, i2c_address); + uint16 temp_var=0; + lua_getfield(L, 1, "value"); + if (!lua_isnil(L, -1)) + { + if( lua_isnumber(L, -1) ) + { + temp_var = lua_tonumber(L, -1); + if(temp_var >= 0 && temp_var<=4095){ + dac_value = temp_var<<4; + } + else + return luaL_argerror( L, 1, "value: Valid range 0-4095" ); + } + else + { + return luaL_argerror( L, 1, "value: Must be number" ); + } + } + else + { + return luaL_argerror( L, 1, "value: value is required" ); + } + lua_pop(L, 1); + + lua_getfield(L, 1, "save"); + if (!lua_isnil(L, -1)) + { + if( lua_isboolean(L, -1) ) + { + if(lua_toboolean(L, -1)){ + cmd_byte |= MCP4725_COMMAND_WRITE_DAC_EEPROM; + } + else{ + cmd_byte |= MCP4725_COMMAND_WRITE_DAC; + } + } + else + { + return luaL_argerror( L, 1, "save: must be boolean" ); + } + } + else + { + cmd_byte |= MCP4725_COMMAND_WRITE_DAC; + } + lua_pop(L, 1); + + lua_getfield(L, 1, "pwrdn"); + if (!lua_isnil(L, -1)) + { + if( lua_isnumber(L, -1) ) + { + temp_var = lua_tonumber(L, -1); + if(temp_var >= 0 && temp_var <= 3){ + cmd_byte |= temp_var << 1; + } + else{ + return luaL_argerror( L, 1, "pwrdn: Valid range 0-3" ); + } + } + else + { + return luaL_argerror( L, 1, "pwrdn: Must be number" ); + } + } + lua_pop(L, 1); + + } + uint8 *dac_value_byte = (uint8*) & dac_value; + + platform_i2c_send_start(mcp4725_i2c_id); + platform_i2c_send_address(mcp4725_i2c_id, i2c_address, PLATFORM_I2C_DIRECTION_TRANSMITTER); + platform_i2c_send_byte(mcp4725_i2c_id, cmd_byte); + platform_i2c_send_byte(mcp4725_i2c_id, dac_value_byte[1]); + platform_i2c_send_byte(mcp4725_i2c_id, dac_value_byte[0]); + platform_i2c_send_stop(mcp4725_i2c_id); + + return 0; +} + +static int mcp4725_read(lua_State* L){ + uint8 i2c_address = MCP4725_I2C_ADDR_BASE; + uint8 recieve_buffer[5] = {0}; + + if(lua_istable(L, 1)) + { + i2c_address = get_address(L, i2c_address); + } + + platform_i2c_send_start(mcp4725_i2c_id); + platform_i2c_send_address(mcp4725_i2c_id, i2c_address, PLATFORM_I2C_DIRECTION_RECEIVER); + for(int i=0;i<5;i++){ + recieve_buffer[i] = platform_i2c_recv_byte(mcp4725_i2c_id, 1); + } + platform_i2c_send_stop(mcp4725_i2c_id); + + lua_pushnumber(L, (recieve_buffer[0] & 0x06)>>1); + lua_pushnumber(L, (recieve_buffer[1] << 4) | (recieve_buffer[2] >> 4)); + lua_pushnumber(L, (recieve_buffer[3] & 0x60) >> 5); + lua_pushnumber(L, ((recieve_buffer[3] & 0xf) << 8) | recieve_buffer[4]); + lua_pushnumber(L, (recieve_buffer[0] & 0x80) >> 7); + lua_pushnumber(L, (recieve_buffer[0] & 0x40) >> 6); + + return 6; +} + + +static const LUA_REG_TYPE mcp4725_map[] = { + { LSTRKEY( "write" ), LFUNCVAL( mcp4725_write ) }, + { LSTRKEY( "read" ), LFUNCVAL( mcp4725_read ) }, + { LSTRKEY( "PWRDN_NONE" ), LNUMVAL(MCP4725_POWER_DOWN_NORMAL) }, + { LSTRKEY( "PWRDN_1K" ), LNUMVAL((MCP4725_POWER_DOWN_RES_1K)>>1) }, + { LSTRKEY( "PWRDN_100K" ), LNUMVAL((MCP4725_POWER_DOWN_RES_100K)>>1) }, + { LSTRKEY( "PWRDN_500K" ), LNUMVAL((MCP4725_POWER_DOWN_RES_500K)>>1) }, + { LNILKEY, LNILVAL} +}; + +NODEMCU_MODULE(MCP4725, "mcp4725", mcp4725_map, NULL); diff --git a/docs/en/modules/mcp4725.md b/docs/en/modules/mcp4725.md new file mode 100644 index 00000000..8db5b4c1 --- /dev/null +++ b/docs/en/modules/mcp4725.md @@ -0,0 +1,146 @@ +# MCP4725 Module +| Since | Origin / Contributor | Maintainer | Source | +| :----- | :-------------------- | :---------- | :------ | +| 2017-05-10 | [dnc40085](https://github.com/dnc40085) | [dnc40085](https://github.com/dnc40085) | [mcp4725.c](../../../app/modules/mcp4725.c)| + + +This module provides access to the [MCP4725 12-bit Digital to Analog Converter](http://ww1.microchip.com/downloads/en/DeviceDoc/22039d.pdf). + +!!!important: + VDD is the power supply pin for the device. The voltage at the VDD pin is used as the supply input as well as the DAC reference input. The power supply at the VDD pin should be clean as possible for good DAC performance. + +!!!note: + The MCP4725 device address contains four fixed bits ( 1100 = device code) and three address bits (A2, A1, A0). The A2 and A1 bits are hard-wired during manufacturing, and A0 bit is determined by the logic state of A0 pin. The A0 pin can be connected to VDD or VSS , or actively driven by digital logic levels. The address pin(A0) can be actively driven by a GPIO to act as a chip select, allowing more than 2 devices to be used on the same bus. + +## mcp4725.read() +Gets contents of the dac register and EEPROM. + +#### Syntax +`mcp4725.read({[a0], [a1], [a2]})` + +#### Parameters +- `A0` Address bit 0. This bit is user configurable via MCP4725 pin 6(A0). (valid states: 0 or 1) (default: 0) +- `A1` Address bit 1. This bit is hard-wired during manufacture. (valid states: 0 or 1) (default: 0) + - Note: Modules purchased from Adafruit have this bit(A1) set high(1). +- `A2` Address bit 2. This bit is hard-wired during manufacture. (valid states: 0 or 1) (default: 0) + +#### Returns +* `cur_pwrdn` Current power down configuration value. +* `cur_val` Current value stored in dac register. +* `eeprom_pwrdn` Power down configuration stored in EEPROM. +* `eeprom_val` DAC value stored in EEPROM. +* `eeprom_state` EEPROM write status + * `0` EEPROM write is incomplete. + * `1` EEPROM write has completed +* `por_state` Power-On-Reset status; + * `0` The MCP4725 is performing reset and is not ready. + * `1` The MCP4725 has sucessfully performed reset. + +#### Example +```lua +-- Get current configuration using default i2c address 0x60(A0=0, A1=0, A2=0). +do +local ID = 0 +local SDA = 6 +local SCL = 5 + +i2c.setup(ID, SDA, SCL, i2c.SLOW) + +local cur_pwrdn, cur_val, eeprom_pwrdn, eeprom_val, eeprom_state, por_state = mcp4725.read() + +print("\n Current configuration:\n\tpower down value: "..cur_pwrdn.."\n\tdac value: "..cur_val) +print(" Configuration stored in EEPROM:\n\tpower down value: "..eeprom_pwrdn.."\n\tdac value: "..eeprom_val) +print(" EEPROM write state: "..(eeprom_state==1 and "Completed" or "incomplete")) +print(" Power-On-Reset state: "..(por_state==1 and "Completed" or "incomplete")) +end + +-- Get current configuration using default i2c address 0x60(A0=0, A1=0, A2=0). +-- The MCP4725's address pin(A0) is being driven with gpio 4(pin 2). +do +local ID = 0 +local SDA = 6 +local SCL = 5 +local mcp4725_chip_sel = 2 + +i2c.setup(ID, SDA, SCL, i2c.SLOW) +gpio.mode(mcp4725_chip_sel, gpio.OUTPUT, gpio.PULLUP) + +gpio.write(mcp4725_chip_sel, 1) +local cur_pwrdn, cur_val, eeprom_pwrdn, eeprom_val, eeprom_state, por_state = mcp4725.read({A0=1}) +gpio.write(mcp4725_chip_sel, 0) + +print("\n Current configuration:\n\tpower down value: "..cur_pwrdn.."\n\tdac value: "..cur_val) +print(" Configuration stored in EEPROM:\n\tpower down value: "..eeprom_pwrdn.."\n\tdac value: "..eeprom_val) +print(" EEPROM write state: "..(eeprom_state==1 and "Completed" or "incomplete")) +print(" Power-On-Reset state: "..(por_state==1 and "Completed" or "incomplete")) +end +``` +#### See also +- [`i2c.setup()`](i2c.md#i2csetup) + + +## mcp4725.write() +Write configuration to dac register or dac register and eeprom. + +#### Syntax +`mcp4725.write({[a0], [a1], [a2], value, [pwrdn], [save]})` + +#### Parameters +- `A0` Address bit 0. This bit is user configurable via MCP4725 pin 6(A0). (valid states: 0 or 1) (default: 0) +- `A1` Address bit 1. This bit is hard-wired during manufacture. (valid states: 0 or 1) (default: 0) + - Note: Modules purchased from Adafruit have this bit(A1) set high(1). +- `A2` Address bit 2. This bit is hard-wired during manufacture. (valid states: 0 or 1) (default: 0) +- `value` The value to be used to configure DAC (and EEPROM). (Range: 0 - 4095) +- `pwrdn` Set power down bits. + - `mcp4725.PWRDN_NONE` MCP4725 output enabled. (Default) + - `mcp4725.PWRDN_1K` MCP4725 output disabled, output pulled to ground via 1K restistor. + - `mcp4725.PWRDN_100K` MCP4725 output disabled, output pulled to ground via 100K restistor. + - `mcp4725.PWRDN_500K` MCP4725 output disabled, output pulled to ground via 500K restistor. +- `save` Save pwrdn and dac values to EEPROM. (Values are loaded on power-up or during reset.) + - `true` Save configuration to EEPROM. + - `false` Do not save configuration to EEPROM. (Default) + +#### Returns +nil + +#### Example +```lua + +-- Set current configuration using default i2c address 0x60(A0=0, A1=0, A2=0). +do + local ID = 0 + local SDA = 6 + local SCL = 5 + + i2c.setup(ID, SDA, SCL, i2c.SLOW) + mcp4725.write({value=2048}) +end + +-- Set current configuration and save to EEPROM using default i2c address 0x60(A0=0, A1=0, A2=0). +do + local ID = 0 + local SDA = 6 + local SCL = 5 + + i2c.setup(ID, SDA, SCL, i2c.SLOW) + mcp4725.write({value=2048, save=true}) +end + +-- Set current configuration using default i2c address 0x60(A0=0, A1=0, A2=0). +-- The MCP4725's address pin(A0) is being driven with gpio 4(pin 2). +do + local ID = 0 + local SDA = 6 + local SCL = 5 + local mcp4725_chip_sel = 2 + + i2c.setup(ID, SDA, SCL, i2c.SLOW) + gpio.mode(mcp4725_chip_sel, gpio.OUTPUT, gpio.PULLUP) + + gpio.write(mcp4725_chip_sel, 1) + mcp4725.read({A0=1, value}) + gpio.write(mcp4725_chip_sel, 0) +end +``` +#### See also +- [`i2c.setup()`](i2c.md#i2csetup) diff --git a/mkdocs.yml b/mkdocs.yml index 613cdd81..e200d50e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ pages: - 'hx711' : 'en/modules/hx711.md' - 'i2c' : 'en/modules/i2c.md' - 'l3g4200d' : 'en/modules/l3g4200d.md' + - 'mcp4725': 'en/modules/mcp4725.md' - 'mdns': 'en/modules/mdns.md' - 'mqtt': 'en/modules/mqtt.md' - 'net': 'en/modules/net.md' From 2c553583ea7d6b7a0f6524668b0869960c0a1db2 Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Wed, 24 May 2017 00:37:21 -0700 Subject: [PATCH 06/27] Fixed incorrect documentation for wifi.sta.setaplimit (#1986) --- docs/en/modules/wifi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/modules/wifi.md b/docs/en/modules/wifi.md index 2339683b..1d9a3026 100644 --- a/docs/en/modules/wifi.md +++ b/docs/en/modules/wifi.md @@ -905,7 +905,7 @@ Set Maximum number of Access Points to store in flash. `wifi.sta.setaplimit(qty)` #### Parameters -`qty` Quantity of Access Points to store in flash. Range: 1-5 (Default: 5) +`qty` Quantity of Access Points to store in flash. Range: 1-5 (Default: 1) #### Returns - `true` Success From 216b820d08ed4c1fc189b00ad3ed7e3b86616f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnim=20L=C3=A4uger?= Date: Thu, 25 May 2017 13:59:45 +0200 Subject: [PATCH 07/27] Ensure standard DHCP message length when sending response to clients (#1985) --- app/include/lwip/app/dhcpserver.h | 2 ++ app/lwip/app/dhcpserver.c | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app/include/lwip/app/dhcpserver.h b/app/include/lwip/app/dhcpserver.h index 1edbc38b..d62b5f40 100644 --- a/app/include/lwip/app/dhcpserver.h +++ b/app/include/lwip/app/dhcpserver.h @@ -5,6 +5,8 @@ #define USE_DNS +#define DHCP_MSGOPTIONS_MIN_LEN 312 + typedef struct dhcps_state{ sint16_t state; } dhcps_state; diff --git a/app/lwip/app/dhcpserver.c b/app/lwip/app/dhcpserver.c index 1353e702..b4ad7e83 100755 --- a/app/lwip/app/dhcpserver.c +++ b/app/lwip/app/dhcpserver.c @@ -298,7 +298,8 @@ static void ICACHE_FLASH_ATTR send_offer(struct dhcps_msg *m) end = add_offer_options(end); end = add_end(end); - p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg), PBUF_RAM); + // ensure that not more than the minimum options length is transmitted + p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg) - sizeof(m->options) + DHCP_MSGOPTIONS_MIN_LEN, PBUF_RAM); #if DHCPS_DEBUG os_printf("udhcp: send_offer>>p->ref = %d\n", p->ref); #endif @@ -358,7 +359,8 @@ static void ICACHE_FLASH_ATTR send_nak(struct dhcps_msg *m) end = add_msg_type(&m->options[4], DHCPNAK); end = add_end(end); - p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg), PBUF_RAM); + // ensure that not more than the minimum options length is transmitted + p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg) - sizeof(m->options) + DHCP_MSGOPTIONS_MIN_LEN, PBUF_RAM); #if DHCPS_DEBUG os_printf("udhcp: send_nak>>p->ref = %d\n", p->ref); #endif @@ -418,8 +420,9 @@ static void ICACHE_FLASH_ATTR send_ack(struct dhcps_msg *m) end = add_msg_type(&m->options[4], DHCPACK); end = add_offer_options(end); end = add_end(end); - - p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg), PBUF_RAM); + + // ensure that not more than the minimum options length is transmitted + p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg) - sizeof(m->options) + DHCP_MSGOPTIONS_MIN_LEN, PBUF_RAM); #if DHCPS_DEBUG os_printf("udhcp: send_ack>>p->ref = %d\n", p->ref); #endif From 438f1609f60f7336e41fe773f0665ac2afbf24c2 Mon Sep 17 00:00:00 2001 From: Philip Gladstone Date: Sun, 18 Jun 2017 02:20:26 -0400 Subject: [PATCH 08/27] Only handle errors if we have not parsed a complete object (#1999) * Only handle errors if we have not parsed a complete object * Fix typo which means that \n was not an ending character --- app/modules/sjson.c | 4 +++- app/sjson/jsonsl.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/modules/sjson.c b/app/modules/sjson.c index 5b4be126..ada01b5b 100644 --- a/app/modules/sjson.c +++ b/app/modules/sjson.c @@ -56,7 +56,9 @@ static int error_callback(jsonsl_t jsn, char *at) { JSN_DATA *data = (JSN_DATA *) jsn->data; - data->error = jsonsl_strerror(err); + if (!data->complete) { + data->error = jsonsl_strerror(err); + } //fprintf(stderr, "Got error at pos %lu: %s\n", jsn->pos, jsonsl_strerror(err)); return 0; diff --git a/app/sjson/jsonsl.c b/app/sjson/jsonsl.c index 9d2c1072..272ee491 100644 --- a/app/sjson/jsonsl.c +++ b/app/sjson/jsonsl.c @@ -1484,7 +1484,7 @@ static const char Special_Endings[0x100] = { /* 0xfe */ 0 /* 0xfe */ }; static const uint32_t Special_Endings_bits[0x80 / 32] = { - 0b00000000110010000000000000000000, + 0b00000000011001000000000000000000, 0b10100000000010000000000000100000, 0b00000000000000000000000000011100, 0b00000000000000000000000000010100 From 827642b49a39dc4d4914bbcbc0b25917d9d004cb Mon Sep 17 00:00:00 2001 From: Terry Ellison Date: Mon, 19 Jun 2017 06:47:02 +0100 Subject: [PATCH 09/27] Version 2.0 of the Lua Develoer FAQ (#1899) --- docs/en/lua-developer-faq.md | 297 ++++++++++++++++++++--------------- 1 file changed, 172 insertions(+), 125 deletions(-) diff --git a/docs/en/lua-developer-faq.md b/docs/en/lua-developer-faq.md index b0a29944..5bc14458 100644 --- a/docs/en/lua-developer-faq.md +++ b/docs/en/lua-developer-faq.md @@ -1,12 +1,28 @@ # FAQ -**# # # Work in Progress # # #** +*This FAQ was started by [Terry Ellison](https://github.com/TerryE) as an unofficial FAQ in mid 2015. This version as at April 2017 includes some significant rewrites.* -*This was started by [Terry Ellison](https://github.com/TerryE) as an unofficial FAQ in mid 2015. It never became officially official and it is in need of an overhaul, see [#937](https://github.com/nodemcu/nodemcu-firmware/issues/937). Yet, it is still very valuable and is, therefore, included here.* - ## What is this FAQ for? -This FAQ does not aim to help you to learn to program or even how to program in Lua. There are plenty of resources on the Internet for this, some of which are listed in [Where to start](#where-to-start). What this FAQ does is to answer some of the common questions that a competent Lua developer would ask in learning how to develop Lua applications for the ESP8266 based boards running the [NodeMcu](http://NodeMCU.com/index_en.html) firmware. +This FAQ does not aim to help you to learn to program or even how to program in Lua. There are plenty of resources on the Internet for this, some of which are listed in [Where to start](#where-to-start). What this FAQ does is to answer some of the common questions that a competent Lua developer would ask in learning how to develop Lua applications for the ESP8266 based boards running the [NodeMcu firmware](https://github.com/nodemcu/nodemcu-firmware). This includes the NodeMCU Devkits. However, the scope of the firmware is far wider than this as it can be used on any ESP8266 module. + +## What has changed since the first version of this FAQ? + +The [NodeMCU company](http://NodeMCU.com/index_en.html) was set up by [Zeroday](zeroday@nodemcu.com) to develop and to market a set of Lua firmware-based development boards which employ the Espressif ESP8266 SoC. The initial development of the firmware was done by Zeroday and a colleague, Vowstar, in-house with the firmware being first open-sourced on Github in late 2014. In mid-2015, Zeroday decided to open the firmware development to a wider group of community developers, so the core group of developers now comprises 6 community developers (including this author), and we are also supported by another dozen or so active contributors, and two NodeMCU originators. + +This larger active team has allowed us to address most of the outstanding issues present at the first version of this FAQ. These include: + +- For some time the project was locked into an old SDK version, but we now regularly rebaseline to the current SDK version. +- Johny Mattsson's software exception handler and my LCD patch have allowed us to move the bulk of constant data out of RAM and into the firmware address space, and as a result current builds now typically boot with over 40Kb free RAM instead of 15Kb free and the code density is roughly 40% better. +- We have fixed error reporting so errors now correctly report line numbers in tracebacks. +- We have addressed most of the various library resource leaks, so memory exhaustion is much less of an issue. +- We have reimplemented the network stack natively over the now Open-sourced Espressif implementation of LwIP. +- Thanks to a documentation effort lead by Marcel Stör, we now have a complete documentation online, and this FAQ forms a small part. +- We have fixed various stability issues relating to the use of GPIO trigger callbacks. +- Johny Mattsson is currently leading an ESP32 port. +- We have a lot more hardware modules supported. + +Because the development is active this list will no doubt continue to be revised and updated. See the [development README](https://github.com/nodemcu/nodemcu-firmware/blob/dev/README.md) for more details. ## Lua Language @@ -17,23 +33,23 @@ The NodeMCU firmware implements Lua 5.1 over the Espressif SDK for its ESP8266 S * The official lua.org **[Lua Language specification](http://www.lua.org/manual/5.1/manual.html)** gives a terse but complete language specification. * Its [FAQ](http://www.lua.org/faq.html) provides information on Lua availability and licensing issues. * The **[unofficial Lua FAQ](http://www.luafaq.org/)** provides a lot of useful Q and A content, and is extremely useful for those learning Lua as a second language. -* The [Lua User's Wiki](http://lua-users.org/wiki/) gives useful example source and relevant discussion. In particular, its [Lua Learning Lua](http://lua-users.org/wiki/Learning) section is a good place to start learning Lua. +* The [Lua User's Wiki](http://lua-users.org/wiki/) gives useful example source and relevant discussion. In particular, its [Lua Learning Lua](http://lua-users.org/wiki/Learning) section is a good place to start learning Lua. * The best book to learn Lua is *Programming in Lua* by Roberto Ierusalimschy, one of the creators of Lua. It's first edition is available free [online](http://www.lua.org/pil/contents.html) . The second edition was aimed at Lua 5.1, but is out of print. The third edition is still in print and available in paperback. It contains a lot more material and clearly identifies Lua 5.1 vs Lua 5.2 differences. **This third edition is widely available for purchase and probably the best value for money**. References of the format [PiL **n.m**] refer to section **n.m** in this edition. * The Espressif ESP8266 architecture is closed source, but the Espressif SDK itself is continually being updated so the best way to get the documentation for this is to [google Espressif IoT SDK Programming Guide](https://www.google.co.uk/search?q=Espressif+IoT+SDK+Programming+Guide) or to look at the Espressif [downloads forum](http://bbs.espressif.com/viewforum.php?f=5) . -* The **[NodeMCU documentation](http://www.NodeMCU.com/docs/)** is available online. However, please remember that the development team are based in China, and English is a second language, so the documentation needs expanding and be could improved with technical proofing. +* The **NodeMCU documentation** is now available online, and this FAQ forms part of this. * As with all Open Source projects the source for the NodeMCU firmware is openly available on the [GitHub NodeMCU-firmware](https://github.com/NodeMCU/NodeMCU-firmware) repository. ### How is NodeMCU Lua different to standard Lua? -Whilst the Lua standard distribution includes a host stand-alone Lua interpreter, Lua itself is primarily an *extension language* that makes no assumptions about a "main" program: Lua works embedded in a host application to provide a powerful, light-weight scripting language for use within the application. This host application can then invoke functions to execute a piece of Lua code, can write and read Lua variables, and can register C functions to be called by Lua code. Through the use of C functions, Lua can be augmented to cope with a wide range of different domains, thus creating customized programming languages sharing a syntactical framework. +Whilst the Lua standard distribution includes a stand-alone Lua interpreter, Lua itself is primarily an *extension language* that makes no assumptions about a "main" program: Lua works embedded in a host application to provide a powerful, lightweight scripting language for use within the application. This host application can then invoke functions to execute a piece of Lua code, can write and read Lua variables, and can register C functions to be called by Lua code. Through the use of C functions, Lua can be augmented to cope with a wide range of different domains, thus creating customized programming languages sharing a syntactical framework. -The ESP8266 was designed and is fabricated in China by [Espressif Systems](http://espressif.com/new-sdk-release/). Espressif have also developed and released a companion software development kit (SDK) to enable developers to build practical IoT applications for the ESP8266. The SDK is made freely available to developers in the form of binary libraries and SDK documentation. However this is in a *closed format*, with no developer access to the source files, so ESP8266 applications *must* rely solely on the SDK API (and the somewhat Spartan SDK API documentation). +The ESP8266 was designed and is fabricated in China by [Espressif Systems](http://espressif.com/new-sdk-release/). Espressif have also developed and released a companion software development kit (SDK) to enable developers to build practical IoT applications for the ESP8266. The SDK is made freely available to developers in the form of binary libraries and SDK documentation. However this is in a *closed format*, with no developer access to the source files, so anyone developing ESP8266 applications must rely solely on the SDK API (and the somewhat Spartan SDK API documentation). (Note that for the ESP32, Espressif have moved to an open-source approach for its ESP-IDF.) -The NodeMCU Lua firmware is an ESP8266 application and must therefore be layered over the ESP8266 SDK. However, the hooks and features of Lua enable it to be seamlessly integrated without loosing any of the standard Lua language features. The firmware has replaced some standard Lua modules that don't align well with the SDK structure with ESP8266-specific versions. For example, the standard `io` and `os` libraries don't work, but have been largely replaced by the NodeMCU `node` and `file` libraries. The `debug` and `math` libraries have also been omitted to reduce the runtime footprint (`modulo` can be done via `%`, `power` via `^`). +The NodeMCU Lua firmware is an ESP8266 application and must therefore be layered over the ESP8266 SDK. However, the hooks and features of Lua enable it to be seamlessly integrated without losing any of the standard Lua language features. The firmware has replaced some standard Lua modules that don't align well with the SDK structure with ESP8266-specific versions. For example, the standard `io` and `os` libraries don't work, but have been largely replaced by the NodeMCU `node` and `file` libraries. The `debug` and `math` libraries have also been omitted to reduce the runtime footprint (`modulo` can be done via `%`, `power` via `^`). NodeMCU Lua is based on [eLua](http://www.eluaproject.net/overview), a fully featured implementation of Lua 5.1 that has been optimized for embedded system development and execution to provide a scripting framework that can be used to deliver useful applications within the limited RAM and Flash memory resources of embedded processors such as the ESP8266. One of the main changes introduced in the eLua fork is to use read-only tables and constants wherever practical for library modules. On a typical build this approach reduces the RAM footprint by some 20-25KB and this makes a Lua implementation for the ESP8266 feasible. This technique is called LTR and this is documented in detail in an eLua technical paper: [Lua Tiny RAM](http://www.eluaproject.net/doc/master/en_arch_ltr.html). -The mains impacts of the ESP8266 SDK and together with its hardware resource limitations are not in the Lua language implementation itself, but in how *application programmers must approach developing and structuring their applications*. As discussed in detail below, the SDK is non-preemptive and event driven. Tasks can be associated with given events by using the SDK API to registering callback functions to the corresponding events. Events are queued internally within the SDK, and it then calls the associated tasks one at a time, with each task returning control to the SDK on completion. *The SDK states that if any tasks run for more than 10 mSec, then services such as Wifi can fail.* +The main impacts of the ESP8266 SDK and together with its hardware resource limitations are not in the Lua language implementation itself, but in how *application programmers must approach developing and structuring their applications*. As discussed in detail below, the SDK is non-preemptive and event driven. Tasks can be associated with given events by using the SDK API to registering callback functions to the corresponding events. Events are queued internally within the SDK, and it then calls the associated tasks one at a time, with each task returning control to the SDK on completion. *The SDK states that if any tasks run for more than 15 mSec, then services such as WiFi can fail.* The NodeMCU libraries act as C wrappers around registered Lua callback functions to enable these to be used as SDK tasks. ***You must therefore use an Event-driven programming style in writing your ESP8266 Lua programs***. Most programmers are used to writing in a procedural style where there is a clear single flow of execution, and the program interfaces to operating system services by a set of synchronous API calls to do network I/O, etc. Whilst the logic of each individual task is procedural, this is not how you code up ESP8266 applications. @@ -46,15 +62,23 @@ The NodeMCU libraries act as C wrappers around registered Lua callback functions ### How is coding for the ESP8266 different to standard Lua? -* The ESP8266 use onchip RAM and offchip Flash memory connected using a dedicated SPI interface. Both of these are *very* limited (when compared to systems than most application programmer use). The SDK and the Lua firmware already use the majority of this resource: the later build versions keep adding useful functionality, and unfortunately at an increased RAM and Flash cost, so depending on the build version and the number of modules installed the runtime can have as little as 17KB RAM and 40KB Flash available at an application level. This Flash memory is formatted an made available as a **SPI Flash File System (SPIFFS)** through the `file` library. -* However, if you choose to use a custom build, for example one which uses integer arithmetic instead of floating point, and which omits libraries that aren't needed for your application, then this can help a lot doubling these available resources. (See Marcel Stör's excellent [custom build tool](http://nodemcu-build.com) that he discusses in [this forum topic](http://www.esp8266.com/viewtopic.php?f=23&t=3001)). Even so, those developers who are used to dealing in MB or GB of RAM and file systems can easily run out of these resources. Some of the techniques discussed below can go a long way to mitigate this issue. -* Current versions of the ESP8266 run the SDK over the native hardware so there is no underlying operating system to capture errors and to provide graceful failure modes, so system or application errors can easily "PANIC" the system causing it to reboot. Error handling has been kept simple to save on the limited code space, and this exacerbates this tendency. Running out of a system resource such as RAM will invariably cause a messy failure and system reboot. -* There is currently no `debug` library support. So you have to use 1980s-style "binary-chop" to locate errors and use print statement diagnostics though the systems UART interface. (This omission was largely because of the Flash memory footprint of this library, but there is no reason in principle why we couldn't make this library available in the near future as an custom build option). -* The LTR implementation means that you can't easily extend standard libraries as you can in normal Lua, so for example an attempt to define `function table.pack()` will cause a runtime error because you can't write to the global `table`. (Yes, there are standard sand-boxing techniques to achieve the same effect by using metatable based inheritance, but if you try to use this type of approach within a real application, then you will find that you run out of RAM before you implement anything useful.) +* The ESP8266 uses a combination of on-chip RAM and off-chip Flash memory connected using a dedicated SPI interface. Code can be executed directly from Flash-mapped address space. In fact the ESP hardware actually executes code in RAM, and in the case of Flash-mapped addresses it executes this code from a RAM-based L1 cache which maps onto the Flash addresses. If the addressed line is in the cache then the code runs at full clock speed, but if not then the hardware transparently handles the adress fault by first copying the code from Flash to RAM. This is largely transparent in terms of programming ESP8266 applications, though the faulting access runs at SRAM speeds and this code runs perhaps 13× slower than already cached code. The Lua firmware largely runs out of Flash, but even so, both the RAM and the Flash memory are *very* limited when compared to systems that most application programmers use. +* Over the last two years, both the Espressif non-OS SDK developers and the NodeMCU team have made a range of improvements and optimisations to increase the amount of RAM available to developers, from a typical 15Kb or so with Version 0.9 builds to some 45Kb with the current firmware Version 2.x builds. See the [ESP8266 Non-OS SDK API Reference](https://espressif.com/sites/default/files/documentation/2c-esp8266_non_os_sdk_api_reference_en.pdf) for more detals on the SDK. +* The early ESP8266 modules were typically configured with 512Kb Flash. Fitting a fully featured Lua build with a number of optional libraries and still enough usable Flash to hold a Lua application was a struggle. However the code-size of the SDK has grown significantly between the early versions and the current 2.0 version. Applications based on the current SDK can no longer fit in 512Kb Flash memory, and so all currently produced ESP modules now contain a minimum of 1Mb with 4 and 16Mb becoming more common. The current NodeMCU firmware will fit comfortably in a 1Mb Flash and still have ample remaining Flash memory to support Lua IoT applications. Note that the [`1.5.4.1-final` branch](https://github.com/nodemcu/nodemcu-firmware/tree/1.5.4.1-final) is the last available release if you still wish to develop applications for 512Kb modules +* The NodeMCU firmware makes any unused Flash memory available as a [SPI Flash File System (SPIFFS)](https://github.com/pellepl/spiffs) through the `file` library. The SPIFFS file system is designed for SPI NOR flash devices on embedded targets, and is optimised for static wear levelling and low RAM footprint. For further details, see the link. How much Flash is available as SPIFFS file space depends on the number of modules included in the specific firmware build. +* The firmware has a wide range of libraries available to support common hardware options. Including any library will increase both the code and RAM size of the build, so our recommended practice is for application developers to choose a custom build that only includes the library that are needed for your application and hardware variants. The developers that don't want to bother with setting up their own build environment can use Marcel Stör's excellent [Cloud build service](http://nodemcu-build.com) instead. +* There are also further tailoring options available, for example you can choose to have a firmware build which uses 32-bit integer arithmetic instead of floating point. Our integer builds have a smaller Flash footprint and execute faster, but working in integer also has a number of pitfalls, so our general recommendation is to use floating point builds. +* Unlike Arduino or ESP8266 development, where each application change requires the flashing of a new copy of the firmware, in the case of Lua the firmware is normally flashed once, and all application development is done by updating files on the SPIFFS file system. In this respect, Lua development on the ESP8266 is far more like developing applications on a more traditional PC. The firmware will only be reflashed if the developer wants to add or update one or more of the hardware-related libraries. +* Those developers who are used to dealing in MB or GB of RAM and file systems can easily run out of memory resources, but with care and using some of the techniques discussed below can go a long way to mitigate this. +* The ESP8266 runs the SDK over the native hardware, so there is no underlying operating system to capture errors and to provide graceful failure modes. Hence system or application errors can easily "PANIC" the system causing it to reboot. Error handling has been kept simple to save on the limited code space, and this exacerbates this tendency. Running out of a system resource such as RAM will invariably cause a messy failure and system reboot. +* Note that in the 3 years since the firmware was first developed, Espressif has developed and released a new RTOS alternative to the non-OS SDK, and and the latest version of the SDK API reference recommends using RTOS. Unfortunately, the richer RTOS has a significantly larger RAM footprint. Whilst our port to the ESP-32 (with its significantly larger RAM) uses the [ESP-IDF](https://github.com/espressif/esp-idf) which is based on RTOS, the ESP8266 RTOS versions don't have enough free RAM for a RTOS-based NodeMCU firmware build to have sufficient free RAM to write usable applications. +* There is currently no `debug` library support. So you have to use 1980s-style "binary-chop" to locate errors and use print statement diagnostics though the system's UART interface. (This omission was largely because of the Flash memory footprint of this library, but there is no reason in principle why we couldn't make this library available in the near future as a custom build option). +* The LTR implementation means that you can't extend standard libraries as easily as you can in normal Lua, so for example an attempt to define `function table.pack()` will cause a runtime error because you can't write to the global `table`. Standard sand-boxing techniques can be used to achieve the same effect by using metatable based inheritance, but if you choose this option, then you need to be aware of the potential runtime and RAM impacts of this approach. * There are standard libraries to provide access to the various hardware options supported by the hardware: WiFi, GPIO, One-wire, I²C, SPI, ADC, PWM, UART, etc. -* The runtime system runs in interactive-mode. In this mode it first executes any `init.lua` script. It then "listens" to the serial port for input Lua chunks, and executes them once syntactically complete. There is no `luac` or batch support, although automated embedded processing is normally achieved by setting up the necessary event triggers in the `init.lua` script. -* The various libraries (`net`, `tmr`, `wifi`, etc.) use the SDK callback mechanism to bind Lua processing to individual events (for example a timer alarm firing). Developers should make full use of these events to keep Lua execution sequences short. *If any individual task takes too long to execute then other queued tasks can time-out and bad things start to happen.* -* Non-Lua processing (e.g. network functions) will usually only take place once the current Lua chunk has completed execution. So any network calls should be viewed at an asynchronous request. A common coding mistake is to assume that they are synchronous, that is if two `socket:send()` are on consecutive lines in a Lua programme, then the first has completed by the time the second is executed. This is wrong. Each `socket:send()` request simply queues the send operation for dispatch. Neither will start to process until the Lua code has return to is calling C function. Stacking up such requests in a single Lua task function burns scarce RAM and can trigger a PANIC. This true for timer, network, and other callbacks. It is even the case for actions such as requesting a system restart, as can be seen by the following example: +* The runtime system runs in interactive-mode. In this mode it first executes any `init.lua` script. It then "listens" to the serial port for input Lua chunks, and executes them once syntactically complete. +* There is no batch support, although automated embedded processing is normally achieved by setting up the necessary event triggers in the [`init.lua`](../upload/#initlua) script. +* The various libraries (`net`, `tmr`, `wifi`, etc.) use the SDK callback mechanism to bind Lua processing to individual events (for example a timer alarm firing). Developers should make full use of these events to keep Lua execution sequences short. +* Non-Lua processing (e.g. network functions) will usually only take place once the current Lua chunk has completed execution. So any network calls should be viewed at an asynchronous request. A common coding mistake is to assume that they are synchronous, that is if two `socket:send()` are on consecutive lines in a Lua programme, then the first has completed by the time the second is executed. This is wrong. A `socket:send()` request simply queues the send task for dispatch by the SDK. This task can't start to process until the Lua code has returned to is calling C function to allow this running task to exit. Stacking up such requests in a single Lua task function burns scarce RAM and can trigger a PANIC. This is true for timer, network, and other callbacks. It is even the case for actions such as requesting a system restart, as can be seen by the following example which will print twenty "not quite yet" messages before restarting. ```lua node.restart(); for i = 1, 20 do print("not quite yet -- ",i); end @@ -64,65 +88,31 @@ node.restart(); for i = 1, 20 do print("not quite yet -- ",i); end ### So how does the SDK event / tasking system work in Lua? -* The SDK employs an event-driven and task-oriented architecture for programming at an applications level. -* The SDK uses a startup hook `void user_init(void)`, defined by convention in the C module `user_main.c`, which it invokes on boot. The `user_init()` function can be used to do any initialisation required and to call the necessary timer alarms or other SDK API calls to bind and callback routines to implement the tasks needed in response to any system events. -* The API provides a set of functions for declaring application functions (written in C) as callbacks to associate application tasks with specific hardware and timer events. These are non-preemptive at an applications level. -* Whilst the SDK provides a number of interrupt driven device drivers, the hardware architecture severely limits the memory available for these drivers, so writing new device drivers is not a viable options for most developers -* The SDK interfaces internally with hardware and device drivers to queue pending events. -* The registered callback routines are invoked sequentially with the associated C task running to completion uninterrupted. -* In the case of Lua, these C tasks are typically functions within the Lua runtime library code and these typically act as C wrappers around the corresponding developer-provided Lua callback functions. An example here is the Lua [`mytimer:alarm(interval, repeat, callback)`](modules/tmr.md#tmralarm) function. The calls a function in the `tmr` library which registers a C function for this alarm using the SDK, and when this C function is called it then invokes the Lua callback. +* The SDK uses a small number of Interrupt Service Routines (ISRs) to handle short time critical hardware interrupt related processing. These are very short duration and can interrupt a running task for up to 10µSec. (Modifying these ISRs or adding new ones is not a viable options for most developers.) +* All other service and application processing is split into code execution blocks, known as **tasks**. The individual tasks are executed one at a time and run to completion. No task can never pre-empt another. +* Runnable tasks are queued in one of three priority queues and the SDK contains a simple scheduler which executes queued tasks FIFO within priority. The high priority queue is used for hardware-related task, the middle for timer and event-driven tasks and the low priority queue for all other tasks. +* It is important to keep task times as short as practical so that the overall system can work smoothly and responsively. The general recommendation is to keep medium priority tasks under 2mSec and low priority tasks under 15 mSec in duration. This is a guideline, and your application *might* work stably if you exceed this, but you might also start to experience intermittent problems because of internal timeout within the WiFi and network services, etc.. +* If tasks take longer than 500mSec then the watchdog timer will reset the processor. This watchdog can be reset at an application level using the [`tmr.wdclr()`](modules/tmr/#tmrwdclr) function, but this should be avoided. +* Application tasks can disable interrupts to prevent an ISR interrupting a time-critical code section, The SDK guideline is that system ISRs might overrun if such critical code section last more than 10µSec. This means that such disabling can only be done within hardware-related library modules, written in C; it is not available at a Lua application level. +* The SDK provide a C API for interfacing to it; this includes a set of functions for declaring application functions (written in C) as callbacks to associate application tasks with specific hardware and timer events, and their execution will be interleaved with the SDKs Wifi and Network processing tasks. -The NodeMCU firmware simply mirrors this structure at a Lua scripting level: +In essence, the NodeMCU firmware is a C application which exploits the ability of Lua to execute as a embedded language and runtime to mirror this structure at a Lua scripting level. All of the complexities of, and interface to, the SDK and the hardware are wrapped in firmware libraries which translate the appropriate calls into the corresponding Lua API. -* A startup module `init.lua` is invoked on boot. This function module can be used to do any initialisation required and to call the necessary timer alarms or library calls to bind and callback routines to implement the tasks needed in response to any system events. -* The Lua libraries provide a set of functions for declaring application functions (written in Lua) as callbacks (which are stored in the [Lua registry](#so-how-is-the-lua-registry-used-and-why-is-this-important)) to associate application tasks with specific hardware and timer events. These are non-preemptive at an applications level* The Lua libraries work in consort with the SDK to queue pending events and invoke any registered Lua callback routines, which then run to completion uninterrupted. -* Excessively long-running Lua functions can therefore cause other system functions and services to timeout, or allocate memory to buffer queued data, which can then trigger either the watchdog timer or memory exhaustion, both of which will ultimately cause the system to reboot. -* By default, the Lua runtime also 'listens' to UART 0, the serial port, in interactive mode and will execute any Lua commands input through this serial port. +* The SDK invokes a startup hook within the firmware on boot-up. This firmware code initialises the Lua environment and then attempts to execute the Lua module `init.lua` from the SPIFFS file system. This `init.lua` module can then be used to do any application initialisation required and to call the necessary timer alarms or library calls to bind and callback routines to implement the tasks needed in response to any system events. +* By default, the Lua runtime also 'listens' to UART 0, the serial port, in interactive mode and will execute any Lua commands input through this serial port. Using the serial port in this way is the most common method of developing and debugging Lua applications on the ESP8266/ +* The Lua libraries provide a set of functions for declaring application functions (written in Lua) as callbacks (which are stored in the [Lua registry](#so-how-is-the-lua-registry-used-and-why-is-this-important)) to associate application tasks with specific hardware and timer events. These are also non-preemptive at an applications level. The Lua libraries work in consort with the SDK to queue pending events and invoke any registered Lua callback routines, which then run to completion uninterrupted. For example the Lua [`mytimer:alarm(interval, repeat, callback)`](modules/tmr/#tmralarm) calls a function in the `tmr` library which registers a C function for this alarm using the SDK, and when this C alarm callback function is called it then in turn invokes the Lua callback. +* Excessively long-running Lua functions (or Lua code chunks executed at the interactive prompt through UART 0) can cause other system functions and services to timeout, or to allocate scarce RAM resources to buffer queued data, which can then trigger either the watchdog timer or memory exhaustion, both of which will ultimately cause the system to reboot. +* Just like their C counterparts, Lua tasks initiated by timer, network, GPIO and other callbacks run non pre-emptively to completion before the next task can run, and this includes SDK tasks. Printing to the default serial port is done by the Lua runtime libraries, but SDK services including even a reboot request are run as individual tasks. This is why in the previous example printout out twenty copies of "not quite yet --" before completing and return control the SDK which then allows the reboot to occur. -This event-driven approach is very different to a conventional procedural implementation of Lua. +This event-driven approach is very different to a conventional procedural applications written in Lua, and different from how you develop C sketches and applications for the Arduino architectures. _There is little point in constructing poll loops in your NodeMCU Lua code since almost always the event that you are polling will not be delivered by the SDK until after your Lua code returns control to the SDK._ The most robust and efficient approach to coding ESP8266 Lua applications is to embrace this event model paradigm, and to decompose your application into atomic tasks that are threaded by events which themselves initiate callback functions. Each event task is established by a callback in an API call in an earlier task. -Consider the following snippet: +Understanding how the system executes your code can help you structure it better and improve both performance and memory usage. -```lua -s=net.createServer(net.TCP) -s:listen(23,function(c) - con_std = c - function s_output(str) - if(con_std~=nil) then - con_std:send(str) - end - end - node.output(s_output, 0) - c:on("receive",function(c,l) node.input(l) end) - c:on("disconnection",function(c) - con_std = nil - node.output(nil) - end) -end) -``` +* _If you are not using timers and other callback, then you are using the wrong approach._ -This example defines five Lua functions: +* _If you are using poll loops, then you are using the wrong approach._ -| Function | Defined in | Parameters | Callback? | -|-----------|------------|------------|-----------| -| Main | Outer module | ... (Not used) | | -| Connection listener | Main | c (connection socket) | | -| s_output | Connection listener | str | Yes | -| On Receive| Connection listener | c, l (socket, input) | Yes | -| On Disconnect | Connection listener | c (socket) | Yes | - -`s`, `con_std` and `s_output` are global, and no [upvalues](#why-is-it-importance-to-understand-how-upvalues-are-implemented-when-programming-for-the-esp8266) are used. There is no "correct" order to define these in, but we could reorder this code for clarity (though doing this adds a few extra globals) and define these functions separately one another. However, let us consider how this is executed: - -* The outer module is compiled including the four internal functions. -* `Main` is then assigning the created `net.createServer()` to the global `s`. The `connection listener` closure is created and bound to a temporary variable which is then passed to the `socket.listen()` as an argument. The routine then exits returning control to the firmware. -* When another computer connects to port 23, the listener handler retrieves the reference to then connection listener and calls it with the socket parameter. This function then binds the s_output closure to the global `s_output`, and registers this function with the `node.output` hook. Likewise the `on receive` and `on disconnection` are bound to temporary variables which are passed to the respective on handlers. We now have four Lua function registered in the Lua runtime libraries associated with four events. This routine then exits returning control to the firmware. -* When a record is received, the on receive handler within the net library retrieves the reference to the `on receive` Lua function and calls it passing it the record. This routine then passes this to the `node.input()` and exits returning control to the firmware. -* The `node.input` handler polls on an 80 mSec timer alarm. If a compete Lua chunk is available (either via the serial port or node input function), then it executes it and any output is then passed to the `note.output` handler. which calls `s_output` function. Any pending sends are then processed. -* This cycle repeats until the other computer disconnects, and `net` library disconnection handler then calls the Lua `on disconnect` handler. This Lua routine dereferences the connected socket and closes the `node.output` hook and exits returning control to the disconnect handler which garbage collects any associated sockets and registered on handlers. - -Whilst this is all going on, The SDK can (and often will) schedule other event tasks in between these Lua executions (e.g. to do the actual TCP stack processing). The longest individual Lua execution in this example is only 18 bytecode instructions (in the main routine). - -Understanding how the system executes your code can help you structure it better and improve memory usage. Each event task is established by a callback in an API call in an earlier task. +* _If you are executing more an a few hundred lines of Lua per callback, then you are using the wrong approach._ ### So what Lua library functions enable the registration of Lua callbacks? @@ -130,42 +120,111 @@ SDK Callbacks include: | Lua Module | Functions which define or remove callbacks | |------------|--------------------------------------------| -| tmr | `alarm(id, interval, repeat, function())` | -| node | `key(type, function())`, `output(function(str), serial_debug)` | -| wifi | `startsmart(chan, function())`, `sta.getap(function(table))` | -| net.server | `sk:listen(port,[ip],function(socket))` | +| tmr | `register([id,] interval, mode, function())` | +| node | `task.post([task_priority], function)`, `output(function(str), serial_debug)` | +| wifi | `startsmart(chan, function())`, `sta.getap(function(table))` | +| net.server | `sk:listen(port,[ip],function(socket))` | | net | `sk:on(event, function(socket, [, data]))`, `sk:send(string, function(sent))`, `sk:dns(domain, function(socket,ip))` | -| gpio | `trig(pin, type, function(level))` | -| mqtt | `client:m:on(event, function(conn[, topic, data])` | -| uart | `uart.on(event, cnt, [function(data)], [run_input])` | +| gpio | `trig(pin, type, function(level))` | +| mqtt | `client:m:on(event, function(conn[, topic, data])` | +| uart | `uart.on(event, cnt, [function(data)], [run_input])` | +For a comprehensive list refer to the Module documentation on this site. + +### So what are the different ways of declaring variables and how is NodeMCU different here? + +The following is all standard Lua and is explained in detail in PiL etc., but it is worth summarising here because understanding this is of particular importance in the NodeMCU environment. + +* All variables in Lua can be classed as globals, locals or upvalues. But by default any variable that is referenced and not previously declared as `local` is **global** and this variable will persist in the global table until it is explicitly deleted. If you want to see what global variables are in scope then try +```Lua + for k,v in pairs(_G) do print(k,v) end +``` +* Local variables are 'lexically scoped', and you may declare any variables as local within nested blocks or functions without affecting the enclosing scope. +* Because locals are lexically scoped you can also refer to local variables in an outer scope and these are still accessible within the inner scope. Such variables are know as **upvalues**.. +* Lua variable can be assigned two broad types of data: **values** such as numbers, booleans, and strings and **references** such as functions, tables and userdata. You can see the difference here when you assign the contents of a variable `a` to `b`. In the case of a value then it is simply copied into `b`. In the case of a reference, both `a` and `b` now refer to the *same object*, and no copying of content takes place. This process of referencing can have some counter-intuitive consequences. For example, in the following code by the time it exists, the variable `timer2func` is out of scope. However a reference to the function has now been stored in the Lua registry by the alarm API call, so it and any upvalues that it uses will persist until it is eventually entirely dereferenced (e.g. by `tmr2:unregister()`. +```Lua + do + local tmr2func = function() ds.convert_T(true); tmr1:start() end + tmr2:alarm(300000, tmr.ALARM_AUTO, tmr2func) + end + -- +``` +* You need to understand the difference between when a function is compiled, when it is bound as a closure and when it is invoked at runtime. The closure is normally bound once pretty much immediately after compile, but this isn't necessarily the case. Consider the following example from my MCP23008 module below. +```Lua + -- Bind the read and write functions for commonly accessed registers + + for reg, regAddr in pairs { + IODOR = 0x00, + GPPU = 0x06, -- Pull-up resistors register for MCP23008 + GPIO = 0x09, + OLAT = 0x0A, + } do + dev['write'..reg] = function(o, dataByte) + write(MCP23008addr, regAddr, dataByte) + end + dev['read'..reg] = function(o) + return read(MCP23008addr, regAddr) + end + end +``` +* This loop is compiled once when the module is required. The opcode vectors for the read and write functions are created during the compile, along with a header which defines how many upvalues and locals are used by each function. However, these two functions are then bound _four_ times as different functions (e.g. `mcp23008.writeIODOR()`) and each closure inherits its own copies of the upvalues it uses so the `regAddr` for this function is `0x00`). The upvalue list is created when the closure is created and through some Lua magic, even if the outer routine that initially declared them is no longer in scope and has been GCed (Garbage Collected), the Lua RTS ensures that any upvalue will still persist whilst the closure persists. +* On the other hand the storage for any locals is allocated each time the routine is called, and this can be many times in a running application. +* The Lua runtime uses hashed key access internally to retrieve keyed data from a table. On the other hand locals and upvalues are stored as a contiguous vector and are accessed directly by an index, which is a lot faster. In NodeMCU Lua accesses to Firmware-based tables is particularly slow, which is why you will often see statements like the following at the beginning of modules. *Using locals and upvalues this way is both a lot faster at runtime and generates less bytecode instructions for their access.* +```Lua +local i2c = i2c +local i2c_start, i2c_stop, i2c_address, i2c_read, i2c_write, i2c_TRANSMITTER, i2c_RECEIVER = + i2c.start, i2c.stop, i2c.address, i2c.read, i2c.write, i2c.TRANSMITTER, i2c.RECEIVER +``` +* I will cover some useful Global and Upvalue techniques in later Qs. ### So how is context passed between Lua event tasks? -* It is important to understand that any event callback task is associated with a single Lua function. This function is executed from the relevant NodeMCU library C code using a `lua_call()`. Even system initialisation which executes the `dofile("init.lua")` can be treated as a special case of this. Each function can invoke other functions and so on, but it must ultimate return control to the C library code. -* By their very nature Lua `local` variables only exist within the context of an executing Lua function, and so all locals are destroyed between these `lua_call()` actions. *No locals are retained across events*. -* So context can only be passed between event routines by one of three mechanisms: -* **Globals** are by nature globally accessible. Any global will persist until explicitly dereference by reassigning `nil` to it. Globals can be readily enumerated by a `for k,v in pairs(_G) do` so their use is transparent. - * The **File system** is a special case of persistent global, so there is no reason in principle why it can't be used to pass context. However the ESP8266 file system uses flash memory and this has a limited write cycle lifetime, so it is best to avoid using the file system to store frequently changing content except as a mechanism of last resort. -* **Upvalues**. When a function is declared within an outer function, all of the local variables in the outer scope are available to the inner function. Since all functions are stored by reference the scope of the inner function might outlast the scope of the outer function, and the Lua runtime system ensures that any such references persist for the life of any functions that reference it. This standard feature of Lua is known as *closure* and is described in [Pil 6]. Such values are often called *upvalues*. Functions which are global or [[#So how is the Lua Registry used and why is this important?|registered]] callbacks will persist between event routines, and hence any upvalues referenced by them can be used for passing context. +* It is important to understand that a single Lua function is associated with / bound to any event callback task. This function is executed from within the relevant NodeMCU library C code using a `lua_call()`. Even system initialisation which executes the `dofile("init.lua")` is really a special case of this. Each function can invoke other functions and so on, but it must ultimately return control to the C library code which then returns control the SDK, terminating the task. +* By their very nature Lua `local` variables only exist within the context of an executing Lua function, and so locals are unreferenced on exit and any local data (unless also a reference type such as a function, table, or user data which is also referenced elsewhere) can therefore be garbage collected between these `lua_call()` actions. + +So context can only be passed between event routines by one of the following mechanisms: + +* **Globals** are by nature globally accessible. Any global will persist until explicitly dereferenced by assigning `nil` to it. Globals can be readily enumerated, e.g. by a `for k,v in pairs(_G) do`, so their use is transparent. +* The **File system** is a special case of persistent global, so there is no reason in principle why it and the files it contains can't be used to pass context. However the ESP8266 file system uses flash memory and even with the SPIFFS file system still has a limited write cycle lifetime, so it is best to avoid using the file system to store frequently changing content except as a mechanism of last resort. +* The **Lua Registry**. This is a normally hidden table used by the library modules to store callback functions and other Lua data types. The GC treats the registry as in scope and hence any content referenced in the registry will not be garbage collected. +* **Upvalues**. These are a standard feature of Lua as described above that is fully implemented in NodeMCU. When a function is declared within an outer function, all of the local variables within the outer scope are available to the inner function. Ierusalimschy's paper, [Closures in Lua](http://www.cs.tufts.edu/~nr/cs257/archive/roberto-ierusalimschy/closures-draft.pdf), gives a lot more detail for those that want to dig deeper. ### So how is the Lua Registry used and why is this important? -So all Lua callbacks are called by C wrapper functions that are themselves callback activated by the SDK as a result of a given event. Such C wrapper functions themselves frequently need to store state for passing between calls or to other wrapper C functions. The Lua registry is simply another Lua table which is used for this purpose, except that it is hidden from direct Lua access. Any content that needs to be saved is created with a unique key. Using a standard Lua table enables standard garbage collection algorithms to operate on its content. +All Lua callbacks are called by C wrapper functions within the NodeMCU libraries that are themselves callbacks that have been activated by the SDK as a result of a given event. Such C wrapper functions themselves frequently need to store state for passing between calls or to other wrapper C functions. The Lua registry is a special Lua table which is used for this purpose, except that it is hidden from direct Lua access, but using a standard Lua table for this store enables standard garbage collection algorithms to operate on its content. Any content that needs to be saved is created with a unique key. The upvalues for functions that are global or referenced in the Lua Registry will persist between event routines, and hence any upvalues used by them will also persist and can be used for passing context. -Note that we have identified a number of cases where library code does not correctly clean up Registry content when closing out an action, leading to memory leaks. +* If you are running out of memory, then you might not be correctly clearing down Registry entries. One example is as above where you are setting up timers but not unregistering them. Another occurs in the following code fragment. The `on()` function passes the socket to the connection callback as it's first argument `sck`. This is local variable in the callback function, and it also references the same socket as the upvalue `srv`. So functionally `srv` and `sck` are interchangeable. So why pass it as an argument? Normally garbage collecting a socket will automatically unregister any of its callbacks, but if you use a socket as an upvalue in the callback, the socket is now referenced through the Register, and now it won't be GCed because it is referenced. Catch-22 and a programming error, not a bug. + +```Lua +srv:on("connection", function(sck, c) + svr:send(reply) +end) +``` + +* One way to check the registry is to use the construct `for k,v in pairs(debug.getregistry()) do print (k,v) end` to track the registry size. If this is growing then you've got a leak. + +### How do I track globals + +* See the Unofficial LUA FAQ: [Detecting Undefined Variables](http://lua-users.org/wiki/DetectingUndefinedVariables). + +* My approach is to avoid using them unless I have a *very* good reason to justify this. I track them statically by running a `luac -p -l XXX.lua | grep GLOBAL` filter on any new modules and replace any accidental globals by local or upvalued local declarations. + +* On NodeMCU, _G's metatable is _G, so you can create any globals that you need and then 'close the barn door' by assigning +`_G.__newindex=function(g,k,v) error ("attempting to set global "..k.." to "..v) end` and any attempt to create new globals with now throw an error and give you a traceback of where this has happened. ### Why is it importance to understand how upvalues are implemented when programming for the ESP8266? -Routines directly or indirectly referenced in the globals table, **_G**, or in the Lua Registry may use upvalues. The number of upvalues associated with a given routine is determined by the compiler and a vector is allocated when the closure is bound to hold these references. Each upvalues is classed as open or closed. All upvalues are initially open which means that the upvalue references back to the outer functions's register set. However, upvalues must be able to outlive the scope of the outer routine where they are declared as a local variable. The runtime VM does this by adding extra checks when executing a function return to scan any defined closures within its scope for back references and allocate memory to hold the upvalue and points the upvalue's reference to this. This is known as a closed upvalue. +The use of upvalues is a core Lua feature. This is explained in detail in PiL. Any Lua routines defined within an outer scope my use them. This can include routines directly or indirectly referenced in the globals table, **_G**, or in the Lua Registry. -This processing is a mature part of the Lua 5.x runtime system, and for normal Lua applications development this "behind-the-scenes" magic ensures that upvalues just work as any programmer might expect. Sufficient garbage collector metadata is also stored so that these hidden values will be garbage collected correctly *when properly dereferenced*. However allocating these internal structures is quite expensive in terms of memory, and this hidden overhead is hard to track or to understand. If you are developing a Lua application for a PC where the working RAM for an application is measured in MB, then this isn't really an issue. However, if you are developing an application for the ESP8266 where you might have 20 KB for your program and data, this could prove a killer. +The number of upvalues associated with a given routine is calculated during compile and a stack vector is allocated for them when the closure is bound to hold these references. Each upvalues is classed as open or closed. All upvalues are initially open which means that the upvalue references back to the outer function's register set. However, upvalues must be able to outlive the scope of the outer routine where they are declared as a local variable. The runtime VM does this by adding extra checks when executing a function return to scan any defined closures within its scope for back references and allocate memory to hold the upvalue and points the upvalue's reference to this. This is known as a closed upvalue. -One further complication is that some library functions don't correctly dereference expired callback references and as a result their upvalues may not be correctly garbage collected (though we are tracking this down and hopefully removing this issue). This will all be manifested as a memory leak. So using upvalues can cause more frequent and difficult to diagnose PANICs during testing. So my general recommendation is still to stick to globals for this specific usecase of passing context between event callbacks, and `nil` them when you have done with them. +This processing is a mature part of the Lua 5.x runtime system, and for normal Lua applications development this "behind-the-scenes" magic ensures that upvalues just work as any programmer might expect. Sufficient garbage collector metadata is also stored so that these hidden values will be garbage collected correctly *when properly dereferenced*. + +One further complication is that some library functions don't implicitly dereference expired callback references and as a result their upvalues may not be garbage collected and this application error can be be manifested as a memory leak. So using upvalues can cause more frequent and difficult to diagnose PANICs during testing. So my general recommendation is still to stick to globals during initial development, and explicitly dereference resources by setting them to `nil` when you have done with them. ### Can I encapsulate actions such as sending an email in a Lua function? -Think about the implications of these last few answers. +Think about the implications of these last few answers. * An action such as composing and sending an email involves a message dialogue with a mail server over TCP. This in turn requires calling multiple API calls to the SDK and your Lua code must return control to the C calling library for this to be scheduled, otherwise these requests will just queue up, you'll run out of RAM and your application will PANIC. * Hence it is simply ***impossible*** to write a Lua module so that you can do something like: @@ -179,7 +238,9 @@ status = mail.send(to, subject, body) ```lua -- prepare message local ms = require("mail_sender") - return ms.send(to, subject, body, function(status) loadfile("process_next.lua")(status) end) + return ms.send(to, subject, body, function(status) + loadfile("process_next.lua")(status) + end) ``` * Building an application on the ESP8266 is a bit like threading pearls onto a necklace. Each pearl is an event task which must be small enough to run within its RAM resources and the string is the variable context that links the pearls together. @@ -187,12 +248,9 @@ status = mail.send(to, subject, body) If you are used coding in a procedural paradigm then it is understandable that you consider using [`tmr.delay()`](modules/tmr.md#tmrdelay) to time sequence your application. However as discussed in the previous section, with NodeMCU Lua you are coding in an event-driven paradigm. -If you look at the `app/modules/tmr.c` code for this function, then you will see that it executes a low level `ets_delay_us(delay)`. This function isn't part of the NodeMCU code or the SDK; it's actually part of the xtensa-lx106 boot ROM, and is a simple timing loop which polls against the internal CPU clock. It does this with interrupts disabled, because if they are enabled then there is no guarantee that the delay will be as requested. - -`tmr.delay()` is really intended to be used where you need to have more precise timing control on an external hardware I/O (e.g. lifting a GPIO pin high for 20 μSec). It will achieve no functional purpose in pretty much every other usecase, as any other system code-based activity will be blocked from execution; at worst it will break your application and create hard-to-diagnose timeout errors. - -The latest SDK includes a caution that if any (callback) task runs for more than 10 mSec, then the Wifi and TCP stacks might fail, so if you want a delay of more than 8 mSec or so, then *using `tmr.delay()` is the wrong approach*. You should be using a timer alarm or another library callback, to allow the other processing to take place. As the NodeMCU documentation correctly advises (translating Chinese English into English): *`tmr.delay()` will make the CPU work in non-interrupt mode, so other instructions and interrupts will be blocked. Take care in using this function.* +If you look at the `app/modules/tmr.c` code for this function, then you will see that it executes a low level `ets_delay_us(delay)`. This function isn't part of the NodeMCU code or the SDK; it's actually part of the xtensa-lx106 boot ROM, and is a simple timing loop which polls against the internal CPU clock. `tmr.delay()` is really intended to be used where you need to have more precise timing control on an external hardware I/O (e.g. lifting a GPIO pin high for 20 μSec). It does this with interrupts enabled, because so there is no guarantee that the delay will be as requested, and the Lua RTS itself may inject operations such as GC, so if you do this level of precise control then you should encode your application as a C library. +It will achieve no functional purpose in pretty much every other usecase, as any other system code-based activity will be blocked from execution; at worst it will break your application and create hard-to-diagnose timeout errors. We therefore deprecate its general use. ### How do I avoid a PANIC loop in init.lua? @@ -200,10 +258,17 @@ Most of us have fallen into the trap of creating an `init.lua` that has a bug in - When this happens, the only robust solution is to reflash the firmware. - The simplest way to avoid having to do this is to keep the `init.lua` as simple as possible -- say configure the wifi and then start your app using a one-time `tmr.alarm()` after a 2-3 sec delay. This delay is long enough to issue a `file.remove("init.lua")` through the serial port and recover control that way. -- Also it is always best to test any new `init.lua` by creating it as `init_test.lua`, say, and manually issuing a `dofile("init_test.lua")` through the serial port, and then only rename it when you are certain it is working as you require. +- Another trick is to poll a spare GPIO input pin in your startup. I do this on my boards by taking this GPIO plus Vcc to a jumper on the board, so that I can set the jumper to jump into debug mode or reprovision the software. +- Also it is always best to test any new `init.lua` by creating it as `init_test.lua`, say, and manually issuing a `dofile("init_test.lua")` through the serial port, and then only rename it when you are certain it is working as you require. See ["Uploading code" → init.lua](upload.md#initlua) for an example. +## Compiling and Debugging + +* We recommend that you install Lua 5.1 on your delopment host. This often is useful for debugging Lua fragments on your PC. You also use it for compile validation. + +* You can also build `luac.cross` on your development host if you have Lua locally installed. This runs on your host and has all of the features of standard `luac`, except that the output code file will run under NodeMCU as an *lc* file. + ## Techniques for Reducing RAM and SPIFFS footprint ### How do I minimise the footprint of an application? @@ -221,14 +286,16 @@ See ["Uploading code" → init.lua](upload.md#initlua) for an example. * Keep a master repository of your code on your PC or a cloud-based versioning repository such as [GitHub](https://github.com/) * Lay it out and comment it for ease of maintenance and debugging * Use a package such as [Esplorer](https://github.com/4refr0nt/ESPlorer) to download modules that you are debugging and to test them. - * Once the code is tested and stable, then compress it using LuaSrcDiet before downloading to the ESP8266. Doing this will reduce the code footprint on the SPIFFS by 2-3x. -* Consider using `node.compile()` to pre-compile any production code. This removes the debug information from the compiled code reducing its size by roughly 40%. (However this is still perhaps 1.5-2x larger than a LuaSrcDiet-compressed source format, so if SPIFFS is tight then you might consider leaving less frequently run modules in Lua format. If you do a compilation, then you should consider removing the Lua source copy from file system as there's little point in keeping both on the ESP8266. + * Once the code is tested and stable, then compress it using LuaSrcDiet before downloading to the ESP8266. Doing this will reduce the code footprint on the SPIFFS by 2-3x. Also note that LuaSrcDiet has a mode which achieves perhaps 95% of the possible code compaction but which still preserves line numbering. This means that any line number-based error messages will still be usable. +* Standard Lua compiled code includes a lot of debug information which almost doubles its RAM size. [node.stripdebug()](modules/node.md#nodestripdebug) can be used to change this default setting either to increase the debug information for a given module or to remove line number information to save a little more space. Using `node.compile()` to pre-compile any production code will remove all compiled code including error line info and so is not recommended except for stable production code where line numbers are not needed. + ### How do I minimise the footprint of running application? * The Lua Garbage collector is very aggressive at scanning and recovering dead resources. It uses an incremental mark-and-sweep strategy which means that any data which is not ultimately referenced back to the Globals table, the Lua registry or in-scope local variables in the current Lua code will be collected. * Setting any variable to `nil` dereferences the previous context of that variable. (Note that reference-based variables such as tables, strings and functions can have multiple variables referencing the same object, but once the last reference has been set to `nil`, the collector will recover the storage. * Unlike other compile-on-load languages such as PHP, Lua compiled code is treated the same way as any other variable type when it comes to garbage collection and can be collected when fully dereferenced, so that the code-space can be reused. +* The default garbage collection mode is very aggressive and results in a GC sweep after every allocation. See [node.egc.setmode()](modules/node/#nodeegcsetmode) for how to turn this down. `node.egc.setmode(node.egc.ON_MEM_LIMIT, 4096)` is a good compromise of performance and having enough free headboard. * Lua execution is intrinsically divided into separate event tasks with each bound to a Lua callback. This, when coupled with the strong dispose on dereference feature, means that it is very easy to structure your application using an classic technique which dates back to the 1950s known as Overlays. * Various approaches can be use to implement this. One is described by DP Whittaker in his [Massive memory optimization: flash functions](http://www.esp8266.com/viewtopic.php?f=19&t=1940) topic. Another is to use *volatile modules*. There are standard Lua templates for creating modules, but the `require()` library function creates a reference for the loaded module in the `package.loaded` table, and this reference prevents the module from being garbage collected. To make a module volatile, you should remove this reference to the loaded module by setting its corresponding entry in `package.loaded` to `nil`. You can't do this in the outermost level of the module (since the reference is only created once execution has returned from the module code), but you can do it in any module function, and typically an initialisation function for the module, as in the following example: @@ -278,9 +345,9 @@ s:listen(80,connector) Note that there are two methods of saving compiled Lua to SPIFFS: - The first is to use `node.compile()` on the `.lua` source file, which generates the equivalent bytecode `.lc` file. This approach strips out all the debug line and variable information. - - The second is to use `loadfile()` to load the source file into memory, followed by `string.dump()` to convert it in-memory to a serialised load format which can then be written back to a `.lc` file. This approach creates a bytecode file which retains the debug information. + - The second is to use `loadfile()` to load the source file into memory, followed by `string.dump()` to convert it in-memory to a serialised load format which can then be written back to a `.lc` file. The amount of debug saved will depend on the [node.stripdebug()](modules/node.md#nodestripdebug) settings. -The memory footprint of the bytecode created by method (2) is the same as when executing source files directly, but the footprint of bytecode created by method (1) is typically **60% of this size**, because the debug information is almost as large as the code itself. So using `.lc` files generated by `node.compile()` considerably reduces code size in memory -- albeit with the downside that any runtime errors are extremely limited. +The memory footprint of the bytecode created by method (2) is the same as when executing source files directly, but the footprint of bytecode created by method (1) is typically 10% smaller than a dump with the stripdebug level of 2 or 60% smaller than a dump with a stripdebug level of 0, because the debug information is almost as large as the code itself. In general consider method (1) if you have stable production code that you want to run in as low a RAM footprint as possible. Yes, method (2) can be used if you are still debugging, but you will probably be changing this code quite frequently, so it is easier to stick with `.lua` files for code that you are still developing. @@ -290,8 +357,8 @@ Note that if you use `require("XXX")` to load your code then this will automatic * You should get an overall understanding of the VM model if you want to make good use of the limited resources available to Lua applications. An essential reference here is [A No Frills Introduction to Lua 5.1 VM Instructions](http://luaforge.net/docman/83/98/ANoFrillsIntroToLua51VMInstructions.pdf) . This explain how the code generator works, how much memory overhead is involved with each table, function, string etc.. * You can't easily get a bytecode listing of your ESP8266 code; however there are two broad options for doing this: - * **Generate a bytecode listing on your development PC**. The Lua 5.1 code generator is basically the same on the PC and on the ESP8266, so whilst it isn't identical, using the standard Lua batch compiler `luac` against your source on your PC with the `-l -s` option will give you a good idea of what your code will generate. The main difference between these two variants is the size_t for ESP8266 is 4 bytes rather than the 8 bytes size_t found on modern 64bit development PCs; and the eLua variants generate different access references for ROM data types. If you want to see what the `string.dump()` version generates then drop the `-s` option to retain the debug information. - * **Upload your `.lc` files to the PC and disassemble then there**. There are a number of Lua code disassemblers which can list off the compiled code that you application modules will generate, `if` you have a script to upload files from your ESP8266 to your development PC. I use [ChunkSpy](http://luaforge.net/projects/chunkspy/) which can be downloaded [here](http://files.luaforge.net/releases/chunkspy/chunkspy/ChunkSpy-0.9.8/ChunkSpy-0.9.8.zip) , but you will need to apply the following patch so that ChunkSpy understands eLua data types: + * **Generate a bytecode listing on your development PC**. The Lua 5.1 code generator is basically the same on the PC and on the ESP8266, so whilst it isn't identical, using the standard Lua batch compiler `luac` against your source on your PC with the `-l -s` option will give you a good idea of what your code will generate. The main difference between these two variants is the size_t for ESP8266 is 4 bytes rather than the 8 bytes size_t found on modern 64bit development PCs; and the eLua variants generate different access references for ROM data types. If you want to see what the `string.dump()` version generates then drop the `-s` option to retain the debug information. You can also build `luac.cross` with this firmware and this generate lc code for the target ESP architecture. + * **Upload your `.lc` files to the PC and disassemble them there**. There are a number of Lua code disassemblers which can list off the compiled code that your application modules will generate, `if` you have a script to upload files from your ESP8266 to your development PC. I use [ChunkSpy](http://luaforge.net/projects/chunkspy/) which can be downloaded [here](http://files.luaforge.net/releases/chunkspy/chunkspy/ChunkSpy-0.9.8/ChunkSpy-0.9.8.zip) , but you will need to apply the following patch so that ChunkSpy understands eLua data types: ```diff --- a/ChunkSpy-0.9.8/5.1/ChunkSpy.lua 2015-05-04 12:39:01.267975498 +0100 @@ -311,27 +378,7 @@ Note that if you use `require("XXX")` to load your code then this will automatic ### What is the cost of using functions? -Consider the output of `dofile("test1a.lua")` on the following code compared to the equivalent where the function `pnh()` is removed and the extra `print(heap())` statement is placed inline: - -```lua --- test1b.lua -collectgarbage() -local heap = node.heap -print(heap()) -local function pnh() print(heap()) end -pnh() -print(heap()) -``` - -|Heap Value | Function Call | Inline | -|-----------|---------------|--------| -| 1 | 20712 | 21064 | -| 2 | 20624 | 21024 | -| 3 | 20576 | 21024 | - -Here bigger means less RAM used. - -Of course you should still use functions to structure your code and encapsulate common repeated processing, but just bear in mind that each function definition has a relatively high overhead for its header record and stack frame (compared to the 20 odd KB RAM available). *So try to avoid overusing functions. If there are less than a dozen or so lines in the function then you should consider putting this code inline if it makes sense to do so.* +Functions have fixed overheads, so in general the more that you group your application code into larger functions, then the less RAM used will be used overall. The main caveat here is that if you are starting to do "copy and paste" coding across functions then you are wasting resources. So of course you should still use functions to structure your code and encapsulate common repeated processing, but just bear in mind that each function definition has a relatively high overhead for its header record and stack frame. *So try to avoid overusing functions. If there are less than a dozen or so lines in the function then you should consider putting this code inline if it makes sense to do so.* ### What other resources are available? @@ -339,7 +386,7 @@ Of course you should still use functions to structure your code and encapsulate ## Firmware and Lua app development -### How to save memory? -* The NodeMCU development team recommends that you consider using a tailored firmware build, which only includes the modules that you plan to use before developing any Lua application. Once you have the ability to make and flash custom builds, the you also have the option of moving time sensitive or logic intensive code into your own custom module. Doing this can save a large amount of RAM as C code can be run directly from Flash memory. If you want an easy-to-use intermediate option then why note try the [cloud based NodeMCU custom build service](https://nodemcu-build.com)?. +### How to reduce the size of the firmware? +* We recommend that you use a tailored firmware build; one which only includes the modules that you plan to use in developing any Lua application. Once you have the ability to make and flash custom builds, the you also have the option of moving time sensitive or logic intensive code into your own custom module. Doing this can save a large amount of RAM as C code can be run directly from Flash memory. See [Building the firmware](../build/) for more details and options. From 435a4cf5a1683b32072a68dcf1a15eb2d1db2b3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnim=20L=C3=A4uger?= Date: Tue, 20 Jun 2017 21:59:12 +0200 Subject: [PATCH 10/27] backport fix for https://github.com/espressif/esp-idf/issues/631 (#2006) * backport fix for https://github.com/espressif/esp-idf/issues/631 * remove code from intermediate fix --- app/include/lwip/app/dhcpserver.h | 6 +--- app/lwip/app/dhcpserver.c | 50 ++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/app/include/lwip/app/dhcpserver.h b/app/include/lwip/app/dhcpserver.h index d62b5f40..68ad37ad 100644 --- a/app/include/lwip/app/dhcpserver.h +++ b/app/include/lwip/app/dhcpserver.h @@ -5,8 +5,6 @@ #define USE_DNS -#define DHCP_MSGOPTIONS_MIN_LEN 312 - typedef struct dhcps_state{ sint16_t state; } dhcps_state; @@ -23,9 +21,7 @@ typedef struct dhcps_msg { uint8_t chaddr[16]; uint8_t sname[64]; uint8_t file[128]; - // Recommendation from Espressif: - // To avoid crash in DHCP big packages modify option length from 312 to MTU - IPHEAD(20) - UDPHEAD(8) - DHCPHEAD(236). - uint8_t options[IP_FRAG_MAX_MTU - 20 - 8 - 236]; + uint8_t options[312]; }dhcps_msg; #ifndef LWIP_OPEN_SRC diff --git a/app/lwip/app/dhcpserver.c b/app/lwip/app/dhcpserver.c index b4ad7e83..62b4f64c 100755 --- a/app/lwip/app/dhcpserver.c +++ b/app/lwip/app/dhcpserver.c @@ -277,6 +277,21 @@ static void ICACHE_FLASH_ATTR create_msg(struct dhcps_msg *m) uint32 magic_cookie1 = magic_cookie; os_memcpy((char *) m->options, &magic_cookie1, sizeof(magic_cookie1)); } + +struct pbuf * dhcps_pbuf_alloc(u16_t len) +{ + u16_t mlen = sizeof(struct dhcps_msg); + + if (len > mlen) { +#if DHCPS_DEBUG + DHCPS_LOG("dhcps: len=%d mlen=%d", len, mlen); +#endif + mlen = len; + } + + return pbuf_alloc(PBUF_TRANSPORT, mlen, PBUF_RAM); +} + /////////////////////////////////////////////////////////////////////////////////// /* * ����һ��OFFER @@ -284,7 +299,7 @@ static void ICACHE_FLASH_ATTR create_msg(struct dhcps_msg *m) * @param -- m ָ����Ҫ���͵�DHCP msg���� */ /////////////////////////////////////////////////////////////////////////////////// -static void ICACHE_FLASH_ATTR send_offer(struct dhcps_msg *m) +static void ICACHE_FLASH_ATTR send_offer(struct dhcps_msg *m, u16_t len) { uint8_t *end; struct pbuf *p, *q; @@ -298,8 +313,7 @@ static void ICACHE_FLASH_ATTR send_offer(struct dhcps_msg *m) end = add_offer_options(end); end = add_end(end); - // ensure that not more than the minimum options length is transmitted - p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg) - sizeof(m->options) + DHCP_MSGOPTIONS_MIN_LEN, PBUF_RAM); + p = dhcps_pbuf_alloc(len); #if DHCPS_DEBUG os_printf("udhcp: send_offer>>p->ref = %d\n", p->ref); #endif @@ -345,7 +359,7 @@ static void ICACHE_FLASH_ATTR send_offer(struct dhcps_msg *m) * @param m ָ����Ҫ���͵�DHCP msg���� */ /////////////////////////////////////////////////////////////////////////////////// -static void ICACHE_FLASH_ATTR send_nak(struct dhcps_msg *m) +static void ICACHE_FLASH_ATTR send_nak(struct dhcps_msg *m, u16_t len) { u8_t *end; @@ -359,8 +373,7 @@ static void ICACHE_FLASH_ATTR send_nak(struct dhcps_msg *m) end = add_msg_type(&m->options[4], DHCPNAK); end = add_end(end); - // ensure that not more than the minimum options length is transmitted - p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg) - sizeof(m->options) + DHCP_MSGOPTIONS_MIN_LEN, PBUF_RAM); + p = dhcps_pbuf_alloc(len); #if DHCPS_DEBUG os_printf("udhcp: send_nak>>p->ref = %d\n", p->ref); #endif @@ -406,7 +419,7 @@ static void ICACHE_FLASH_ATTR send_nak(struct dhcps_msg *m) * @param m ָ����Ҫ���͵�DHCP msg���� */ /////////////////////////////////////////////////////////////////////////////////// -static void ICACHE_FLASH_ATTR send_ack(struct dhcps_msg *m) +static void ICACHE_FLASH_ATTR send_ack(struct dhcps_msg *m, u16_t len) { u8_t *end; @@ -421,8 +434,7 @@ static void ICACHE_FLASH_ATTR send_ack(struct dhcps_msg *m) end = add_offer_options(end); end = add_end(end); - // ensure that not more than the minimum options length is transmitted - p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct dhcps_msg) - sizeof(m->options) + DHCP_MSGOPTIONS_MIN_LEN, PBUF_RAM); + p = dhcps_pbuf_alloc(len); #if DHCPS_DEBUG os_printf("udhcp: send_ack>>p->ref = %d\n", p->ref); #endif @@ -605,7 +617,7 @@ static void ICACHE_FLASH_ATTR handle_dhcp(void *arg, uint16_t port) { struct dhcps_msg *pmsg_dhcps = NULL; - sint16_t tlen = 0; + sint16_t tlen = 0, malloc_len; u16_t i = 0; u16_t dhcps_msg_cnt = 0; u8_t *p_dhcps_msg = NULL; @@ -616,11 +628,21 @@ static void ICACHE_FLASH_ATTR handle_dhcp(void *arg, #endif if (p==NULL) return; - pmsg_dhcps = (struct dhcps_msg *)os_zalloc(sizeof(struct dhcps_msg)); + malloc_len = sizeof(struct dhcps_msg); +#if DHCPS_DEBUG + DHCPS_LOG("dhcps: handle_dhcp malloc_len=%d rx_len=%d", malloc_len, p->tot_len); +#endif + if (malloc_len < p->tot_len) { + malloc_len = p->tot_len; + } + + pmsg_dhcps = (struct dhcps_msg *)os_malloc(malloc_len); if (NULL == pmsg_dhcps){ pbuf_free(p); return; } + memset(pmsg_dhcps , 0x00 , malloc_len); + p_dhcps_msg = (u8_t *)pmsg_dhcps; tlen = p->tot_len; data = p->payload; @@ -660,19 +682,19 @@ static void ICACHE_FLASH_ATTR handle_dhcp(void *arg, #if DHCPS_DEBUG os_printf("dhcps: handle_dhcp-> DHCPD_STATE_OFFER\n"); #endif - send_offer(pmsg_dhcps); + send_offer(pmsg_dhcps, malloc_len); break; case DHCPS_STATE_ACK://3 #if DHCPS_DEBUG os_printf("dhcps: handle_dhcp-> DHCPD_STATE_ACK\n"); #endif - send_ack(pmsg_dhcps); + send_ack(pmsg_dhcps, malloc_len); break; case DHCPS_STATE_NAK://4 #if DHCPS_DEBUG os_printf("dhcps: handle_dhcp-> DHCPD_STATE_NAK\n"); #endif - send_nak(pmsg_dhcps); + send_nak(pmsg_dhcps, malloc_len); break; default : break; From 26df4a32adc83e0a7516b32575caa9b52fda21af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnim=20L=C3=A4uger?= Date: Sun, 25 Jun 2017 20:41:36 +0200 Subject: [PATCH 11/27] Change default flash mode to 'dio' in fw image header (#2013) --- Makefile | 2 +- docs/en/flash.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ec1632b..28b69fec 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,7 @@ endef $(BINODIR)/%.bin: $(IMAGEODIR)/%.out @mkdir -p $(BINODIR) - $(ESPTOOL) elf2image $< -o $(FIRMWAREDIR) + $(ESPTOOL) elf2image --flash_mode dio --flash_freq 40m $< -o $(FIRMWAREDIR) ############################################################# # Rules base diff --git a/docs/en/flash.md b/docs/en/flash.md index 255ce7ca..8c5cdb11 100644 --- a/docs/en/flash.md +++ b/docs/en/flash.md @@ -30,6 +30,7 @@ Run the following command to flash an *aggregated* binary as is produced for exa - See [below](#determine-flash-size) if you don't know or are uncertain about the capacity of the flash chip on your device. It might help to double check as e.g. some ESP-01 modules come with 512kB while others are equipped with 1MB. - esptool.py is under heavy development. It's advised you run the latest version (check with `esptool.py version`). Since this documentation may not have been able to keep up refer to the [esptool flash modes documentation](https://github.com/themadinventor/esptool#flash-modes) for current options and parameters. +- The firmware image file contains default settings `dio` for flash mode and `40m` for flash frequency. - In some uncommon cases, the [SDK init data](#sdk-init-data) may be invalid and NodeMCU may fail to boot. The easiest solution is to fully erase the chip before flashing: `esptool.py --port erase_flash` From 583afc0f337a6918a29fa01c101d9151c93f68ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20St=C3=B6r?= Date: Tue, 27 Jun 2017 17:17:28 +0200 Subject: [PATCH 12/27] Remove hardware FAQ, fixes #2015 --- docs/en/hardware-faq.md | 44 +---------------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/docs/en/hardware-faq.md b/docs/en/hardware-faq.md index ff31c124..ab81d0e1 100644 --- a/docs/en/hardware-faq.md +++ b/docs/en/hardware-faq.md @@ -1,45 +1,3 @@ # Hardware FAQ -## What is this FAQ for? - -This FAQ addresses hardware-specific issues relating to the NodeMcu firmware on -NoceMCU Inc Devkits and other ESP-8266 modules. - -## Hardware Specifics - -## Why file writes fail all the time on DEVKIT V1.0? - -NodeMCU DEVKIT V1.0 uses ESP12-E-DIO(ESP-12-D) module. This module runs the -Flash memory in [Dual IO SPI](#whats-the-different-between-dio-and-qio-mode) -(DIO) mode. This firmware will not be correctly loaded if you use old flashtool -versions, and the filesystem will not work if you used a pre 0.9.6 firmware -version (<0.9.5) or old. The easiest way to resolve this problem s update all -the firmware and flash tool to current version. - -- Use the latest [esptool.py](https://github.com/themadinventor/esptool) with -DIO support and command option to flash firmware, or - -- Use the latest [NodeMCU flasher](https://github.com/NodeMCU/NodeMCU-flasher) -with default option. (You must select the `restore to default` option in advanced -menu tab), or - -- Use the latest Espressif's flash tool -- see [this Espressif forum -topic](http://bbs.espressif.com/viewtopic.php?f=5&t=433) (without auto download -support). Use DIO mode and 32M flash size option, and flash latest firmware to -0x00000. Before flashing firmware, remember to hold FLASH button, and press RST -button once. Note that the new NodeMCU our firmware download tool, when -released, will be capable of flashing firmware automatically without any button -presses. - -## What's the different between DIO and QIO mode? -Whether DIO or QIO modes are available depends on the physical connection -between the ESP8266 CPU and its onboard flash chip. QIO connects to the flash -using 5 data pins as compared to DIO's 3. This frees up an extra 2 IO pins for -GPIO use, but this also halves the read/write data-rate to Flash compared to -QIO modules. - -## How to use DEVKIT V0.9 on Mac OS X? - - -### How does DEVKIT use DTR and RTS enter download mode? - +This content is now maintained at [http://www.esp8266.com/wiki/doku.php?id=nodemcu-unofficial-faq](http://www.esp8266.com/wiki/doku.php?id=nodemcu-unofficial-faq). From 4095c26bd0d3c859c5b66ad7e460485b068b8d8e Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Wed, 28 Jun 2017 01:02:09 -0700 Subject: [PATCH 13/27] Update documentation for wifi.sta.setaplimit() (#2017) --- docs/en/modules/wifi.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/en/modules/wifi.md b/docs/en/modules/wifi.md index 1d9a3026..fd8c8185 100644 --- a/docs/en/modules/wifi.md +++ b/docs/en/modules/wifi.md @@ -899,7 +899,10 @@ Set Maximum number of Access Points to store in flash. - This value is written to flash !!! Attention - If 5 Access Points are stored and AP limit is set to 4, the AP at index 5 will remain until [`node.restore()`](node.md#noderestore) is called or AP limit is set to 5 and AP is overwritten. + New setting will not take effect until restart. + +!!! Note + If 5 Access Points are stored and AP limit is set to 4, the AP at index 5 will remain until [`node.restore()`](node.md#noderestore) is called or AP limit is set to 5 and AP is overwritten. #### Syntax `wifi.sta.setaplimit(qty)` @@ -913,7 +916,7 @@ Set Maximum number of Access Points to store in flash. #### Example ```lua -wifi.sta.setaplimit(true) +wifi.sta.setaplimit(5) ``` #### See also From e2fc37fa17a7a48eaa7a215212f7b2aa19903a00 Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Thu, 29 Jun 2017 13:05:40 -0700 Subject: [PATCH 14/27] Removed code allowing argument style station configuration (#2018) --- app/modules/wifi.c | 56 ++-------------------------------------------- 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/app/modules/wifi.c b/app/modules/wifi.c index 9bb24e71..df8932e8 100644 --- a/app/modules/wifi.c +++ b/app/modules/wifi.c @@ -930,61 +930,9 @@ static int wifi_station_config( lua_State* L ) #endif } - else //to be deprecated + else { - platform_print_deprecation_note("Argument style station configuration is replaced by table style station configuration", "in the next version"); - - const char *ssid = luaL_checklstring( L, 1, &sl ); - luaL_argcheck(L, (sl>=0 && sl=0 && pl<=sizeof(sta_conf.password)), 2, "length:0-64"); /* WPA = min 8, WEP = min 5 ASCII characters for a 40-bit key */ - - memcpy(sta_conf.password, password, pl); - - if(lua_isnumber(L, 3)) - { - lua_Integer lint=luaL_checkinteger( L, 3 ); - if ( lint != 0 && lint != 1) - return luaL_error( L, "wrong arg type" ); - auto_connect=(bool)lint; - } - else if (lua_isstring(L, 3)&& !(lua_isnumber(L, 3))) - { - lua_pushnil(L); - lua_insert(L, 3); - - } - else - { - if(lua_isnil(L, 3)) - return luaL_error( L, "wrong arg type" ); - auto_connect=1; - } - - if(lua_isnumber(L, 4)) - { - sta_conf.bssid_set = 0; - memset(sta_conf.bssid, 0, sizeof(sta_conf.bssid)); - } - else - { - if (lua_isstring(L, 4)) - { - const char *macaddr = luaL_checklstring( L, 4, &ml ); - luaL_argcheck(L, ml==sizeof("AA:BB:CC:DD:EE:FF")-1, 1, INVALID_MAC_STR); - memset(sta_conf.bssid, 0, sizeof(sta_conf.bssid)); - ets_str2macaddr(sta_conf.bssid, macaddr); - sta_conf.bssid_set = 1; - } - else - { - sta_conf.bssid_set = 0; - memset(sta_conf.bssid, 0, sizeof(sta_conf.bssid)); - } - } + return luaL_argerror(L, 1, "config table not found!"); } #if defined(WIFI_DEBUG) From 15b4fa24fd4666f1b6d993c8f96e80886cfd1f34 Mon Sep 17 00:00:00 2001 From: Gregor Hartmann Date: Sat, 1 Jul 2017 18:29:54 +0200 Subject: [PATCH 15/27] Call HTTP callback in all cases (#2020) * fix 2007 Call callback in all cases, call callback with errorcode -1 if no connection could be establioshed * change logging from ERR to DEBUG * make debug output more clear (hopefully) * add handling of errors to docs, note error handling on every call instead of only in the main documentation --- app/http/httpclient.c | 17 +++++++++++++++-- docs/en/modules/http.md | 10 +++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/app/http/httpclient.c b/app/http/httpclient.c index 0ac8bf4f..094c5ece 100644 --- a/app/http/httpclient.c +++ b/app/http/httpclient.c @@ -451,20 +451,33 @@ static void ICACHE_FLASH_ATTR http_timeout_callback( void *arg ) struct espconn * conn = (struct espconn *) arg; if ( conn == NULL ) { + HTTPCLIENT_DEBUG( "Connection is NULL" ); return; } if ( conn->reverse == NULL ) { + HTTPCLIENT_DEBUG( "Connection request data (reverse) is NULL" ); return; } request_args_t * req = (request_args_t *) conn->reverse; + HTTPCLIENT_DEBUG( "Calling disconnect" ); /* Call disconnect */ + sint8 result; #ifdef CLIENT_SSL_ENABLE if ( req->secure ) - espconn_secure_disconnect( conn ); + result = espconn_secure_disconnect( conn ); else #endif - espconn_disconnect( conn ); + result = espconn_disconnect( conn ); + + if (result == ESPCONN_OK || result == ESPCONN_INPROGRESS) + return; + else + { + /* not connected; execute the callback ourselves. */ + HTTPCLIENT_DEBUG( "manually Calling disconnect callback due to error %d", result ); + http_disconnect_callback( arg ); + } } diff --git a/docs/en/modules/http.md b/docs/en/modules/http.md index 6ead9903..8e5d4626 100644 --- a/docs/en/modules/http.md +++ b/docs/en/modules/http.md @@ -33,7 +33,7 @@ Executes a HTTP DELETE request. Note that concurrent requests are not supported. - `url` The URL to fetch, including the `http://` or `https://` prefix - `headers` Optional additional headers to append, *including \r\n*; may be `nil` - `body` The body to post; must already be encoded in the appropriate format, but may be empty -- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers` +- `callback` The callback function to be invoked when the response has been received or an error occurred; it is invoked with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` is set to -1. #### Returns `nil` @@ -62,7 +62,7 @@ Executes a HTTP GET request. Note that concurrent requests are not supported. #### Parameters - `url` The URL to fetch, including the `http://` or `https://` prefix - `headers` Optional additional headers to append, *including \r\n*; may be `nil` -- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers` +- `callback` The callback function to be invoked when the response has been received or an error occurred; it is invoked with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` is set to -1 #### Returns `nil` @@ -89,7 +89,7 @@ Executes a HTTP POST request. Note that concurrent requests are not supported. - `url` The URL to fetch, including the `http://` or `https://` prefix - `headers` Optional additional headers to append, *including \r\n*; may be `nil` - `body` The body to post; must already be encoded in the appropriate format, but may be empty -- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers` +- `callback` The callback function to be invoked when the response has been received or an error occurred; it is invoked with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` is set to -1. #### Returns `nil` @@ -119,7 +119,7 @@ Executes a HTTP PUT request. Note that concurrent requests are not supported. - `url` The URL to fetch, including the `http://` or `https://` prefix - `headers` Optional additional headers to append, *including \r\n*; may be `nil` - `body` The body to post; must already be encoded in the appropriate format, but may be empty -- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers` +- `callback` The callback function to be invoked when the response has been received or an error occurred; it is invoked with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` is set to -1. #### Returns `nil` @@ -150,7 +150,7 @@ Execute a custom HTTP request for any HTTP method. Note that concurrent requests - `method` The HTTP method to use, e.g. "GET", "HEAD", "OPTIONS" etc - `headers` Optional additional headers to append, *including \r\n*; may be `nil` - `body` The body to post; must already be encoded in the appropriate format, but may be empty -- `callback` The callback function to be invoked when the response has been received; it is invoked with the arguments `status_code`, `body` and `headers` +- `callback` The callback function to be invoked when the response has been received or an error occurred; it is invoked with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` is set to -1. #### Returns `nil` From 4ce2d6830138c05c8172ad13ea8e536bd4fbf0c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20St=C3=B6r?= Date: Sat, 1 Jul 2017 18:32:44 +0200 Subject: [PATCH 16/27] Add missing period --- docs/en/modules/http.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/modules/http.md b/docs/en/modules/http.md index 8e5d4626..68c552d2 100644 --- a/docs/en/modules/http.md +++ b/docs/en/modules/http.md @@ -62,7 +62,7 @@ Executes a HTTP GET request. Note that concurrent requests are not supported. #### Parameters - `url` The URL to fetch, including the `http://` or `https://` prefix - `headers` Optional additional headers to append, *including \r\n*; may be `nil` -- `callback` The callback function to be invoked when the response has been received or an error occurred; it is invoked with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` is set to -1 +- `callback` The callback function to be invoked when the response has been received or an error occurred; it is invoked with the arguments `status_code`, `body` and `headers`. In case of an error `status_code` is set to -1. #### Returns `nil` From 2061167bd925b0b4197c35ada5476de2e560214d Mon Sep 17 00:00:00 2001 From: Johny Mattsson Date: Mon, 3 Jul 2017 13:29:47 +1000 Subject: [PATCH 17/27] Add check for unresolved-but-unused symbols at build. From what I can tell they *should* be harmless, but I'd rather we keep a neat house in the first place. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 28b69fec..5380a86e 100644 --- a/Makefile +++ b/Makefile @@ -188,6 +188,7 @@ endef $(BINODIR)/%.bin: $(IMAGEODIR)/%.out @mkdir -p $(BINODIR) + @$(NM) $< | grep -w U && { echo "Firmware has unresolved (but unused) symbols - should be fixed nevertheless!"; exit 1; } || true $(ESPTOOL) elf2image --flash_mode dio --flash_freq 40m $< -o $(FIRMWAREDIR) ############################################################# From ea4d33715f23e7df256debf2b0fd80ab1241b694 Mon Sep 17 00:00:00 2001 From: Johny Mattsson Date: Mon, 3 Jul 2017 13:31:32 +1000 Subject: [PATCH 18/27] Cleaned up sjson module build. - Move jsonsl build to regular library build rather than #include the .c file - Provide wrappers for malloc/calloc/free to fix undefined symbol warnings. --- Makefile | 2 +- app/Makefile | 10 +++++---- app/modules/sjson.c | 6 ++---- app/sjson/Makefile | 47 +++++++++++++++++++++++++++++++++++++++++ app/sjson/json_config.h | 7 ++++++ app/sjson/memcompat.h | 11 ++++++++++ 6 files changed, 74 insertions(+), 9 deletions(-) create mode 100644 app/sjson/Makefile create mode 100644 app/sjson/json_config.h create mode 100644 app/sjson/memcompat.h diff --git a/Makefile b/Makefile index 5380a86e..395339d4 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,7 @@ endef $(BINODIR)/%.bin: $(IMAGEODIR)/%.out @mkdir -p $(BINODIR) - @$(NM) $< | grep -w U && { echo "Firmware has unresolved (but unused) symbols - should be fixed nevertheless!"; exit 1; } || true + @$(NM) $< | grep -w U && { echo "Firmware has undefined (but unused) symbols!"; exit 1; } || true $(ESPTOOL) elf2image --flash_mode dio --flash_freq 40m $< -o $(FIRMWAREDIR) ############################################################# diff --git a/app/Makefile b/app/Makefile index e18139a9..314f0f18 100644 --- a/app/Makefile +++ b/app/Makefile @@ -48,7 +48,8 @@ SUBDIRS= \ websocket \ swTimer \ misc \ - pm \ + pm \ + sjson \ endif # } PDIR @@ -94,14 +95,15 @@ COMPONENTS_eagle.app.v6 = \ dhtlib/libdhtlib.a \ tsl2561/tsl2561lib.a \ http/libhttp.a \ - pm/libpm.a \ + pm/libpm.a \ websocket/libwebsocket.a \ esp-gdbstub/libgdbstub.a \ net/libnodemcu_net.a \ - mbedtls/libmbedtls.a \ + mbedtls/libmbedtls.a \ modules/libmodules.a \ swTimer/libswtimer.a \ - misc/libmisc.a \ + misc/libmisc.a \ + sjson/libsjson.a \ # Inspect the modules library and work out which modules need to be linked. diff --git a/app/modules/sjson.c b/app/modules/sjson.c index ada01b5b..e5ed92f5 100644 --- a/app/modules/sjson.c +++ b/app/modules/sjson.c @@ -11,10 +11,8 @@ #include "c_limits.h" #endif -#define JSONSL_STATE_USER_FIELDS int lua_object_ref; int used_count; -#define JSONSL_NO_JPR - -#include "jsonsl.c" +#include "json_config.h" +#include "jsonsl.h" #define LUA_SJSONLIBNAME "sjson" diff --git a/app/sjson/Makefile b/app/sjson/Makefile new file mode 100644 index 00000000..a4398004 --- /dev/null +++ b/app/sjson/Makefile @@ -0,0 +1,47 @@ +############################################################# +# Required variables for each makefile +# Discard this section from all parent makefiles +# Expected variables (with automatic defaults): +# CSRCS (all "C" files in the dir) +# SUBDIRS (all subdirs with a Makefile) +# GEN_LIBS - list of libs to be generated () +# GEN_IMAGES - list of images to be generated () +# COMPONENTS_xxx - a list of libs/objs in the form +# subdir/lib to be extracted and rolled up into +# a generated lib/image xxx.a () +# +ifndef PDIR +GEN_LIBS = libsjson.a +endif + +STD_CFLAGS=-std=gnu11 -Wimplicit + +############################################################# +# Configuration i.e. compile options etc. +# Target specific stuff (defines etc.) goes in here! +# Generally values applying to a tree are captured in the +# makefile at its root level - these are then overridden +# for a subtree within the makefile rooted therein +# +DEFINES += -include memcompat.h -include json_config.h + +############################################################# +# Recursion Magic - Don't touch this!! +# +# Each subtree potentially has an include directory +# corresponding to the common APIs applicable to modules +# rooted at that subtree. Accordingly, the INCLUDE PATH +# of a module can only contain the include directories up +# its parent path, and not its siblings +# +# Required for each makefile to inherit from the parent +# + +INCLUDES := $(INCLUDES) -I $(PDIR)include +INCLUDES += -I ./ +INCLUDES += -I ./include +INCLUDES += -I ../include +INCLUDES += -I ../../include +PDIR := ../$(PDIR) +sinclude $(PDIR)Makefile + diff --git a/app/sjson/json_config.h b/app/sjson/json_config.h new file mode 100644 index 00000000..8bc13b8d --- /dev/null +++ b/app/sjson/json_config.h @@ -0,0 +1,7 @@ +#ifndef __JSON_CONFIG_H__ +#define __JSON_CONFIG_H__ + +#define JSONSL_STATE_USER_FIELDS int lua_object_ref; int used_count; +#define JSONSL_NO_JPR + +#endif diff --git a/app/sjson/memcompat.h b/app/sjson/memcompat.h new file mode 100644 index 00000000..2e3f84fb --- /dev/null +++ b/app/sjson/memcompat.h @@ -0,0 +1,11 @@ +#ifndef __MEMCOMPAT_H__ +#define __MEMCOMPAT_H__ + +#include "c_types.h" +#include "mem.h" + +static inline void *malloc(size_t sz) { return os_malloc(sz); } +static inline void free(void *p) { return os_free(p); } +static inline void *calloc(size_t n, size_t sz) { return os_zalloc(n*sz); } + +#endif From c01f653736e2e04fc522873ef50b3126670d2a2b Mon Sep 17 00:00:00 2001 From: Johny Mattsson Date: Thu, 6 Jul 2017 02:00:26 +1000 Subject: [PATCH 19/27] Unbreak build when SPIFFS_CACHE==0. (#2028) --- app/spiffs/spiffs_config.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/spiffs/spiffs_config.h b/app/spiffs/spiffs_config.h index c6f1b18f..10ccf0e1 100644 --- a/app/spiffs/spiffs_config.h +++ b/app/spiffs/spiffs_config.h @@ -91,6 +91,11 @@ #ifndef SPIFFS_CACHE_STATS #define SPIFFS_CACHE_STATS 1 #endif +#else +// No SPIFFS_CACHE, also disable SPIFFS_CACHE_WR +#ifndef SPIFFS_CACHE_WR +#define SPIFFS_CACHE_WR 0 +#endif #endif // Always check header of each accessed page to ensure consistent state. From 761c9dff7ffe825d788f08df07f1ebc35c458507 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 6 Jul 2017 23:18:06 -0400 Subject: [PATCH 20/27] Try to fix the blocksize issues --- app/spiffs/spiffs.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app/spiffs/spiffs.c b/app/spiffs/spiffs.c index 08d0dfb5..96ea7927 100644 --- a/app/spiffs/spiffs.c +++ b/app/spiffs/spiffs.c @@ -73,7 +73,7 @@ static bool myspiffs_set_location(spiffs_config *cfg, int align, int offset, int * Returns TRUE if FS was found * align must be a power of two */ -static bool myspiffs_set_cfg(spiffs_config *cfg, int align, int offset, bool force_create) { +static bool myspiffs_set_cfg_block(spiffs_config *cfg, int align, int offset, int block_size, bool force_create) { cfg->phys_erase_block = INTERNAL_FLASH_SECTOR_SIZE; // according to datasheet cfg->log_page_size = LOG_PAGE_SIZE; // as we said @@ -81,10 +81,8 @@ static bool myspiffs_set_cfg(spiffs_config *cfg, int align, int offset, bool for cfg->hal_write_f = my_spiffs_write; cfg->hal_erase_f = my_spiffs_erase; - if (!myspiffs_set_location(cfg, align, offset, LOG_BLOCK_SIZE)) { - if (!myspiffs_set_location(cfg, align, offset, LOG_BLOCK_SIZE_SMALL_FS)) { - return FALSE; - } + if (!myspiffs_set_location(cfg, align, offset, block_size)) { + return FALSE; } NODE_DBG("fs.start:%x,max:%x\n",cfg->phys_addr,cfg->phys_size); @@ -109,6 +107,16 @@ static bool myspiffs_set_cfg(spiffs_config *cfg, int align, int offset, bool for #endif } +static bool myspiffs_set_cfg(spiffs_config *cfg, int align, int offset, bool force_create) { + if (force_create) { + return myspiffs_set_cfg_block(cfg, align, offset, LOG_BLOCK_SIZE , TRUE) || + myspiffs_set_cfg_block(cfg, align, offset, LOG_BLOCK_SIZE_SMALL_FS, TRUE); + } + + return myspiffs_set_cfg_block(cfg, align, offset, LOG_BLOCK_SIZE_SMALL_FS, FALSE) || + myspiffs_set_cfg_block(cfg, align, offset, LOG_BLOCK_SIZE , FALSE); +} + static bool myspiffs_find_cfg(spiffs_config *cfg, bool force_create) { int i; From 2e33abe198ce9d32762bbd45d3dbade2fe5ff755 Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Sat, 8 Jul 2017 13:49:02 -0700 Subject: [PATCH 21/27] Modify wifi.sta.get*config() to return AP's MAC (#2026) * Modified wifi.sta.get*config() to return AP's MAC even if bssid_set==0 * Improved documentation for wifi.sta.getapinfo, fixes #2025 --- app/modules/wifi.c | 15 ++++++++------- docs/en/modules/wifi.md | 18 +++++++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/app/modules/wifi.c b/app/modules/wifi.c index df8932e8..cbefe33c 100644 --- a/app/modules/wifi.c +++ b/app/modules/wifi.c @@ -666,13 +666,14 @@ static int wifi_station_getconfig( lua_State* L, bool get_flash_cfg) lua_setfield(L, -2, "pwd"); } - if(sta_conf.bssid_set==1) - { - memset(temp, 0, sizeof(temp)); - c_sprintf(temp, MACSTR, MAC2STR(sta_conf.bssid)); - lua_pushstring( L, temp); - lua_setfield(L, -2, "bssid"); - } + lua_pushboolean(L, sta_conf.bssid_set); + lua_setfield(L, -2, "bssid_set"); + + memset(temp, 0, sizeof(temp)); + c_sprintf(temp, MACSTR, MAC2STR(sta_conf.bssid)); + lua_pushstring( L, temp); + lua_setfield(L, -2, "bssid"); + return 1; } else diff --git a/docs/en/modules/wifi.md b/docs/en/modules/wifi.md index fd8c8185..abddf471 100644 --- a/docs/en/modules/wifi.md +++ b/docs/en/modules/wifi.md @@ -662,7 +662,8 @@ Get information of APs cached by ESP8266 station. - `1-5` index of AP. (the index corresponds to index used by [`wifi.sta.changeap()`](#wifistachangeap) and [`wifi.sta.getapindex()`](#wifistagetapindex)) - `ssid` ssid of Access Point - `pwd` password for Access Point, `nil` if no password was configured - - `bssid` MAC address of Access Point, `nil` if no MAC address was configured + - `bssid` MAC address of Access Point + - `nil` will be returned if no MAC address was configured during station configuration. #### Example ```lua @@ -733,7 +734,9 @@ If `return_table` is `true`: - `config_table` - `ssid` ssid of Access Point. - `pwd` password to Access Point, `nil` if no password was configured - - `bssid` MAC address of Access Point, `nil` if no MAC address was configured + - `bssid_set` will return `true` if the station was configured specifically to connect to the AP with the matching `bssid`. + - `bssid` If a connection has been made to the configured AP this field will contain the AP's MAC address. Otherwise "ff:ff:ff:ff:ff:ff" will be returned. + If `return_table` is `false`: @@ -744,8 +747,8 @@ If `return_table` is `false`: ```lua --Get current Station configuration (NEW FORMAT) do -local def_sta_config=wifi.sta.getconfig(true) -print(string.format("\tDefault station config\n\tssid:\"%s\"\tpassword:\"%s\"%s", def_sta_config.ssid, def_sta_config.pwd, (type(def_sta_config.bssid)=="string" and "\tbssid:\""..def_sta_config.bssid.."\"" or ""))) +local sta_config=wifi.sta.getconfig(true) +print(string.format("\tCurrent station config\n\tssid:\"%s\"\tpassword:\"%s\"\n\tbssid:\"%s\"\tbssid_set:%s", sta_config.ssid, sta_config.pwd, sta_config.bssid, (sta_config.bssid_set and "true" or "false"))) end --Get current Station configuration (OLD FORMAT) @@ -780,7 +783,8 @@ If `return_table` is `true`: - `config_table` - `ssid` ssid of Access Point. - `pwd` password to Access Point, `nil` if no password was configured - - `bssid` MAC address of Access Point, `nil` if no MAC address was configured + - `bssid_set` will return `true` if the station was configured specifically to connect to the AP with the matching `bssid`. + - `bssid` If a connection has been made to the configured AP this field will contain the AP's MAC address. Otherwise "ff:ff:ff:ff:ff:ff" will be returned. If `return_table` is `false`: @@ -791,8 +795,8 @@ If `return_table` is `false`: ```lua --Get default Station configuration (NEW FORMAT) do - local def_sta_config=wifi.sta.getdefaultconfig(true) - print(string.format("\tDefault station config\n\tssid:\"%s\"\tpassword:\"%s\"%s", def_sta_config.ssid, def_sta_config.pwd, (type(def_sta_config.bssid)=="string" and "\tbssid:\""..def_sta_config.bssid.."\"" or ""))) +local def_sta_config=wifi.sta.getdefaultconfig(true) +print(string.format("\tDefault station config\n\tssid:\"%s\"\tpassword:\"%s\"\n\tbssid:\"%s\"\tbssid_set:%s", def_sta_config.ssid, def_sta_config.pwd, def_sta_config.bssid, (def_sta_config.bssid_set and "true" or "false"))) end --Get default Station configuration (OLD FORMAT) From 9edcce5b44374e6647ea809dd99cb42f21a5de19 Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Sat, 8 Jul 2017 13:51:33 -0700 Subject: [PATCH 22/27] Update wifi.sta.config to save configuration to flash by default (#1998) --- app/modules/wifi.c | 2 +- docs/en/modules/wifi.md | 4 ++-- docs/en/upload.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/modules/wifi.c b/app/modules/wifi.c index cbefe33c..1a1e73d7 100644 --- a/app/modules/wifi.c +++ b/app/modules/wifi.c @@ -830,7 +830,7 @@ static int wifi_station_config( lua_State* L ) } else { - save_to_flash=false; + save_to_flash=true; } lua_pop(L, 1); diff --git a/docs/en/modules/wifi.md b/docs/en/modules/wifi.md index abddf471..0b50506a 100644 --- a/docs/en/modules/wifi.md +++ b/docs/en/modules/wifi.md @@ -410,8 +410,8 @@ Sets the WiFi station configuration. - "AC-1D-1C-B1-0B-22" - "DE AD BE EF 7A C0" - `save` Save station configuration to flash. - - `true` configuration **will** be retained through power cycle. - - `false` configuration **will not** be retained through power cycle. (Default). + - `true` configuration **will** be retained through power cycle. (Default). + - `false` configuration **will not** be retained through power cycle. - Event callbacks will only be available if `WIFI_SDK_EVENT_MONITOR_ENABLE` is uncommented in `user_config.h` - Please note: To ensure all station events are handled at boot time, all relevant callbacks must be registered as early as possible in `init.lua` with either `wifi.sta.config()` or `wifi.eventmon.register()`. - `connected_cb`: Callback to execute when station is connected to an access point. (Optional) diff --git a/docs/en/upload.md b/docs/en/upload.md index 441bf3d8..2f56a79b 100644 --- a/docs/en/upload.md +++ b/docs/en/upload.md @@ -111,7 +111,7 @@ wifi.eventmon.register(wifi.eventmon.STA_DISCONNECTED, wifi_disconnect_event) print("Connecting to WiFi access point...") wifi.setmode(wifi.STATION) -wifi.sta.config({ssid=SSID, pwd=PASSWORD, save=true}) +wifi.sta.config({ssid=SSID, pwd=PASSWORD}) -- wifi.sta.connect() not necessary because config() uses auto-connect=true by default ``` From d93465cd865dc21efac277e920cf7169d88273d7 Mon Sep 17 00:00:00 2001 From: Philip Gladstone Date: Tue, 18 Jul 2017 16:51:20 -0400 Subject: [PATCH 23/27] Add tracking and control of the rate error in the clock crystal. (#1697) * Improve RTC timekeeping -- includes clock rate tracking * Improved division by 1M * Fix crash in sntp * Disable RTC debug * Get the offset correct * Add comments on where the mysterious numbers came from * Fix a crash with auto repeat mode and errors on repeat --- app/include/rtc/rtctime.h | 1 + app/include/rtc/rtctime_internal.h | 294 ++++++++++++++++++++++------- app/modules/rtctime.c | 13 +- app/modules/sntp.c | 2 +- docs/en/modules/rtctime.md | 25 ++- 5 files changed, 255 insertions(+), 80 deletions(-) diff --git a/app/include/rtc/rtctime.h b/app/include/rtc/rtctime.h index 3677b949..81bf8f67 100644 --- a/app/include/rtc/rtctime.h +++ b/app/include/rtc/rtctime.h @@ -62,6 +62,7 @@ struct rtc_tm{ void TEXT_SECTION_ATTR rtctime_early_startup (void); void rtctime_late_startup (void); +void rtctime_adjust_rate (int rate); void rtctime_gettimeofday (struct rtc_timeval *tv); void rtctime_settimeofday (const struct rtc_timeval *tv); bool rtctime_have_time (void); diff --git a/app/include/rtc/rtctime_internal.h b/app/include/rtc/rtctime_internal.h index d69fab40..b9a4068d 100644 --- a/app/include/rtc/rtctime_internal.h +++ b/app/include/rtc/rtctime_internal.h @@ -32,6 +32,11 @@ * @author Johny Mattsson */ +/* + * It is vital that this file is only included once in the entire + * system. + */ + #ifndef _RTCTIME_INTERNAL_H_ #define _RTCTIME_INTERNAL_H_ @@ -181,6 +186,17 @@ #define CPU_DEFAULT_MHZ 80 #define CPU_BOOTUP_MHZ 52 +#ifdef RTC_DEBUG_ENABLED +#define RTC_DBG(...) do { if (rtc_dbg_enabled == 'R') { dbg_printf(__VA_ARGS__); } } while (0) +static bool rtc_dbg_enabled; +#define RTC_DBG_ENABLED() rtc_dbg_enabled = 'R' +#define RTC_DBG_NOT_ENABLED() rtc_dbg_enabled = 0 +#else +#define RTC_DBG(...) +#define RTC_DBG_ENABLED() +#define RTC_DBG_NOT_ENABLED() +#endif + // RTCTIME storage #define RTC_TIME_MAGIC_POS (RTC_TIME_BASE+0) #define RTC_CYCLEOFFSETL_POS (RTC_TIME_BASE+1) @@ -190,8 +206,22 @@ #define RTC_CALIBRATION_POS (RTC_TIME_BASE+5) #define RTC_SLEEPTOTALUS_POS (RTC_TIME_BASE+6) #define RTC_SLEEPTOTALCYCLES_POS (RTC_TIME_BASE+7) -#define RTC_TODOFFSETUS_POS (RTC_TIME_BASE+8) -#define RTC_LASTTODUS_POS (RTC_TIME_BASE+9) +//#define RTC_TODOFFSETUS_POS (RTC_TIME_BASE+8) +//#define RTC_LASTTODUS_POS (RTC_TIME_BASE+9) +#define RTC_USRATE_POS (RTC_TIME_BASE+8) + +static uint32_t rtc_time_magic; +static uint64_t rtc_cycleoffset; +static uint32_t rtc_lastsourceval; +static uint32_t rtc_sourcecycleunits; +static uint32_t rtc_calibration; +static uint32_t rtc_sleeptotalus; +static uint32_t rtc_sleeptotalcycles; +static uint64_t rtc_usatlastrate; +static uint64_t rtc_rateadjustedus; +static uint32_t rtc_todoffsetus; +static uint32_t rtc_lasttodus; +static uint32_t rtc_usrate; struct rtc_timeval @@ -200,12 +230,78 @@ struct rtc_timeval uint32_t tv_usec; }; +static void bbram_load() { + rtc_time_magic = rtc_mem_read(RTC_TIME_MAGIC_POS); + rtc_cycleoffset = rtc_mem_read64(RTC_CYCLEOFFSETL_POS); + rtc_lastsourceval = rtc_mem_read(RTC_LASTSOURCEVAL_POS); + rtc_sourcecycleunits = rtc_mem_read(RTC_SOURCECYCLEUNITS_POS); + rtc_calibration = rtc_mem_read(RTC_CALIBRATION_POS); + rtc_sleeptotalus = rtc_mem_read(RTC_SLEEPTOTALUS_POS); + rtc_sleeptotalcycles = rtc_mem_read(RTC_SLEEPTOTALCYCLES_POS); + rtc_usrate = rtc_mem_read(RTC_USRATE_POS); +} + +static void bbram_save() { + RTC_DBG("bbram_save\n"); + rtc_mem_write(RTC_TIME_MAGIC_POS , rtc_time_magic); + rtc_mem_write64(RTC_CYCLEOFFSETL_POS , rtc_cycleoffset); + rtc_mem_write(RTC_LASTSOURCEVAL_POS , rtc_lastsourceval); + rtc_mem_write(RTC_SOURCECYCLEUNITS_POS , rtc_sourcecycleunits); + rtc_mem_write(RTC_CALIBRATION_POS , rtc_calibration); + rtc_mem_write(RTC_SLEEPTOTALUS_POS , rtc_sleeptotalus); + rtc_mem_write(RTC_SLEEPTOTALCYCLES_POS , rtc_sleeptotalcycles); + rtc_mem_write(RTC_USRATE_POS , rtc_usrate); +} + +static inline uint64_t div2080(uint64_t n) { + n = n >> 5; + uint64_t t = n >> 7; + + uint64_t q = t + (t >> 1) + (t >> 2); + + q += q >> 3; + q += q >> 12; + q += q >> 24; + q += q >> 48; + + uint32_t r = (uint32_t) n - (uint32_t) q * 65; + + uint32_t off = (r - (r >> 6)) >> 6; + + q = q + off; + + return q; +} + +static uint32_t div1m(uint32_t *rem, unsigned long long n) { + // 0 -> 0002000000000000 sub + // 0 >> 5 - 0 >> 19 + [-1] -> 00020fffc0000000 add + // 0 >> 9 - 0 >> 6 + [-1] -> 000208ffc0000000 add + // 2 >> 12 - 2 >> 23 -> 000000208bea0080 sub + + uint64_t q1 = (n >> 5) - (n >> 19) + n; + uint64_t q2 = q1 + (n >> 9) - (n >> 6); + uint64_t q3 = (q2 >> 12) - (q2 >> 23); + + uint64_t q = q1 - n + q2 - q3; + + q = q >> 20; + + uint32_t r = (uint32_t) n - (uint32_t) q * 1000000; + + if (r >= 1000000) { + r -= 1000000; + q++; + } + + *rem = r; + + return q; +} + static inline uint64_t rtc_time_get_now_us_adjusted(); -static inline uint32_t rtc_time_get_magic(void) -{ - return rtc_mem_read(RTC_TIME_MAGIC_POS); -} +#define rtc_time_get_magic() rtc_time_magic static inline bool rtc_time_check_sleep_magic(void) { @@ -225,9 +321,11 @@ static inline bool rtc_time_check_magic(void) return (magic==RTC_TIME_MAGIC_FRC2 || magic==RTC_TIME_MAGIC_CCOUNT || magic==RTC_TIME_MAGIC_SLEEP); } -static inline void rtc_time_set_magic(uint32_t new_magic) +static void rtc_time_set_magic(uint32_t new_magic) { - rtc_mem_write(RTC_TIME_MAGIC_POS,new_magic); + RTC_DBG("Set magic to %08x\n", new_magic); + rtc_time_magic = new_magic; + bbram_save(); } static inline void rtc_time_set_sleep_magic(void) @@ -249,7 +347,7 @@ static inline void rtc_time_set_frc2_magic(void) static inline void rtc_time_unset_magic(void) { - rtc_mem_write(RTC_TIME_MAGIC_POS,0); + rtc_time_set_magic(0); } static inline uint32_t rtc_time_read_raw(void) @@ -281,16 +379,16 @@ static inline uint64_t rtc_time_source_offset(void) case RTC_TIME_MAGIC_FRC2: raw=rtc_time_read_raw_frc2(); break; default: return 0; // We are not in a position to offer time } - uint32_t multiplier=rtc_mem_read(RTC_SOURCECYCLEUNITS_POS); - uint32_t previous=rtc_mem_read(RTC_LASTSOURCEVAL_POS); + uint32_t multiplier = rtc_sourcecycleunits; + uint32_t previous = rtc_lastsourceval; if (raw> 32; + usadj = usadj + rtc_rateadjustedus; + + if (usoff > 1000000000 || force) { + rtc_usatlastrate = us; + rtc_rateadjustedus = usadj; + } + + return usadj; +} + +static inline void rtc_time_set_rate(int32_t rate) { + uint64_t now=rtc_time_get_now_us_adjusted(); + rtc_time_adjust_us_by_rate(now, 1); + rtc_usrate = rate; +} + +static inline int32_t rtc_time_get_rate() { + return rtc_usrate; +} + +static inline void rtc_time_tmrfn(void* arg) +{ + uint64_t now=rtc_time_get_now_us_adjusted(); + rtc_time_adjust_us_by_rate(now, 0); + rtc_time_source_offset(); +} + + static inline void rtc_time_gettimeofday(struct rtc_timeval* tv) { uint64_t now=rtc_time_get_now_us_adjusted(); - uint32_t sec=now/1000000; - uint32_t usec=now%1000000; - uint32_t to_adjust=rtc_mem_read(RTC_TODOFFSETUS_POS); + now = rtc_time_adjust_us_by_rate(now, 0); + uint32_t usec; + uint32_t sec = div1m(&usec, now); + uint32_t to_adjust=rtc_todoffsetus; if (to_adjust) { uint32_t us_passed=rtc_time_us_since_time_reached(sec,usec); @@ -680,9 +826,8 @@ static inline void rtc_time_gettimeofday(struct rtc_timeval* tv) adjust=to_adjust; to_adjust-=adjust; now-=adjust; - now/1000000; - now%1000000; - rtc_mem_write(RTC_TODOFFSETUS_POS,to_adjust); + sec = div1m(&usec, now); + rtc_todoffsetus = to_adjust; } } tv->tv_sec=sec; @@ -690,23 +835,24 @@ static inline void rtc_time_gettimeofday(struct rtc_timeval* tv) rtc_time_register_time_reached(sec,usec); } -static inline void rtc_time_settimeofday(const struct rtc_timeval* tv) +static void rtc_time_settimeofday(const struct rtc_timeval* tv) { if (!rtc_time_check_magic()) return; - uint32_t sleep_us=rtc_mem_read(RTC_SLEEPTOTALUS_POS); - uint32_t sleep_cycles=rtc_mem_read(RTC_SLEEPTOTALCYCLES_POS); + uint32_t sleep_us=rtc_sleeptotalus; + uint32_t sleep_cycles=rtc_sleeptotalcycles; // At this point, the CPU clock will definitely be at the default rate (nodemcu fully booted) uint64_t now_esp_us=rtc_time_get_now_us_adjusted(); + now_esp_us = rtc_time_adjust_us_by_rate(now_esp_us, 1); uint64_t now_ntp_us=((uint64_t)tv->tv_sec)*1000000+tv->tv_usec; int64_t diff_us=now_esp_us-now_ntp_us; // Store the *actual* time. uint64_t target_unitcycles=now_ntp_us*UNITCYCLE_MHZ; uint64_t sourcecycles=rtc_time_source_offset(); - rtc_mem_write64(RTC_CYCLEOFFSETL_POS,target_unitcycles-sourcecycles); + rtc_cycleoffset = target_unitcycles-sourcecycles; // calibrate sleep period based on difference between expected time and actual time if (sleep_us>0 && sleep_us<0xffffffff && @@ -715,11 +861,15 @@ static inline void rtc_time_settimeofday(const struct rtc_timeval* tv) uint64_t actual_sleep_us=sleep_us-diff_us; uint32_t cali=(actual_sleep_us<<12)/sleep_cycles; if (rtc_time_calibration_is_sane(cali)) - rtc_mem_write(RTC_CALIBRATION_POS,cali); + rtc_calibration = cali; } - rtc_mem_write(RTC_SLEEPTOTALUS_POS,0); - rtc_mem_write(RTC_SLEEPTOTALCYCLES_POS,0); + rtc_sleeptotalus = 0; + rtc_sleeptotalcycles = 0; + + rtc_usatlastrate = now_ntp_us; + rtc_rateadjustedus = now_ntp_us; + rtc_usrate = 0; // Deal with time adjustment if necessary if (diff_us>0) // Time went backwards. Avoid that.... @@ -730,7 +880,7 @@ static inline void rtc_time_settimeofday(const struct rtc_timeval* tv) } else diff_us=0; - rtc_mem_write(RTC_TODOFFSETUS_POS,diff_us); + rtc_todoffsetus = diff_us; uint32_t now_s=now_ntp_us/1000000; uint32_t now_us=now_ntp_us%1000000; diff --git a/app/modules/rtctime.c b/app/modules/rtctime.c index b6558e5f..f8656a6e 100644 --- a/app/modules/rtctime.c +++ b/app/modules/rtctime.c @@ -57,6 +57,11 @@ void rtctime_late_startup (void) rtc_time_switch_system (); } +void rtctime_adjust_rate (int rate) +{ + rtc_time_set_rate (rate); +} + void rtctime_gettimeofday (struct rtc_timeval *tv) { rtc_time_gettimeofday (tv); @@ -66,7 +71,9 @@ void rtctime_settimeofday (const struct rtc_timeval *tv) { if (!rtc_time_check_magic ()) rtc_time_prepare (); + int32_t rate = rtc_time_get_rate(); rtc_time_settimeofday (tv); + rtc_time_set_rate(rate); } bool rtctime_have_time (void) @@ -131,6 +138,9 @@ static int rtctime_set (lua_State *L) struct rtc_timeval tv = { sec, usec }; rtctime_settimeofday (&tv); + + if (lua_isnumber(L, 3)) + rtc_time_set_rate(lua_tonumber(L, 3)); return 0; } @@ -142,7 +152,8 @@ static int rtctime_get (lua_State *L) rtctime_gettimeofday (&tv); lua_pushnumber (L, tv.tv_sec); lua_pushnumber (L, tv.tv_usec); - return 2; + lua_pushnumber (L, rtc_time_get_rate()); + return 3; } static void do_sleep_opt (lua_State *L, int idx) diff --git a/app/modules/sntp.c b/app/modules/sntp.c index e1a06a16..f783eaa1 100644 --- a/app/modules/sntp.c +++ b/app/modules/sntp.c @@ -254,7 +254,7 @@ static void sntp_handle_result(lua_State *L) { int64_t f = ((state->best.delta * PLL_A) >> 32) + pll_increment; pll_increment += (state->best.delta * PLL_B) >> 32; sntp_dbg("f=%d, increment=%d\n", (int32_t) f, (int32_t) pll_increment); - //rtctime_adjust_rate((int32_t) f); + rtctime_adjust_rate((int32_t) f); } else { rtctime_settimeofday (&tv); } diff --git a/docs/en/modules/rtctime.md b/docs/en/modules/rtctime.md index 860e1138..3b04ede2 100644 --- a/docs/en/modules/rtctime.md +++ b/docs/en/modules/rtctime.md @@ -5,19 +5,28 @@ The rtctime module provides advanced timekeeping support for NodeMCU, including keeping time across deep sleep cycles (provided [`rtctime.dsleep()`](#rtctimedsleep) is used instead of [`node.dsleep()`](node.md#nodedsleep)). This can be used to significantly extend battery life on battery powered sensor nodes, as it is no longer necessary to fire up the RF module each wake-up in order to obtain an accurate timestamp. -This module is intended for use together with [NTP](https://en.wikipedia.org/wiki/Network_Time_Protocol) (Network Time Protocol) for keeping highly accurate real time at all times. Timestamps are available with microsecond precision, based on the Unix Epoch (1970/01/01 00:00:00). +This module is intended for use together with [NTP](https://en.wikipedia.org/wiki/Network_Time_Protocol) (Network Time Protocol) for keeping highly accurate real time at all times. Timestamps are available with microsecond precision, based on the Unix Epoch (1970/01/01 00:00:00). However, the accuracy is (in practice) no better then 1ms, and often worse than that. Time keeping on the ESP8266 is technically quite challenging. Despite being named [RTC](https://en.wikipedia.org/wiki/Real-time_clock), the RTC is not really a Real Time Clock in the normal sense of the word. While it does keep a counter ticking while the module is sleeping, the accuracy with which it does so is *highly* dependent on the temperature of the chip. Said temperature changes significantly between when the chip is running and when it is sleeping, meaning that any calibration performed while the chip is active becomes useless mere moments after the chip has gone to sleep. As such, calibration values need to be deduced across sleep cycles in order to enable accurate time keeping. This is one of the things this module does. Further complicating the matter of time keeping is that the ESP8266 operates on three different clock frequencies - 52MHz right at boot, 80MHz during regular operation, and 160MHz if boosted. This module goes to considerable length to take all of this into account to properly keep the time. -To enable this module, it needs to be given a reference time at least once (via [`rtctime.set()`](#rtctimeset)). For best accuracy it is recommended to provide a reference time twice, with the second time being after a deep sleep. +To enable this module, it needs to be given a reference time at least once (via [`rtctime.set()`](#rtctimeset)). For best accuracy it is recommended to provide reference +times at regular intervals. The [`sntp.sync()`](sntp.md#sntpsync) function has an easy way to do this. It is important that a reference time is provided at least twice, with the second time being after a deep sleep. Note that while the rtctime module can keep time across deep sleeps, it *will* lose the time if the module is unexpectedly reset. +This module can compensate for the underlying clock not running at exactly the required rate. The adjustment is in steps of 1 part in 2^32 (i.e. around 0.25 ppb). This adjustment +is done automatically if the [`sntp.sync()`](sntp.md#sntpsync) is called with the `autorepeat` flag set. The rate is settable using the [`set()`](#rtctimeset) function below. When the platform +is booted, it defaults to 0 (i.e. nominal). A sample of modules shows that the actual clock rate is temperature dependant, but is normally within 5ppm of the nominal rate. This translates to around 15 seconds per month. + +In the automatic update mode it can take a couple of hours for the clock rate to settle down to the correct value. After that, how well it tracks will depend on the amount +of variation in timestamps from the NTP servers. If they are close, then the time will track to within a millisecond or so. If they are further away (say 100ms round trip), then +time tracking is somewhat worse, but normally within 10ms. + !!! important - This module uses RTC memory slots 0-9, inclusive. As soon as [`rtctime.set()`](#rtctimeset) (or [`sntp.sync()`](sntp.md#sntpsync)) has been called these RTC memory slots will be used. +This module uses RTC memory slots 0-9, inclusive. As soon as [`rtctime.set()`](#rtctimeset) (or [`sntp.sync()`](sntp.md#sntpsync)) has been called these RTC memory slots will be used. This is a companion module to the [rtcmem](rtcmem.md) and [SNTP](sntp.md) modules. @@ -30,6 +39,8 @@ Puts the ESP8266 into deep sleep mode, like [`node.dsleep()`](node.md#nodedsleep - The time slept will generally be considerably more accurate than with [`node.dsleep()`](node.md#nodedsleep). - A sleep time of zero does not mean indefinite sleep, it is interpreted as a zero length sleep instead. +When the sleep timer expires, the platform is rebooted and the lua code is started with the `init.lua` file. The clock is set reasonably accurately. + #### Syntax `rtctime.dsleep(microseconds [, option])` @@ -107,14 +118,15 @@ Returns the current time. If current time is not available, zero is returned. none #### Returns -A two-value timestamp containing: +A three-value timestamp containing: - `sec` seconds since the Unix epoch - `usec` the microseconds part +- `rate` the current clock rate offset. This is an offset of `rate / 2^32` (where the nominal rate is 1). For example, a value of 4295 corresponds to 1 part per million. #### Example ```lua -sec, usec = rtctime.get() +sec, usec, rate = rtctime.get() ``` #### See also [`rtctime.set()`](#rtctimeset) @@ -128,11 +140,12 @@ It is highly recommended that the timestamp is obtained via NTP (see [SNTP modul Values very close to the epoch are not supported. This is a side effect of keeping the memory requirements as low as possible. Considering that it's no longer 1970, this is not considered a problem. #### Syntax -`rtctime.set(seconds, microseconds)` +`rtctime.set(seconds, microseconds, [rate])` #### Parameters - `seconds` the seconds part, counted from the Unix epoch - `microseconds` the microseconds part +- `rate` the rate in the same units as for `rtctime.get()`. The stored rate is not modified if not specified. #### Returns `nil` From e09e830d4b7d00265577292dc620e445641c7184 Mon Sep 17 00:00:00 2001 From: Johny Mattsson Date: Mon, 31 Jul 2017 22:12:34 +1000 Subject: [PATCH 24/27] Fixed alignment assumptions in SHA2 update. (#2034) Unaligned loads are a no-no on many architectures, the ESP8266 included. --- app/crypto/sha2.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/app/crypto/sha2.c b/app/crypto/sha2.c index 5fadfcc4..4ea1f925 100644 --- a/app/crypto/sha2.c +++ b/app/crypto/sha2.c @@ -32,6 +32,8 @@ * */ +/* ESP8266-specific tweaks by Johny Mattsson */ + #include "user_config.h" #ifdef SHA2_ENABLE @@ -491,7 +493,14 @@ void ICACHE_FLASH_ATTR SHA256_Update(SHA256_CTX* context, const sha2_byte *data, } while (len >= SHA256_BLOCK_LENGTH) { /* Process as many complete blocks as we can */ - SHA256_Transform(context, (sha2_word32*)data); + if ((int)data & (sizeof(sha2_word32)-1)) + { + // have to bounce via buffer, otherwise we'll hit unaligned load exception + MEMCPY_BCOPY(context->buffer, data, SHA256_BLOCK_LENGTH); + SHA256_Transform(context, (sha2_word32*)context->buffer); + } + else + SHA256_Transform(context, (sha2_word32*)data); context->bitcount += SHA256_BLOCK_LENGTH << 3; len -= SHA256_BLOCK_LENGTH; data += SHA256_BLOCK_LENGTH; @@ -782,7 +791,14 @@ void ICACHE_FLASH_ATTR SHA512_Update(SHA512_CTX* context, const sha2_byte *data, } while (len >= SHA512_BLOCK_LENGTH) { /* Process as many complete blocks as we can */ - SHA512_Transform(context, (sha2_word64*)data); + if ((int)data & (sizeof(sha2_word64)-1)) + { + // have to bounce via buffer, otherwise we'll hit unaligned load exception + MEMCPY_BCOPY(context->buffer, data, SHA512_BLOCK_LENGTH); + SHA512_Transform(context, (sha2_word64*)context->buffer); + } + else + SHA512_Transform(context, (sha2_word64*)data); ADDINC128(context->bitcount, SHA512_BLOCK_LENGTH << 3); len -= SHA512_BLOCK_LENGTH; data += SHA512_BLOCK_LENGTH; From 61562b45bd40bacd834d99423e023ec484cc665a Mon Sep 17 00:00:00 2001 From: wolfg Date: Sun, 6 Aug 2017 04:10:49 +0800 Subject: [PATCH 25/27] Fix typo in code sample (#2063) --- docs/en/modules/hmc5883l.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/modules/hmc5883l.md b/docs/en/modules/hmc5883l.md index 43f535f8..443a299f 100644 --- a/docs/en/modules/hmc5883l.md +++ b/docs/en/modules/hmc5883l.md @@ -20,7 +20,7 @@ temperature multiplied with 10 (integer) ```lua local sda, scl = 1, 2 i2c.setup(0, sda, scl, i2c.SLOW) -- call i2c.setup() only once -hmc58831.setup() +hmc5883l.setup() local x,y,z = hmc5883l.read() print(string.format("x = %d, y = %d, z = %d", x, y, z)) ``` From 295e640a7a96501447f7052d9577ceb6339acc26 Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Sat, 5 Aug 2017 14:05:26 -0700 Subject: [PATCH 26/27] Comment out pmsleep and timer_suspend options in user_config.h for master drop --- app/include/user_config.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/include/user_config.h b/app/include/user_config.h index f7c2a467..0160d840 100644 --- a/app/include/user_config.h +++ b/app/include/user_config.h @@ -114,8 +114,8 @@ extern void luaL_assertfail(const char *file, int line, const char *message); #define WIFI_SDK_EVENT_MONITOR_ENABLE #define WIFI_EVENT_MONITOR_DISCONNECT_REASON_LIST_ENABLE -#define ENABLE_TIMER_SUSPEND -#define PMSLEEP_ENABLE +////#define ENABLE_TIMER_SUSPEND +//#define PMSLEEP_ENABLE #define STRBUF_DEFAULT_INCREMENT 32 From c9e86218213b52a93713f7a6bf80c455ca3899ea Mon Sep 17 00:00:00 2001 From: dnc40085 Date: Sat, 5 Aug 2017 14:46:14 -0700 Subject: [PATCH 27/27] fix for travisCI --- app/modules/tmr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/modules/tmr.c b/app/modules/tmr.c index 19f416e4..c81c5365 100755 --- a/app/modules/tmr.c +++ b/app/modules/tmr.c @@ -428,7 +428,7 @@ static int tmr_create( lua_State *L ) { } -#if defined(SWTMR_DEBUG) +#if defined(ENABLE_TIMER_SUSPEND) && defined(SWTMR_DEBUG) static void tmr_printRegistry(lua_State* L){ swtmr_print_registry(); } @@ -463,7 +463,7 @@ static const LUA_REG_TYPE tmr_dyn_map[] = { { LNILKEY, LNILVAL } }; -#if defined(SWTMR_DEBUG) +#if defined(ENABLE_TIMER_SUSPEND) && defined(SWTMR_DEBUG) static const LUA_REG_TYPE tmr_dbg_map[] = { { LSTRKEY( "printRegistry" ), LFUNCVAL( tmr_printRegistry ) }, { LSTRKEY( "printSuspended" ), LFUNCVAL( tmr_printSuspended ) }, @@ -492,7 +492,7 @@ static const LUA_REG_TYPE tmr_map[] = { { LSTRKEY( "state" ), LFUNCVAL( tmr_state ) }, { LSTRKEY( "interval" ), LFUNCVAL( tmr_interval ) }, { LSTRKEY( "create" ), LFUNCVAL( tmr_create ) }, -#if defined(SWTMR_DEBUG) +#if defined(ENABLE_TIMER_SUSPEND) && defined(SWTMR_DEBUG) { LSTRKEY( "debug" ), LROVAL( tmr_dbg_map ) }, #endif { LSTRKEY( "ALARM_SINGLE" ), LNUMVAL( TIMER_MODE_SINGLE ) },