diff --git a/app/include/user_config.h b/app/include/user_config.h index 9f200daf..c786e909 100644 --- a/app/include/user_config.h +++ b/app/include/user_config.h @@ -76,6 +76,9 @@ extern void luaL_assertfail(const char *file, int line, const char *message); // maximum length of a filename #define FS_OBJ_NAME_LEN 31 +// maximum number of open files for SPIFFS +#define SPIFFS_MAX_OPEN_FILES 4 + // Uncomment this next line for fastest startup // It reduces the format time dramatically // #define SPIFFS_MAX_FILESYSTEM_SIZE 32768 diff --git a/app/modules/file.c b/app/modules/file.c index 559fb8e0..66d3eeb7 100644 --- a/app/modules/file.c +++ b/app/modules/file.c @@ -14,8 +14,12 @@ #define FILE_READ_CHUNK 1024 static int file_fd = 0; +static int file_fd_ref = LUA_NOREF; static int rtc_cb_ref = LUA_NOREF; +typedef struct _file_fd_ud { + int fd; +} file_fd_ud; static void table2tm( lua_State *L, vfs_time *tm ) { @@ -96,13 +100,48 @@ static int file_on(lua_State *L) // Lua: close() static int file_close( lua_State* L ) { - if(file_fd){ - vfs_close(file_fd); - file_fd = 0; + int need_pop = FALSE; + file_fd_ud *ud; + + if (lua_type( L, 1 ) != LUA_TUSERDATA) { + // fall back to last opened file + if (file_fd_ref != LUA_NOREF) { + lua_rawgeti( L, LUA_REGISTRYINDEX, file_fd_ref ); + // top of stack is now default file descriptor + ud = (file_fd_ud *)luaL_checkudata(L, -1, "file.obj"); + lua_pop( L, 1 ); + } else { + // no default file currently opened + return 0; + } + } else { + ud = (file_fd_ud *)luaL_checkudata(L, 1, "file.obj"); + } + + // unref default file descriptor + luaL_unref( L, LUA_REGISTRYINDEX, file_fd_ref ); + file_fd_ref = LUA_NOREF; + + if(ud->fd){ + vfs_close(ud->fd); + // mark as closed + ud->fd = 0; } return 0; } +static int file_obj_free( lua_State *L ) +{ + file_fd_ud *ud = (file_fd_ud *)luaL_checkudata(L, 1, "file.obj"); + if (ud->fd) { + // close file if it's still open + vfs_close(ud->fd); + ud->fd = 0; + } + + return 0; +} + // Lua: format() static int file_format( lua_State* L ) { @@ -135,10 +174,10 @@ static int file_fscfg (lua_State *L) static int file_open( lua_State* L ) { size_t len; - if(file_fd){ - vfs_close(file_fd); - file_fd = 0; - } + + // unref last file descriptor to allow gc'ing if not kept by user script + luaL_unref( L, LUA_REGISTRYINDEX, file_fd_ref ); + file_fd_ref = LUA_NOREF; const char *fname = luaL_checklstring( L, 1, &len ); const char *basename = vfs_basename( fname ); @@ -151,7 +190,14 @@ static int file_open( lua_State* L ) if(!file_fd){ lua_pushnil(L); } else { - lua_pushboolean(L, 1); + file_fd_ud *ud = (file_fd_ud *) lua_newuserdata( L, sizeof( file_fd_ud ) ); + ud->fd = file_fd; + luaL_getmetatable( L, "file.obj" ); + lua_setmetatable( L, -2 ); + + // store reference to opened file + lua_pushvalue( L, -1 ); + file_fd_ref = luaL_ref( L, LUA_REGISTRYINDEX ); } return 1; } @@ -175,19 +221,36 @@ static int file_list( lua_State* L ) return 0; } -static int file_seek (lua_State *L) +static int get_file_obj( lua_State *L, int *argpos ) { + if (lua_type( L, 1 ) == LUA_TUSERDATA) { + file_fd_ud *ud = (file_fd_ud *)luaL_checkudata(L, 1, "file.obj"); + *argpos = 2; + return ud->fd; + } else { + *argpos = 1; + return file_fd; + } +} + +#define GET_FILE_OBJ int argpos; \ + int fd = get_file_obj( L, &argpos ); + +static int file_seek (lua_State *L) +{ + GET_FILE_OBJ; + static const int mode[] = {VFS_SEEK_SET, VFS_SEEK_CUR, VFS_SEEK_END}; static const char *const modenames[] = {"set", "cur", "end", NULL}; - if(!file_fd) + if(!fd) return luaL_error(L, "open a file first"); - int op = luaL_checkoption(L, 1, "cur", modenames); - long offset = luaL_optlong(L, 2, 0); - op = vfs_lseek(file_fd, offset, mode[op]); + int op = luaL_checkoption(L, argpos, "cur", modenames); + long offset = luaL_optlong(L, ++argpos, 0); + op = vfs_lseek(fd, offset, mode[op]); if (op < 0) lua_pushnil(L); /* error */ else - lua_pushinteger(L, vfs_tell(file_fd)); + lua_pushinteger(L, vfs_tell(fd)); return 1; } @@ -215,7 +278,6 @@ static int file_remove( lua_State* L ) const char *fname = luaL_checklstring( L, 1, &len ); const char *basename = vfs_basename( fname ); luaL_argcheck(L, c_strlen(basename) <= FS_OBJ_NAME_LEN && c_strlen(fname) == len, 1, "filename invalid"); - file_close(L); vfs_remove((char *)fname); return 0; } @@ -223,9 +285,11 @@ static int file_remove( lua_State* L ) // Lua: flush() static int file_flush( lua_State* L ) { - if(!file_fd) + GET_FILE_OBJ; + + if(!fd) return luaL_error(L, "open a file first"); - if(vfs_flush(file_fd) == 0) + if(vfs_flush(fd) == 0) lua_pushboolean(L, 1); else lua_pushnil(L); @@ -236,10 +300,6 @@ static int file_flush( lua_State* L ) static int file_rename( lua_State* L ) { size_t len; - if(file_fd){ - vfs_close(file_fd); - file_fd = 0; - } const char *oldname = luaL_checklstring( L, 1, &len ); const char *basename = vfs_basename( oldname ); @@ -258,12 +318,14 @@ static int file_rename( lua_State* L ) } // g_read() -static int file_g_read( lua_State* L, int n, int16_t end_char ) +static int file_g_read( lua_State* L, int n, int16_t end_char, int fd ) { static char *heap_mem = NULL; // free leftover memory - if (heap_mem) + if (heap_mem) { luaM_free(L, heap_mem); + heap_mem = NULL; + } if(n <= 0) n = FILE_READ_CHUNK; @@ -271,7 +333,8 @@ static int file_g_read( lua_State* L, int n, int16_t end_char ) if(end_char < 0 || end_char >255) end_char = EOF; - if(!file_fd) + + if(!fd) return luaL_error(L, "open a file first"); char *p; @@ -285,7 +348,7 @@ static int file_g_read( lua_State* L, int n, int16_t end_char ) p = alloca(n); } - n = vfs_read(file_fd, p, n); + n = vfs_read(fd, p, n); // bypass search if no end character provided for (i = end_char != EOF ? 0 : n; i < n; ++i) if (p[i] == end_char) @@ -302,7 +365,7 @@ static int file_g_read( lua_State* L, int n, int16_t end_char ) return 0; } - vfs_lseek(file_fd, -(n - i), VFS_SEEK_CUR); + vfs_lseek(fd, -(n - i), VFS_SEEK_CUR); lua_pushlstring(L, p, i); if (heap_mem) { luaM_free(L, heap_mem); @@ -320,36 +383,43 @@ static int file_read( lua_State* L ) unsigned need_len = FILE_READ_CHUNK; int16_t end_char = EOF; size_t el; - if( lua_type( L, 1 ) == LUA_TNUMBER ) + + GET_FILE_OBJ; + + if( lua_type( L, argpos ) == LUA_TNUMBER ) { - need_len = ( unsigned )luaL_checkinteger( L, 1 ); + need_len = ( unsigned )luaL_checkinteger( L, argpos ); } - else if(lua_isstring(L, 1)) + else if(lua_isstring(L, argpos)) { - const char *end = luaL_checklstring( L, 1, &el ); + const char *end = luaL_checklstring( L, argpos, &el ); if(el!=1){ return luaL_error( L, "wrong arg range" ); } end_char = (int16_t)end[0]; } - return file_g_read(L, need_len, end_char); + return file_g_read(L, need_len, end_char, fd); } // Lua: readline() static int file_readline( lua_State* L ) { - return file_g_read(L, FILE_READ_CHUNK, '\n'); + GET_FILE_OBJ; + + return file_g_read(L, LUAL_BUFFERSIZE, '\n', fd); } // Lua: write("string") static int file_write( lua_State* L ) { - if(!file_fd) + GET_FILE_OBJ; + + if(!fd) return luaL_error(L, "open a file first"); size_t l, rl; - const char *s = luaL_checklstring(L, 1, &l); - rl = vfs_write(file_fd, s, l); + const char *s = luaL_checklstring(L, argpos, &l); + rl = vfs_write(fd, s, l); if(rl==l) lua_pushboolean(L, 1); else @@ -360,13 +430,15 @@ static int file_write( lua_State* L ) // Lua: writeline("string") static int file_writeline( lua_State* L ) { - if(!file_fd) + GET_FILE_OBJ; + + if(!fd) return luaL_error(L, "open a file first"); size_t l, rl; - const char *s = luaL_checklstring(L, 1, &l); - rl = vfs_write(file_fd, s, l); + const char *s = luaL_checklstring(L, argpos, &l); + rl = vfs_write(fd, s, l); if(rl==l){ - rl = vfs_write(file_fd, "\n", 1); + rl = vfs_write(fd, "\n", 1); if(rl==1) lua_pushboolean(L, 1); else @@ -441,6 +513,20 @@ static int file_vol_umount( lua_State *L ) } +static const LUA_REG_TYPE file_obj_map[] = +{ + { LSTRKEY( "close" ), LFUNCVAL( file_close ) }, + { LSTRKEY( "read" ), LFUNCVAL( file_read ) }, + { LSTRKEY( "readline" ), LFUNCVAL( file_readline ) }, + { LSTRKEY( "write" ), LFUNCVAL( file_write ) }, + { LSTRKEY( "writeline" ), LFUNCVAL( file_writeline ) }, + { LSTRKEY( "seek" ), LFUNCVAL( file_seek ) }, + { LSTRKEY( "flush" ), LFUNCVAL( file_flush ) }, + { LSTRKEY( "__gc" ), LFUNCVAL( file_obj_free ) }, + { LSTRKEY( "__index" ), LROVAL( file_obj_map ) }, + { LNILKEY, LNILVAL } +}; + static const LUA_REG_TYPE file_vol_map[] = { { LSTRKEY( "umount" ), LFUNCVAL( file_vol_umount )}, @@ -480,6 +566,7 @@ static const LUA_REG_TYPE file_map[] = { int luaopen_file( lua_State *L ) { luaL_rometatable( L, "file.vol", (void *)file_vol_map ); + luaL_rometatable( L, "file.obj", (void *)file_obj_map ); return 0; } diff --git a/app/spiffs/spiffs.c b/app/spiffs/spiffs.c index 093534fc..1cf211dd 100644 --- a/app/spiffs/spiffs.c +++ b/app/spiffs/spiffs.c @@ -2,6 +2,8 @@ #include "platform.h" #include "spiffs.h" +#include "spiffs_nucleus.h" + spiffs fs; #define LOG_PAGE_SIZE 256 @@ -10,9 +12,9 @@ spiffs fs; #define MIN_BLOCKS_FS 4 static u8_t spiffs_work_buf[LOG_PAGE_SIZE*2]; -static u8_t spiffs_fds[32*4]; +static u8_t spiffs_fds[sizeof(spiffs_fd) * SPIFFS_MAX_OPEN_FILES]; #if SPIFFS_CACHE -static u8_t spiffs_cache[(LOG_PAGE_SIZE+32)*2]; +static u8_t myspiffs_cache[(LOG_PAGE_SIZE+32)*2]; #endif static s32_t my_spiffs_read(u32_t addr, u32_t size, u8_t *dst) { @@ -168,8 +170,8 @@ static bool myspiffs_mount_internal(bool force_mount) { spiffs_fds, sizeof(spiffs_fds), #if SPIFFS_CACHE - spiffs_cache, - sizeof(spiffs_cache), + myspiffs_cache, + sizeof(myspiffs_cache), #else 0, 0, #endif diff --git a/docs/en/modules/file.md b/docs/en/modules/file.md index b58d5fb2..9cbfa93e 100644 --- a/docs/en/modules/file.md +++ b/docs/en/modules/file.md @@ -7,8 +7,6 @@ The file module provides access to the file system and its individual files. The file system is a flat file system, with no notion of subdirectories/folders. -Only one file can be open at any given time. - Besides the SPIFFS file system on internal flash, this module can also access FAT partitions on an external SD card is [FatFS is enabled](../sdcard.md). ```lua @@ -43,30 +41,6 @@ Current directory defaults to the root of internal SPIFFS (`/FLASH`) after syste #### Returns `true` on success, `false` otherwise -## file.close() - -Closes the open file, if any. - -#### Syntax -`file.close()` - -#### Parameters -none - -#### Returns -`nil` - -#### Example -```lua --- open 'init.lua', print the first line. -if file.open("init.lua", "r") then - print(file.readline()) - file.close() -end -``` -#### See also -[`file.open()`](#fileopen) - ## file.exists() Determines whether the specified file exists. @@ -95,34 +69,6 @@ end #### See also [`file.list()`](#filelist) -## file.flush() - -Flushes any pending writes to the file system, ensuring no data is lost on a restart. Closing the open file using [`file.close()`](#fileclose) performs an implicit flush as well. - -#### Syntax -`file.flush()` - -#### Parameters -none - -#### Returns -`nil` - -#### Example -```lua --- open 'init.lua' in 'a+' mode -if file.open("init.lua", "a+") then - -- write 'foo bar' to the end of the file - file.write('foo bar') - file.flush() - -- write 'baz' too - file.write('baz') - file.close() -end -``` -#### See also -[`file.close()`](#fileclose) - ## file.format() Format the file system. Completely erases any existing file system and writes a new one. Depending on the size of the flash chip in the ESP, this may take several seconds. @@ -280,9 +226,9 @@ When done with the file, it must be closed using `file.close()`. - "a+": append update mode, previous data is preserved, writing is only allowed at the end of file #### Returns -`nil` if file not opened, or not exists (read modes). `true` if file opened ok. +file object if file opened ok. `nil` if file not opened, or not exists (read modes). -#### Example +#### Example (basic model) ```lua -- open 'init.lua', print the first line. if file.open("init.lua", "r") then @@ -290,74 +236,19 @@ if file.open("init.lua", "r") then file.close() end ``` -#### See also -- [`file.close()`](#fileclose) -- [`file.readline()`](#filereadline) - -## file.read() - -Read content from the open file. - -!!! note - - The function temporarily allocates 2 * (number of requested bytes) on the heap for buffering and processing the read data. Default chunk size (`FILE_READ_CHUNK`) is 1024 bytes and is regarded to be safe. Pushing this by 4x or more can cause heap overflows depending on the application. Consider this when selecting a value for parameter `n_or_char`. - -#### Syntax -`file.read([n_or_char])` - -#### Parameters -- `n_or_char`: - - if nothing passed in, then read up to `FILE_READ_CHUNK` bytes or the entire file (whichever is smaller). - - if passed a number `n`, then read up to `n` bytes or the entire file (whichever is smaller). - - if passed a string containing the single character `char`, then read until `char` appears next in the file, `FILE_READ_CHUNK` bytes have been read, or EOF is reached. - -#### Returns -File content as a string, or nil when EOF - -#### Example +#### Example (object model) ```lua --- print the first line of 'init.lua' -if file.open("init.lua", "r") then - print(file.read('\n')) - file.close() -end - --- print the first 5 bytes of 'init.lua' -if file.open("init.lua", "r") then - print(file.read(5)) - file.close() +-- open 'init.lua', print the first line. +fd = file.open("init.lua", "r") +if fd then + print(fd:readline()) + fd:close(); fd = nil end ``` #### See also -- [`file.open()`](#fileopen) -- [`file.readline()`](#filereadline) - -## file.readline() - -Read the next line from the open file. Lines are defined as zero or more bytes ending with a EOL ('\n') byte. If the next line is longer than 1024, this function only returns the first 1024 bytes. - -#### Syntax -`file.readline()` - -#### Parameters -none - -#### Returns -File content in string, line by line, including EOL('\n'). Return `nil` when EOF. - -#### Example -```lua --- print the first line of 'init.lua' -if file.open("init.lua", "r") then - print(file.readline()) - file.close() -end -``` -#### See also -- [`file.open()`](#fileopen) - [`file.close()`](#fileclose) -- [`file.read()`](#filereade) +- [`file.readline()`](#filereadline) ## file.remove() @@ -402,12 +293,188 @@ Renames a file. If a file is currently open, it will be closed first. file.rename("temp.lua","init.lua") ``` +# File access functions + +The `file` module provides several functions to access the content of a file after it has been opened with [`file.open()`](#fileopen). They can be used as part of a basic model or an object model: + +## Basic model +In the basic model there is max one file opened at a time. The file access functions operate on this file per default. If another file is opened, the previous default file needs to be closed beforehand. + +```lua +-- open 'init.lua', print the first line. +if file.open("init.lua", "r") then + print(file.readline()) + file.close() +end +``` + +## Object model +Files are represented by file objects which are created by `file.open()`. File access functions are available as methods of this object, and multiple file objects can coexist. + +```lua +src = file.open("init.lua", "r") +if src then + dest = file.open("copy.lua", "w") + if dest then + local line + repeat + line = src:read() + if line then + dest:write(line) + end + until line == nil + dest:close(); dest = nil + end + src:close(); dest = nil +end +``` + +!!! Attention + + It is recommended to use only one single model within the application. Concurrent use of both models can yield unpredictable behavior: Closing the default file from basic model will also close the correspoding file object. Closing a file from object model will also close the default file if they are the same file. + +!!! Note + + The maximum number of open files on SPIFFS is determined at compile time by `SPIFFS_MAX_OPEN_FILES` in `user_config.h`. + +## file.close() +## file.obj:close() + +Closes the open file, if any. + +#### Syntax +`file.close()` + +`fd:close()` + +#### Parameters +none + +#### Returns +`nil` + +#### See also +[`file.open()`](#fileopen) + +## file.flush() +## file.obj:flush() + +Flushes any pending writes to the file system, ensuring no data is lost on a restart. Closing the open file using [`file.close()` / `fd:close()`](#fileclose) performs an implicit flush as well. + +#### Syntax +`file.flush()` + +`fd:flush()` + +#### Parameters +none + +#### Returns +`nil` + +#### Example (basic model) +```lua +-- open 'init.lua' in 'a+' mode +if file.open("init.lua", "a+") then + -- write 'foo bar' to the end of the file + file.write('foo bar') + file.flush() + -- write 'baz' too + file.write('baz') + file.close() +end +``` + +#### See also +[`file.close()` / `file.obj:close()`](#fileclose) + +## file.read() +## file.obj:read() + +Read content from the open file. + +!!! note + + The function temporarily allocates 2 * (number of requested bytes) on the heap for buffering and processing the read data. Default chunk size (`FILE_READ_CHUNK`) is 1024 bytes and is regarded to be safe. Pushing this by 4x or more can cause heap overflows depending on the application. Consider this when selecting a value for parameter `n_or_char`. + +#### Syntax +`file.read([n_or_char])` + +`fd:read([n_or_char])` + +#### Parameters +- `n_or_char`: + - if nothing passed in, then read up to `FILE_READ_CHUNK` bytes or the entire file (whichever is smaller). + - if passed a number `n`, then read up to `n` bytes or the entire file (whichever is smaller). + - if passed a string containing the single character `char`, then read until `char` appears next in the file, `FILE_READ_CHUNK` bytes have been read, or EOF is reached. + +#### Returns +File content as a string, or nil when EOF + +#### Example (basic model) +```lua +-- print the first line of 'init.lua' +if file.open("init.lua", "r") then + print(file.read('\n')) + file.close() +end +``` + +#### Example (object model) +```lua +-- print the first 5 bytes of 'init.lua' +fd = file.open("init.lua", "r") +if fd then + print(fd:read(5)) + fd:close(); fd = nil +end +``` + +#### See also +- [`file.open()`](#fileopen) +- [`file.readline()` / `file.obj:readline()`](#filereadline) + +## file.readline() +## file.obj:readline() + +Read the next line from the open file. Lines are defined as zero or more bytes ending with a EOL ('\n') byte. If the next line is longer than 1024, this function only returns the first 1024 bytes. + +#### Syntax +`file.readline()` + +`fd:readline()` + +#### Parameters +none + +#### Returns +File content in string, line by line, including EOL('\n'). Return `nil` when EOF. + +#### Example (basic model) +```lua +-- print the first line of 'init.lua' +if file.open("init.lua", "r") then + print(file.readline()) + file.close() +end +``` + +#### See also +- [`file.open()`](#fileopen) +- [`file.close()` / `file.obj:close()`](#fileclose) +- [`file.read()` / `file.obj:read()`](#fileread) + + ## file.seek() +## file.obj:seek() + Sets and gets the file position, measured from the beginning of the file, to the position given by offset plus a base specified by the string whence. #### Syntax `file.seek([whence [, offset]])` +`fd:seek([whence [, offset]])` + #### Parameters - `whence` - "set": base is position 0 (beginning of the file) @@ -420,7 +487,7 @@ If no parameters are given, the function simply returns the current file offset. #### Returns the resulting file position, or `nil` on error -#### Example +#### Example (basic model) ```lua if file.open("init.lua", "r") then -- skip the first 5 bytes of the file @@ -433,19 +500,22 @@ end [`file.open()`](#fileopen) ## file.write() +## file.obj:write() Write a string to the open file. #### Syntax `file.write(string)` +`fd:write(string)` + #### Parameters `string` content to be write to file #### Returns `true` if the write is ok, `nil` on error -#### Example +#### Example (basic model) ```lua -- open 'init.lua' in 'a+' mode if file.open("init.lua", "a+") then @@ -455,24 +525,38 @@ if file.open("init.lua", "a+") then end ``` +#### Example (object model) +```lua +-- open 'init.lua' in 'a+' mode +fd = file.open("init.lua", "a+") +if fd then + -- write 'foo bar' to the end of the file + fd:write('foo bar') + fd:close() +end +``` + #### See also - [`file.open()`](#fileopen) -- [`file.writeline()`](#filewriteline) +- [`file.writeline()` / `file.obj:writeline()`](#filewriteline) ## file.writeline() +## file.obj:writeline() Write a string to the open file and append '\n' at the end. #### Syntax `file.writeline(string)` +`fd:writeline(string)` + #### Parameters `string` content to be write to file #### Returns `true` if write ok, `nil` on error -#### Example +#### Example (basic model) ```lua -- open 'init.lua' in 'a+' mode if file.open("init.lua", "a+") then @@ -484,4 +568,4 @@ end #### See also - [`file.open()`](#fileopen) -- [`file.readline()`](#filereadline) +- [`file.readline()` / `file.obj:readline()`](#filereadline)