MirrorSave
/
nodemcu-firmware
mirror of https://github.com/nodemcu/nodemcu-firmware.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for external modules (#3100)

This commit is contained in:
Javier Peletier 2020-07-27 19:08:19 +02:00 committed by GitHub
parent cd53865c78
commit 084d6cabc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 668 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -8,9 +8,10 @@ build/
app/ app/
components/*/.output/ components/*/.output/
tools/toolchains tools/toolchains
extmods.ini
#ignore Eclipse project files #ignore Eclipse project files
.cproject .cproject
.project .project
.settings/ .settings/
.vscode/** .vscode/**

9
Makefile
View File

@ -12,6 +12,9 @@ TOOLCHAIN_RELEASES_BASEURL:=https://github.com/jmattsson/esp-toolchains/releases
TOOLCHAIN_VERSION:=20181106.1 TOOLCHAIN_VERSION:=20181106.1
PLATFORM:=linux-$(shell uname --machine) PLATFORM:=linux-$(shell uname --machine)
## Directory to place external modules:
export EXTRA_COMPONENT_DIRS:=$(THIS_DIR)/components/modules/external
ESP32_BIN:=$(THIS_DIR)/tools/toolchains/esp32-$(PLATFORM)-$(TOOLCHAIN_VERSION)/bin ESP32_BIN:=$(THIS_DIR)/tools/toolchains/esp32-$(PLATFORM)-$(TOOLCHAIN_VERSION)/bin
ESP32_GCC:=$(ESP32_BIN)/xtensa-esp32-elf-gcc ESP32_GCC:=$(ESP32_BIN)/xtensa-esp32-elf-gcc
ESP32_TOOLCHAIN_DL:=$(THIS_DIR)/cache/toolchain-esp32-$(PLATFORM)-$(TOOLCHAIN_VERSION).tar.xz ESP32_TOOLCHAIN_DL:=$(THIS_DIR)/cache/toolchain-esp32-$(PLATFORM)-$(TOOLCHAIN_VERSION).tar.xz
@ -61,3 +64,9 @@ include $(IDF_PATH)/make/project.mk
CC:=$(CC) $(BASIC_TYPES) -D__ESP32__ $(MORE_CFLAGS) CC:=$(CC) $(BASIC_TYPES) -D__ESP32__ $(MORE_CFLAGS)
endif endif
extmod-update:
@tools/extmod/extmod.sh update
extmod-clean:
@tools/extmod/extmod.sh clean

22
components/base_nodemcu/include/module.h
View File

@ -70,3 +70,25 @@
luaR_entry MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(CONFIG_LUA_MODULE_,cfgname))) \ luaR_entry MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(CONFIG_LUA_MODULE_,cfgname))) \
= {luaname, LRO_ROVAL(map ## _map)} = {luaname, LRO_ROVAL(map ## _map)}
#endif #endif
// helper stringing macros
#define xstr(s) str(s)
#define str(s) #s
// EXTMODNAME is injected by the generated component.mk
#ifdef EXTMODNAME
#define MODNAME xstr(EXTMODNAME)
#else
#define MODNAME "module"
#endif
// use NODEMCU_MODULE_METATABLE() to generate a unique metatable name for your objects:
#define NODEMCU_MODULE_METATABLE() MODULE_EXPAND_(MODNAME xstr(__COUNTER__))
// NODEMCU_MODULE_STD() defines the entry points for an external module:
#define NODEMCU_MODULE_STD() \
static const LOCK_IN_SECTION(libs) \
luaR_entry lua_lib_module = {MODNAME, LRO_FUNCVAL(module_init)}; \
const LOCK_IN_SECTION(rotable) \
luaR_entry MODULE_EXPAND_PASTE_(EXTMODNAME, _entry) = {MODNAME, LRO_ROVAL(module_map)};

1
components/modules/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
external/

134
docs/modules/extmods.md Normal file
View File

@ -0,0 +1,134 @@
# External modules. Plugging your own C modules. **BETA**
Note: this feature is still in beta. Configuration files / API may change.
In order to make the most of NodeMCU, you will undoubtedly have to connect your ESP device to many different hardware modules, which will require you to write driver code in C and expose a Lua interface.
To make this easy, we have come up with the concept of "external modules". External modules allow you to refer to an external git repository containing the module, which will be downloaded/updated and built along the firmware. It is similar to git submodules, but without the complexity and having to alter the parent repository, while also adapted to the compilation requirements of NodeMCU and Lua.
## How to use external modules:
To use external modules, simply create an `extmods.ini` file in the repository root, with the following syntax:
```ini
[lua_mount_point]
url=<git repo URL>
ref=<branch name, tag or commit id>
...
```
Where:
* **lua_mount_point**: Name you want the referenced module to have in Lua.
* **url**: Repository URL where to fetch the code from
* **ref**: branch, tag or commit hash of the version of the module you want to pull in.
* **disabled**: (optional) Whether or not to actually compile the module (see below)
For example:
```ini
[helloworld]
url=git@github.com:espore-ide/nodemcu-module-helloworld.git
ref=master
```
You can add further sections to `extmods.ini`, one for each module you want to add. Once this file is ready, run the update command:
```shell
make extmod-update
```
This will download or update the modules to the external modules directory, `components/modules/external`.
You can now compile the firmware with `make` as usual. The build system will find your external modules and compile them along the core modules.
After this is flashed to the device, the module in the example will be available in lua as `helloworld`.
### Updating to the latest code
If your external module entry in `extmods.ini` points to a branch, e.g., `master`, you can update your local version to the latest code anytime by simply running `make extmod-update`.
### Temporarily disabling an external module
If you want to stop compiling and including a module in the build for any reason, without having to remove the entry from `extmods.ini`, simply add a `disabled=true` entry in the module section and run `make extmod-update`.
Example:
```ini
[helloworld]
url=https://github.com/espore-ide/nodemcu-module-helloworld.git
ref=master
disabled=true
```
### Mounting different module versions
Provided the module is well written (no global variables, etc), it is even possible to easily mount different versions of the same module simultaneously for testing:
```ini
[helloworld]
url=https://github.com/espore-ide/nodemcu-module-helloworld.git
ref=master
[helloworld_dev]
url=https://github.com/espore-ide/nodemcu-module-helloworld.git
ref=dev
```
Note that the second one points to a different branch, `dev`. Both modules will be visible in Lua under `helloworld` and `helloworld_dev` respectively.
## How to write external modules:
To write your own external module do the following:
1. Create an empty repository in your favorite remote, GitHub, BitBucket, GitLab, etc, or fork the helloworld example.
2. Create an entry in `extmods.ini` as explained above, with the `url=` key pointing to your repository. For modules that you author, it is recommended to use an updateable git URL in SSH format, such as `git@github.com:espore-ide/nodemcu-module-helloworld.git`.
3. Run `make extmod-update`
You can now change to `components/modules/external/your_module` and begin work. Since that is your own repository, you can work normally, commit, create branches, etc.
### External module scaffolding
External modules must follow a specific structure to declare the module in C. Please refer to the [helloworld.c](https://github.com/nodemcu/nodemcu-firmware/blob/dev-esp32/tools/example/helloworld.c) example, or use it as a template. In particular:
1. Include `module.h`
2. Define a Module Function Map with name `module`
3. Define a `module_init` function
4. Include the module lua entries by adding a call to the `NODEMCU_MODULE_STD` macro
Here is a bare minimum module:
```c
#include "module.h"
// Module function map
LROT_BEGIN(module)
/* module-level functions go here*/
LROT_END(module, NULL, 0)
// module_init is invoked on device startup
static int module_init(lua_State* L) {
// initialize your module, register metatables, etc
return 0;
}
NODEMCU_MODULE_STD(); // define Lua entries
```
For a full example module boilerplate, check the [helloworld.c](https://github.com/nodemcu/nodemcu-firmware/blob/dev-esp32/tools/example/helloworld.c) file.
### Special makefile or compilation requirements
If your module has special makefile requirements, you can create a `module.mk` file at the root of your module repository. This will be executed during compilation.
### What is this "component.mk" file that appeared in my repo when running `make extmod-update` ?
This file is ignored by your repository. Do not edit or check it in!. This file contains the necessary information to compile your module along with the others.
Note that you don't even need to add this file to your `.gitignore`, since the `make extmod-update` operation configures your local copy to ignore it (via `.git/info/exclude`).
## Further work:
* Support for per-module menuconfig (`Kconfig`). This is actually possible already, but need to work around potential config variable collisions in case two module authors happen to choose the same variable names.
* Module registry: Create an official repository of known external modules.
* Move all non-essential and specific hardware-related C modules currently in the NodeMCU repository to external modules, each in their own repository.
* Create the necessary scaffolding to facilitate writing modules that will work both in ESP8266 and ESP32.
* Port this work to the ESP8266 branch. Mostly, the scripts that make this possible could work in both branches directly.

1
mkdocs.yml
View File

@ -32,6 +32,7 @@ pages:
- Extension Developer FAQ: 'extn-developer-faq.md' - Extension Developer FAQ: 'extn-developer-faq.md'
- Whitepapers: - Whitepapers:
- Filesystem on SD card: 'sdcard.md' - Filesystem on SD card: 'sdcard.md'
- Writing external C modules: 'modules/extmods.md'
- C Modules: - C Modules:
- 'adc': 'modules/adc.md' - 'adc': 'modules/adc.md'
- 'bit': 'modules/bit.md' - 'bit': 'modules/bit.md'

16
tools/extmod/component.mk.template Normal file
View File

@ -0,0 +1,16 @@
# This file is autogenerated! DO NOT EDIT!!
# To add special makefile directives, create a module.mk file in your repo
# Make sure you have this file in your .gitignore to avoid checking it in your module repo
MODNAME="%%MODNAME%%"
MODULE_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
COMPONENT_ADD_LDFLAGS=-u $(MODNAME)_entry -l$(MODNAME)
CFLAGS += \
-DEXTMODNAME=$(MODNAME) \
-Werror=unused-function \
-Werror=unused-but-set-variable \
-Werror=unused-variable
-include $(MODULE_DIR)module.mk

61
tools/extmod/example/helloworld.c Normal file
View File

@ -0,0 +1,61 @@
// Helloworld sample module
#include "esp_log.h"
#include "lauxlib.h"
#include "lnodeaux.h"
#include "module.h"
static const char* HELLOWORLD_METATABLE = NODEMCU_MODULE_METATABLE();
// hello_context_t struct contains information to wrap a "hello world object"
typedef struct {
char* my_name; // pointer to the greeter's name
} hello_context_t;
// Lua: helloworldobj:hello(text)
static int helloworld_hello(lua_State* L) {
hello_context_t* context = (hello_context_t*)luaL_checkudata(L, 1, HELLOWORLD_METATABLE);
printf("Hello, %s: %s\n", context->my_name, luaL_optstring(L, 2, "How are you?"));
return 0;
}
// helloworld_delete is called on garbage collection
static int helloworld_delete(lua_State* L) {
hello_context_t* context = (hello_context_t*)luaL_checkudata(L, 1, HELLOWORLD_METATABLE);
printf("Helloworld object with name '%s' garbage collected\n", context->my_name);
luaX_free_string(L, context->my_name);
return 0;
}
// Lua: modulename.new(string)
static int helloworld_new(lua_State* L) {
//create a new lua userdata object and initialize to 0.
hello_context_t* context = (hello_context_t*)lua_newuserdata(L, sizeof(hello_context_t));
context->my_name = luaX_alloc_string(L, 1, 100);
luaL_getmetatable(L, HELLOWORLD_METATABLE);
lua_setmetatable(L, -2);
return 1; //one object returned, the helloworld context wrapped in a lua userdata object
}
// object function map:
LROT_BEGIN(helloworld_metatable)
LROT_FUNCENTRY(hello, helloworld_hello)
LROT_FUNCENTRY(__gc, helloworld_delete)
LROT_TABENTRY(__index, helloworld_metatable)
LROT_END(helloworld_metatable, NULL, 0)
// Module function map
LROT_BEGIN(module)
LROT_FUNCENTRY(new, helloworld_new)
LROT_END(module, NULL, 0)
// module_init is invoked on device startup
static int module_init(lua_State* L) {
luaL_rometatable(L, HELLOWORLD_METATABLE, (void*)helloworld_metatable_map); // create metatable for helloworld
return 0;
}
NODEMCU_MODULE_STD(); // define Lua entries

166
tools/extmod/extmod.sh Executable file
View File

@ -0,0 +1,166 @@
#!/bin/bash
# External modules update script.
# This script parses the repository root extmods.ini file to locate
# external modules to download
export EXTMOD_DIR="components/modules/external" # Location where to store external modules:
EXTMOD_BIN_PATH="./tools/extmod" # Location of this script
TEMPLATE_MK="$EXTMOD_BIN_PATH/component.mk.template" # Location of the template component.mk
# Include the ini file reader script
. $EXTMOD_BIN_PATH/read_ini.sh
# Returns the given value in the INI file, passing section and value
function sectionVar() {
local varname="INI__$1__$2"
echo "${!varname}"
}
# helper pushd to make it silent
function pushd() {
command pushd "$@" >/dev/null
}
# helper popd to make it silent
function popd() {
command popd >/dev/null
}
function usage() {
echo ""
echo "extmod.sh - Manages external modules"
echo "Usage:"
echo "extmod.sh <commands>"
echo "update : Parses extmods.ini and updates all modules"
echo "clean : Effectively cleans the contents of the external modules directory ($EXTMOD_DIR)"
echo ""
}
# Generic command line parser
function readCommandLine() {
while test ${#} -gt 0; do
case "$1" in
"clean")
CLEAN=1
;;
"update")
UPDATE=1
;;
*)
echo -e "Error: Unrecognized parameter\n"
usage
exit 1
;;
esac
shift
done
}
function updateMod() {
local modname="$1"
local url="$(sectionVar "$modname" "url")"
local ref="$(sectionVar "$modname" "ref")"
local disabled="$(sectionVar "$modname" "disabled")"
local path="$EXTMOD_DIR/$modname"
local component_mk="$path/component.mk"
if [[ ! -d "$path" ]]; then
echo "$modname not present. Downloading from $url ..."
if ! git clone --quiet "$url" -b "$ref" "$path"; then
echo "Error cloning $modname in $url"
return 1
fi
# Add "component.mk" to local repo gitignore
echo "component.mk" >>"$path/.git/info/exclude"
fi
echo "Updating $modname ..."
if ! pushd "$path"; then
echo "Cannot change to $path".
return 1
fi
if ! git status >/dev/null; then
echo "Error processing $modname. Error reading git repo status."
popd
return 1
fi
if [ "$(git status --short)" == "" ]; then
if ! git fetch --quiet; then
echo "Error fetching $modname"
popd
return 1
fi
if ! git checkout "$ref" --quiet; then
echo "Error setting ref $ref in $modname. Does $ref exist?"
popd
return 1
fi
if ! git clean -d -f --quiet; then
echo "Error repo $modname after checkout."
popd
return 1
fi
# check if HEAD was detached (like when checking out a tag or commit)
if git symbolic-ref HEAD 2>/dev/null; then
# This is a branch. Update it.
if ! git pull --quiet; then
echo "Error pulling $ref from $url."
popd
return 1
fi
fi
else
echo "$modname working directory in $path is not clean. Skipping..."
popd
return 0
fi
popd
if [ "$disabled" != "1" ]; then
if ! sed 's/%%MODNAME%%/'"$modname"'/g' "$TEMPLATE_MK" >"$component_mk"; then
echo "Error generating $component_mk"
return 1
fi
else
echo "Warning: Module $modname is disabled and won't be included in build"
[ -f "$component_mk" ] && rm "$component_mk"
fi
echo "Successfully updated $modname."
return 0
}
function update() {
if ! read_ini "extmods.ini"; then
echo "Error reading extmods.ini"
fi
mkdir -p "$EXTMOD_DIR"
for modname in $INI__ALL_SECTIONS; do
if ! updateMod "$modname"; then
echo "Error updating $modname"
return 1
fi
done
echo "Successfully updated all modules"
}
function main() {
if [ "$CLEAN" == "1" ]; then
echo "Cleaning ${EXTMOD_DIR:?} ..."
rm -rf "${EXTMOD_DIR:?}/"*
fi
if [ "$UPDATE" == "1" ]; then
update
fi
}
readCommandLine "$@"
main

256
tools/extmod/read_ini.sh Normal file
View File

@ -0,0 +1,256 @@
#!/bin/bash
# Copyright (c) 2009 Kevin Porter / Advanced Web Construction Ltd
# (http://coding.tinternet.info, http://webutils.co.uk)
# Copyright (c) 2010-2014 Ruediger Meier <sweet_f_a@gmx.de>
# (https://github.com/rudimeier/)
#
# License: BSD-3-Clause, see LICENSE file
#
# Simple INI file parser.
#
# See README for usage.
#
#
function read_ini() {
# Be strict with the prefix, since it's going to be run through eval
function check_prefix() {
if ! [[ "${VARNAME_PREFIX}" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
echo "read_ini: invalid prefix '${VARNAME_PREFIX}'" >&2
return 1
fi
}
function check_ini_file() {
if [ ! -r "$INI_FILE" ]; then
echo "read_ini: '${INI_FILE}' doesn't exist or not" \
"readable" >&2
return 1
fi
}
# enable some optional shell behavior (shopt)
function pollute_bash() {
if ! shopt -q extglob; then
SWITCH_SHOPT="${SWITCH_SHOPT} extglob"
fi
if ! shopt -q nocasematch; then
SWITCH_SHOPT="${SWITCH_SHOPT} nocasematch"
fi
shopt -q -s ${SWITCH_SHOPT}
}
# unset all local functions and restore shopt settings before returning
# from read_ini()
function cleanup_bash() {
shopt -q -u ${SWITCH_SHOPT}
unset -f check_prefix check_ini_file pollute_bash cleanup_bash
}
local INI_FILE=""
local INI_SECTION=""
# {{{ START Deal with command line args
# Set defaults
local BOOLEANS=1
local VARNAME_PREFIX=INI
local CLEAN_ENV=0
# {{{ START Options
# Available options:
# --boolean Whether to recognise special boolean values: ie for 'yes', 'true'
# and 'on' return 1; for 'no', 'false' and 'off' return 0. Quoted
# values will be left as strings
# Default: on
#
# --prefix=STRING String to begin all returned variables with (followed by '__').
# Default: INI
#
# First non-option arg is filename, second is section name
while [ $# -gt 0 ]; do
case $1 in
--clean | -c)
CLEAN_ENV=1
;;
--booleans | -b)
shift
BOOLEANS=$1
;;
--prefix | -p)
shift
VARNAME_PREFIX=$1
;;
*)
if [ -z "$INI_FILE" ]; then
INI_FILE=$1
else
if [ -z "$INI_SECTION" ]; then
INI_SECTION=$1
fi
fi
;;
esac
shift
done
if [ -z "$INI_FILE" ] && [ "${CLEAN_ENV}" = 0 ]; then
echo -e "Usage: read_ini [-c] [-b 0| -b 1]] [-p PREFIX] FILE" \
"[SECTION]\n or read_ini -c [-p PREFIX]" >&2
cleanup_bash
return 1
fi
if ! check_prefix; then
cleanup_bash
return 1
fi
local INI_ALL_VARNAME="${VARNAME_PREFIX}__ALL_VARS"
local INI_ALL_SECTION="${VARNAME_PREFIX}__ALL_SECTIONS"
local INI_NUMSECTIONS_VARNAME="${VARNAME_PREFIX}__NUMSECTIONS"
if [ "${CLEAN_ENV}" = 1 ]; then
eval unset "\$${INI_ALL_VARNAME}"
fi
unset ${INI_ALL_VARNAME}
unset ${INI_ALL_SECTION}
unset ${INI_NUMSECTIONS_VARNAME}
if [ -z "$INI_FILE" ]; then
cleanup_bash
return 0
fi
if ! check_ini_file; then
cleanup_bash
return 1
fi
# Sanitise BOOLEANS - interpret "0" as 0, anything else as 1
if [ "$BOOLEANS" != "0" ]; then
BOOLEANS=1
fi
# }}} END Options
# }}} END Deal with command line args
local LINE_NUM=0
local SECTIONS_NUM=0
local SECTION=""
# IFS is used in "read" and we want to switch it within the loop
local IFS=$' \t\n'
local IFS_OLD="${IFS}"
# we need some optional shell behavior (shopt) but want to restore
# current settings before returning
local SWITCH_SHOPT=""
pollute_bash
while read -r line || [ -n "$line" ]; do
#echo line = "$line"
((LINE_NUM++))
# Skip blank lines and comments
if [ -z "$line" -o "${line:0:1}" = ";" -o "${line:0:1}" = "#" ]; then
continue
fi
# Section marker?
if [[ "${line}" =~ ^\[[a-zA-Z0-9_]{1,}\]$ ]]; then
# Set SECTION var to name of section (strip [ and ] from section marker)
SECTION="${line#[}"
SECTION="${SECTION%]}"
eval "${INI_ALL_SECTION}=\"\${${INI_ALL_SECTION}# } $SECTION\""
((SECTIONS_NUM++))
continue
fi
# Are we getting only a specific section? And are we currently in it?
if [ ! -z "$INI_SECTION" ]; then
if [ "$SECTION" != "$INI_SECTION" ]; then
continue
fi
fi
# Valid var/value line? (check for variable name and then '=')
if ! [[ "${line}" =~ ^[a-zA-Z0-9._]{1,}[[:space:]]*= ]]; then
echo "Error: Invalid line:" >&2
echo " ${LINE_NUM}: $line" >&2
cleanup_bash
return 1
fi
# split line at "=" sign
IFS="="
read -r VAR VAL <<<"${line}"
IFS="${IFS_OLD}"
# delete spaces around the equal sign (using extglob)
VAR="${VAR%%+([[:space:]])}"
VAL="${VAL##+([[:space:]])}"
VAR=$(echo $VAR)
# Construct variable name:
# ${VARNAME_PREFIX}__$SECTION__$VAR
# Or if not in a section:
# ${VARNAME_PREFIX}__$VAR
# In both cases, full stops ('.') are replaced with underscores ('_')
if [ -z "$SECTION" ]; then
VARNAME=${VARNAME_PREFIX}__${VAR//./_}
else
VARNAME=${VARNAME_PREFIX}__${SECTION}__${VAR//./_}
fi
eval "${INI_ALL_VARNAME}=\"\${${INI_ALL_VARNAME}# } ${VARNAME}\""
if [[ "${VAL}" =~ ^\".*\"$ ]]; then
# remove existing double quotes
VAL="${VAL##\"}"
VAL="${VAL%%\"}"
elif [[ "${VAL}" =~ ^\'.*\'$ ]]; then
# remove existing single quotes
VAL="${VAL##\'}"
VAL="${VAL%%\'}"
elif [ "$BOOLEANS" = 1 ]; then
# Value is not enclosed in quotes
# Booleans processing is switched on, check for special boolean
# values and convert
# here we compare case insensitive because
# "shopt nocasematch"
case "$VAL" in
yes | true | on)
VAL=1
;;
no | false | off)
VAL=0
;;
esac
fi
# enclose the value in single quotes and escape any
# single quotes and backslashes that may be in the value
VAL="${VAL//\\/\\\\}"
VAL="\$'${VAL//\'/\'}'"
eval "$VARNAME=$VAL"
done <"${INI_FILE}"
# return also the number of parsed sections
eval "$INI_NUMSECTIONS_VARNAME=$SECTIONS_NUM"
cleanup_bash
}