From 6316b33296e180de4dbcfedecb8fe5c18383d11d Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sat, 16 Jan 2021 21:26:22 +0000 Subject: [PATCH] More NTest prep work for eventual test harness (#3353) * Rename to tests/README.md * Expand tests/README.md a bit * NTest: remove report() in favor of named fields Use a metatable to provide defaults which can be shadowed by the calling code. * NTest: remove old interface flag I think we have few enough tests that we can verify not needing this alert for ourselves. * NTest tests: new standard prelude Allow for NTest constructor to be passed in to the test itself. The test harness can use this to provide a wrapper that will pre-configure NTest itself. * NTest output handler for TAP messages * expect tests: core library functions * expect tests: file xfer TCL module * expect tests: add TAP-based test runner * Begin documenting TCL goo * Add .gitattributes to make sure lineends are correct ... ... if checked out under windows and executed under linux (say docker) * tests/README: enumerate dependencies * tests: more README.md Co-authored-by: Gregor Hartmann --- tests/.gitattributes | 13 ++ tests/NTest/NTest.lua | 46 +++--- tests/NTest/NTest.md | 11 +- tests/NTest/NTest_NTest.lua | 4 +- tests/NTest_adc_env.lua | 3 +- tests/NTest_file.lua | 3 +- tests/NTest_gpio_env.lua | 3 +- tests/NTest_tmr.lua | 3 +- tests/NodeMCU_Test_Environment.md | 106 ------------ tests/README.md | 215 ++++++++++++++++++++++++ tests/expectnmcu/core.tcl | 136 +++++++++++++++ tests/expectnmcu/pkgIndex.tcl | 12 ++ tests/expectnmcu/xfer.tcl | 148 +++++++++++++++++ tests/tap-driver.expect | 266 ++++++++++++++++++++++++++++++ tests/utils/NTestTapOut.lua | 30 ++++ 15 files changed, 856 insertions(+), 143 deletions(-) create mode 100644 tests/.gitattributes delete mode 100644 tests/NodeMCU_Test_Environment.md create mode 100644 tests/README.md create mode 100644 tests/expectnmcu/core.tcl create mode 100644 tests/expectnmcu/pkgIndex.tcl create mode 100644 tests/expectnmcu/xfer.tcl create mode 100755 tests/tap-driver.expect create mode 100644 tests/utils/NTestTapOut.lua diff --git a/tests/.gitattributes b/tests/.gitattributes new file mode 100644 index 00000000..a8c75d0c --- /dev/null +++ b/tests/.gitattributes @@ -0,0 +1,13 @@ +# Enforce Unix newlines +*.css text eol=lf +*.html text eol=lf +*.js text eol=lf +*.json text eol=lf +*.less text eol=lf +*.md text eol=lf +*.svg text eol=lf +*.yml text eol=lf +*.py text eol=lf +*.sh text eol=lf +*.tcl text eol=lf +*.expect text eol=lf diff --git a/tests/NTest/NTest.lua b/tests/NTest/NTest.lua index 32e5205e..416b6e43 100644 --- a/tests/NTest/NTest.lua +++ b/tests/NTest/NTest.lua @@ -171,22 +171,26 @@ local function fail(handler, name, func, expected, msg) handler('pass', name, msg) end -local function NTest(testrunname, failoldinterface) +local nmt = { + env = _G, + outputhandler = TERMINAL_HANDLER +} +nmt.__index = nmt - if failoldinterface then error("The interface has changed. Please see documentstion.") end +return function(testrunname) local pendingtests = {} - local env = _G - local outputhandler = TERMINAL_HANDLER local started + local N = setmetatable({}, nmt) + local function runpending() if pendingtests[1] ~= nil then node.task.post(node.task.LOW_PRIORITY, function() pendingtests[1](runpending) end) else - outputhandler('finish', testrunname) + N.outputhandler('finish', testrunname) end end @@ -202,9 +206,9 @@ local function NTest(testrunname, failoldinterface) local testfn = function(next) local prev = {} - copyenv(prev, env) + copyenv(prev, N.env) - local handler = outputhandler + local handler = N.outputhandler local restore = function(err) if err then @@ -214,8 +218,8 @@ local function NTest(testrunname, failoldinterface) end end if node then node.setonerror() end - copyenv(env, prev) - outputhandler('end', name) + copyenv(N.env, prev) + handler('end', name) table.remove(pendingtests, 1) collectgarbage() if next then next() end @@ -233,6 +237,7 @@ local function NTest(testrunname, failoldinterface) restore() end + local env = N.env env.eq = deepeq env.spy = spy env.ok = function (cond, msg) wrap(assertok, false, cond, msg) end @@ -258,7 +263,7 @@ local function NTest(testrunname, failoldinterface) end if not started then - outputhandler('start', testrunname) + N.outputhandler('start', testrunname) started = true end @@ -270,25 +275,20 @@ local function NTest(testrunname, failoldinterface) end end - local function test(name, f) + function N.test(name, f) testimpl(name, f) end - local function testasync(name, f) + function N.testasync(name, f) testimpl(name, f, true) end - local function report(f, envP) - outputhandler = f or outputhandler - env = envP or env - end - local currentCoName - local function testco(name, func) + function N.testco(name, func) -- local t = tmr.create(); local co - testasync(name, function(Next) + N.testasync(name, function(Next) currentCoName = name local function getCB(cbName) @@ -299,7 +299,7 @@ local function NTest(testrunname, failoldinterface) currentCoName = nil Next(err) else - outputhandler('fail', name, "Found stray Callback '"..cbName.."' from test '"..name.."'") + N.outputhandler('fail', name, "Found stray Callback '"..cbName.."' from test '"..name.."'") end elseif coroutine.status(co) == "dead" then currentCoName = nil @@ -327,9 +327,5 @@ local function NTest(testrunname, failoldinterface) end) end - - return {test = test, testasync = testasync, testco = testco, report = report} + return N end - -return NTest - diff --git a/tests/NTest/NTest.md b/tests/NTest/NTest.md index ca4e9d66..2daac3ff 100644 --- a/tests/NTest/NTest.md +++ b/tests/NTest/NTest.md @@ -145,7 +145,7 @@ ok(f.errors[3] ~= nil) ## Reports -Another useful feature is that you can customize test reports as you need. The default `reports` just more or less prints out a basic report. You can easily override this behavior as well as add any other information you need (number of passed/failed assertions, time the test took etc): +Another useful feature is that you can customize test reports as you need. The default `outputhandler` just more or less prints out a basic report. You can easily override (or augment by wrapping, e.g.) this behavior as well as add any other information you need (number of passed/failed assertions, time the test took etc): Events are: `start` when testing starts @@ -161,7 +161,7 @@ Events are: local passed = 0 local failed = 0 -tests.report(function(event, testfunc, msg) +tests.outputhandler = function(event, testfunc, msg) if event == 'begin' then print('Started test', testfunc) passed = 0 @@ -176,7 +176,7 @@ tests.report(function(event, testfunc, msg) elseif event == 'except' then print('ERROR', testfunc, msg) end -end) +end ``` Additionally, you can pass a different environment to keep `_G` unpolluted: @@ -184,8 +184,7 @@ You need to set it, so the helper functions mentioned above can be added before ``` Lua local myenv = {} - -tests.report(function() ... end, myenv) +tests.env = myenv tests.test('Some test', function() myenv.ok(myenv.eq(...)) @@ -193,7 +192,7 @@ tests.test('Some test', function() end) ``` -You can set any of the parameters to `nil` to leave the value unchanged. +You can restore `env` or `outputhandler` to their defaults by setting their values to `nil`. ## Appendix diff --git a/tests/NTest/NTest_NTest.lua b/tests/NTest/NTest_NTest.lua index 0701c681..f0b71434 100644 --- a/tests/NTest/NTest_NTest.lua +++ b/tests/NTest/NTest_NTest.lua @@ -434,7 +434,7 @@ end local pass -- Set meta test handler -N.report(function(e, test, msg, errormsg) +N.outputhandler = function(e, test, msg, errormsg) local function consumemsg(msg, area) -- luacheck: ignore if not expected[1][area][1] then print("--- FAIL "..expected[1].name..' ('..area..'ed): unexpected "'.. @@ -487,7 +487,7 @@ N.report(function(e, test, msg, errormsg) else print("Extra output: ", e, test, msg, errormsg) end -end) +end local async_queue = {} async = function(f) table.insert(async_queue, cbWrap(f)) end diff --git a/tests/NTest_adc_env.lua b/tests/NTest_adc_env.lua index 2904fe7d..6a3267f0 100644 --- a/tests/NTest_adc_env.lua +++ b/tests/NTest_adc_env.lua @@ -1,7 +1,8 @@ -- Walk the ADC through a stepped triangle wave using the attached voltage -- divider and I2C GPIO expander. -local N = require('NTest')("adc-env") +local N = ... +N = (N or require "NTest")("adc-env") -- TODO: Preflight test that we are in the correct environment with an I2C -- expander in the right place with the right connections. diff --git a/tests/NTest_file.lua b/tests/NTest_file.lua index 2d3e4fef..c4cb823d 100644 --- a/tests/NTest_file.lua +++ b/tests/NTest_file.lua @@ -1,4 +1,5 @@ -local N = require('NTest')("file") +local N = ... +N = (N or require "NTest")("file") local function cleanup() file.remove("testfile") diff --git a/tests/NTest_gpio_env.lua b/tests/NTest_gpio_env.lua index 9715790f..8d15d73b 100644 --- a/tests/NTest_gpio_env.lua +++ b/tests/NTest_gpio_env.lua @@ -3,7 +3,8 @@ -- Node GPIO 13 (index 7) is connected to I2C expander channel B6; node OUT -- Node GPIO 15 (index 8) is connected to I2C expander channel B7; node IN -local N = require('NTest')("gpio-env") +local N = ... +N = (N or require "NTest")("gpio-env") -- TODO: Preflight test that we are in the correct environment with an I2C -- expander in the right place with the right connections. diff --git a/tests/NTest_tmr.lua b/tests/NTest_tmr.lua index 2b77ce70..570d75b5 100644 --- a/tests/NTest_tmr.lua +++ b/tests/NTest_tmr.lua @@ -1,4 +1,5 @@ -local N = require('NTest')("tmr") +local N = ... +N = (N or require "NTest")("tmr") N.testasync('SINGLE alarm', function(next) local t = tmr.create(); diff --git a/tests/NodeMCU_Test_Environment.md b/tests/NodeMCU_Test_Environment.md deleted file mode 100644 index 5a86e8d4..00000000 --- a/tests/NodeMCU_Test_Environment.md +++ /dev/null @@ -1,106 +0,0 @@ -NodeMCU Testing Environment -=========================== - -Herein we define the environment our testing framework expects to see -when it runs. It is composed of two ESP8266 devices, each capable of -holding an entire NodeMCU firmware, LFS image, and SPIFFS file system, -as well as additional peripheral hardware. It is designed to fit -comfortably on a breadboard and so should be easily replicated and -integrated into any firmware validation testing. - -The test harness runs from a dedicated host computer, which is expected -to have reset- and programming-capable UART links to both ESP8266 -devices, as found on almost all ESP8266 boards with USB to UART -adapters, but the host does not necessarily need to use USB to connect, -so long as TXD, RXD, DTR, and RTS are wired across. - -Peripherals ------------ - -### I2C Bus - -There is an I2C bus hanging off DUT 0. Attached hardware is used both as -tests of modules directly and also to facilitate testing other modules -(e.g., gpio). - -#### MCP23017: I/O Expander - -At address 0x20. An 16-bit tristate GPIO expander, this chip is used to -test I2C, GPIO, and ADC functionality. This chip's interconnections are -as follows: - -MPC23017 | Purpose ----------|-------------------------------------------------------------- -/RESET |DUT0 reset. This resets the chip whenever the host computer resets DUT 0 over its serial link (using DTR/RTS). -B 0 |4K7 resistor to DUT 0 ADC. -B 1 |2K2 resistor to DUT 0 ADC. -B 5 |DUT1 GPIO16/WAKE via 4K7 resitor -B 6 |DUT0 GPIO13 via 4K7 resistor and DUT1 GPIO15 via 4K7 resistor -B 7 |DUT0 GPIO15 via 4K7 resistor and DUT1 GPIO13 via 4K7 resistor - -Notes: - -- DUT 0's ADC pin is connected via a 2K2 reistor to this chip's port - B, pin 1 and via a 4K7 resistor to port B, pin 0. This gives us the - ability to produce approximately 0 (both pins low), 1.1 (pin 0 high, - pin 1 low), 2.2 (pin 1 high, pin 0 low), and 3.3V (both pins high) - on the ADC pin. -- Port B pins 6 and 7 sit on the UART cross-wiring between DUT 0 and - DUT 1. The 23017 will be tristated for inter-DUT UART tests, but - these -- Port B pins 2, 3, and 4, as well as all of port A, remain available - for expansion. -- The interrupt pins are not yet routed, but could be. We reserve DUT - 0 GPIO 2 for this purpose with the understanding that the 23017's - interrupt functionality will be disabled (INTA, INTB set to - open-drain, GPINTEN set to 0) when not explicitly under test. - -ESP8266 Device 0 Connections ----------------------------- - -ESP | Usage -----------|---------------------------------------------------------- -GPIO 0 |Used to enter programming mode; otherwise unused in test environment. -GPIO 1 |Primary UART transmit; reserved for host communication -GPIO 2 |[reserved for 1-Wire] [+ reserved for 23017 INT[AB] connections] -GPIO 3 |Primary UART recieve; reserved for host communication -GPIO 4 |I2C SDA -GPIO 5 |I2C SCL -GPIO 6 |[Reserved for on-chip flash] -GPIO 7 |[Reserved for on-chip flash] -GPIO 8 |[Reserved for on-chip flash] -GPIO 9 |[Reserved for on-chip flash] -GPIO 10 |[Reserved for on-chip flash] -GPIO 11 |[Reserved for on-chip flash] -GPIO 12 | -GPIO 13 |Secondary UART RX; DUT 1 GPIO 15, I/O expander B 6 -GPIO 14 | -GPIO 15 |Secondary UART TX; DUT 1 GPIO 13, I/O expander B 7 -GPIO 16 | -ADC 0 |Resistor divider with I/O expander - -ESP8266 Device 1 Connections ----------------------------- - -ESP | Usage -----------|---------------------------------------------------------- -GPIO 0 |Used to enter programming mode; otherwise unused in test environment. -GPIO 1 |Primary UART transmit; reserved for host communication -GPIO 2 |[Reserved for WS2812] -GPIO 3 |Primary UART recieve; reserved for host communication -GPIO 4 | -GPIO 5 | -GPIO 6 |[Reserved for on-chip flash] -GPIO 7 |[Reserved for on-chip flash] -GPIO 8 |[Reserved for on-chip flash] -GPIO 9 |[Reserved for on-chip flash] -GPIO 10 |[Reserved for on-chip flash] -GPIO 11 |[Reserved for on-chip flash] -GPIO 12 |HSPI MISO -GPIO 13 |Secondary UART RX; DUT 0 GPIO 15, I/O exp B 7 via 4K7 Also used as HSPI MOSI for SPI tests -GPIO 14 |HSPI CLK -GPIO 15 |Secondary UART TX; DUT 0 GPIO 13, I/O exp B 6 via 4K7 Also used as HSPI /CS for SPI tests -GPIO 16 |I/O expander B 5 via 4K7 resistor, for deep-sleep tests -ADC 0 | - - diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..13561d1a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,215 @@ +# Introduction + +Welcome to the NodeMCU self-test suite. Here you will find our growing effort +to ensure that our software behaves as we think it should and that we do not +regress against earlier versions. + +Our tests are written using [NTest](./NTest/NTest.md), a lightweight yet +featureful framework for specifying unit tests. + +# Building and Running Test Software on NodeMCU Devices + +Naturally, to test NodeMCU on its intended hardware, you will need one or more +NodeMCU-capable boards. At present, the test environment is specified using +two ESP8266 Devices Under Test (DUTs), but we envision expanding this to mixed +ESP8266/ESP32 environments as well. + +Test programs live beside this file. While many test programs run on the +NodeMCU DUTs, but there is reason to want to orchestrate DUTs and the +environment using the host. Files matching the glob `NTest_*.lua` are intended +for on-DUT execution. + +## Manual Test Invocation + +At the moment, the testing regime and host-based orchestration is still in +development, and so things are a little more manual than perhaps desired. The +`NTest`-based test programs all assume that they can `require "NTest"`, and so +the easiest route to success is to + +* build an LFS image containing + + * [package.loader support for LFS](../lua_examples/lfs/_init.lua) + + * [NTest itself](./NTest/NTest.lua) + + * Any additional Lua support modules required (e.g., [mcp23017 + support](../lua_modules/mcp23017/mcp23017.lua) ) + +* build a firmware with the appropriate C modules + +* program the board with your firmware and LFS images + +* ensure that `package.loader` is patched appropriately on startup + +* transfer the `NTest_foo` program you wish to run to the device SPIFFS + (or have included it in the LFS). + +* at the interpreter prompt, say `dofile("NTest_foo.lua")` (or + `node.LFS.get("NTest_foo")()`) to run the `foo` test program. + +## Experimental Host Orchestration + +Enthusiastic testers are encouraged to try using our very new, very +experimental host test runner, [tap-driver.expect](./tap-driver.expect). To +use this program, in addition to the above, the LFS environment should contain +[NTestTapOut](./tests/utils/NTestTapOut.lua), an output adapter for `NTest`, +making it speak a slight variant of the [Test Anything +Protocol](https://testanything.org/). This structured output is scanned for +by the script on the host. + +You'll need `expect` and TCL and some TCL libraries available; on Debian, that +amounts to + + apt install tcl tcllib tclx8.4 expect + +This program should be invoked from beside this file with something like + + TCLLIBPATH=./expectnmcu ./tap-driver.expect -serial /dev/ttyUSB3 -lfs ./lfs.img NTest_file.lua + +This will... + +* transfer and install the specified LFS module (and reboot the device to load LFS) + +* transfer the test program + +* run the test program with `NTest` shimmed to use the `NTestTapOut` output + handler + +* summarize the results + +* return 0 if and only if all tests have passed + +This tool is quite flexible and takes a number of other options and flags +controlling aspects of its behavior: + +* Additional files, Lua or otherwise, may be transferred by specifing them + before the test to run (e.g., `./tap-driver.expect a.lua b.lua + NTest_foo.lua`); dually, a `-noxfer` flag will suppress transferring even the + last file. All transferred files are moved byte-for-byte to the DUT's + SPIFFS with names, but not directory components, preserved. + +* The `-lfs LFS.img` option need not be specified and, if not given, any + existing `LFS` image will remain on the device for use by the test. + +* A `-nontestshim` flag will skip attempting to shim the given test program + with `NTestTapOut`; the test program is expected to provide its own TAP + output. The `-tpfx` argument can be used to override the leading `TAP: ` + sigil used by the `NTestTapOut` output handler. + +* A `-runfunc` option indicates that the last argument is not a file to + transfer but rather a function to be run. It will be invoked at the REPL + with a single argument, the shimmed `NTest` constructor, unless `-nontestshim` + is given, in which case the argument will be `nil`. + +* A `-notests` option suppresses running tests (making the tool merely another + option for loading files to the device). + +Transfers will be significantly faster if +[pipeutils](../lua_examples/pipeutils.lua) is available to `require` on the +DUT, but a fallback strategy exists if not. We suggest either including +`pipeutils` in LFS images, in SPIFFS, or as the first file to be transferred. + +# NodeMCU Testing Environment + +Herein we define the environment our testing framework expects to see +when it runs. It is composed of two ESP8266 devices, each capable of +holding an entire NodeMCU firmware, LFS image, and SPIFFS file system, +as well as additional peripheral hardware. It is designed to fit +comfortably on a breadboard and so should be easily replicated and +integrated into any firmware validation testing. + +The test harness runs from a dedicated host computer, which is expected +to have reset- and programming-capable UART links to both ESP8266 +devices, as found on almost all ESP8266 boards with USB to UART +adapters, but the host does not necessarily need to use USB to connect, +so long as TXD, RXD, DTR, and RTS are wired across. + +## Peripherals + +### I2C Bus + +There is an I2C bus hanging off DUT 0. Attached hardware is used both as +tests of modules directly and also to facilitate testing other modules +(e.g., gpio). + +#### MCP23017: I/O Expander + +At address 0x20. An 16-bit tristate GPIO expander, this chip is used to +test I2C, GPIO, and ADC functionality. This chip's interconnections are +as follows: + +MPC23017 | Purpose +---------|-------------------------------------------------------------- +/RESET |DUT0 reset. This resets the chip whenever the host computer resets DUT 0 over its serial link (using DTR/RTS). +B 0 |4K7 resistor to DUT 0 ADC. +B 1 |2K2 resistor to DUT 0 ADC. +B 5 |DUT1 GPIO16/WAKE via 4K7 resitor +B 6 |DUT0 GPIO13 via 4K7 resistor and DUT1 GPIO15 via 4K7 resistor +B 7 |DUT0 GPIO15 via 4K7 resistor and DUT1 GPIO13 via 4K7 resistor + +Notes: + +- DUT 0's ADC pin is connected via a 2K2 reistor to this chip's port + B, pin 1 and via a 4K7 resistor to port B, pin 0. This gives us the + ability to produce approximately 0 (both pins low), 1.1 (pin 0 high, + pin 1 low), 2.2 (pin 1 high, pin 0 low), and 3.3V (both pins high) + on the ADC pin. +- Port B pins 6 and 7 sit on the UART cross-wiring between DUT 0 and + DUT 1. The 23017 will be tristated for inter-DUT UART tests, but + these +- Port B pins 2, 3, and 4, as well as all of port A, remain available + for expansion. +- The interrupt pins are not yet routed, but could be. We reserve DUT + 0 GPIO 2 for this purpose with the understanding that the 23017's + interrupt functionality will be disabled (INTA, INTB set to + open-drain, GPINTEN set to 0) when not explicitly under test. + +ESP8266 Device 0 Connections +---------------------------- + +ESP | Usage +----------|---------------------------------------------------------- +GPIO 0 |Used to enter programming mode; otherwise unused in test environment. +GPIO 1 |Primary UART transmit; reserved for host communication +GPIO 2 |[reserved for 1-Wire] [+ reserved for 23017 INT[AB] connections] +GPIO 3 |Primary UART recieve; reserved for host communication +GPIO 4 |I2C SDA +GPIO 5 |I2C SCL +GPIO 6 |[Reserved for on-chip flash] +GPIO 7 |[Reserved for on-chip flash] +GPIO 8 |[Reserved for on-chip flash] +GPIO 9 |[Reserved for on-chip flash] +GPIO 10 |[Reserved for on-chip flash] +GPIO 11 |[Reserved for on-chip flash] +GPIO 12 | +GPIO 13 |Secondary UART RX; DUT 1 GPIO 15, I/O expander B 6 +GPIO 14 | +GPIO 15 |Secondary UART TX; DUT 1 GPIO 13, I/O expander B 7 +GPIO 16 | +ADC 0 |Resistor divider with I/O expander + +ESP8266 Device 1 Connections +---------------------------- + +ESP | Usage +----------|---------------------------------------------------------- +GPIO 0 |Used to enter programming mode; otherwise unused in test environment. +GPIO 1 |Primary UART transmit; reserved for host communication +GPIO 2 |[Reserved for WS2812] +GPIO 3 |Primary UART recieve; reserved for host communication +GPIO 4 | +GPIO 5 | +GPIO 6 |[Reserved for on-chip flash] +GPIO 7 |[Reserved for on-chip flash] +GPIO 8 |[Reserved for on-chip flash] +GPIO 9 |[Reserved for on-chip flash] +GPIO 10 |[Reserved for on-chip flash] +GPIO 11 |[Reserved for on-chip flash] +GPIO 12 |HSPI MISO +GPIO 13 |Secondary UART RX; DUT 0 GPIO 15, I/O exp B 7 via 4K7 Also used as HSPI MOSI for SPI tests +GPIO 14 |HSPI CLK +GPIO 15 |Secondary UART TX; DUT 0 GPIO 13, I/O exp B 6 via 4K7 Also used as HSPI /CS for SPI tests +GPIO 16 |I/O expander B 5 via 4K7 resistor, for deep-sleep tests +ADC 0 | + + diff --git a/tests/expectnmcu/core.tcl b/tests/expectnmcu/core.tcl new file mode 100644 index 00000000..66ec5299 --- /dev/null +++ b/tests/expectnmcu/core.tcl @@ -0,0 +1,136 @@ +namespace eval expectnmcu::core { + set panicre "powered by Lua \[0-9.\]+ on SDK \[0-9.\]+" + set promptstr "\n> " + + namespace export reboot waitboot connect + namespace export send_exp_prompt send_exp_res_prompt send_exp_prompt_c +} + +package require cmdline + +# Use DTR/RTS signaling to reboot the device +## I'm not sure why we have to keep resetting the mode, but so it goes. +proc ::expectnmcu::core::reboot { dev } { + set victimfd [open ${dev} ] + set mode [fconfigure ${victimfd} -mode ] + fconfigure ${victimfd} -mode ${mode} -ttycontrol {DTR 0 RTS 1} + sleep 0.1 + fconfigure ${victimfd} -mode ${mode} -ttycontrol {DTR 0 RTS 0} + close ${victimfd} +} + +proc ::expectnmcu::core::waitboot { victim } { + expect { + -i ${victim} "Formatting file system" { + set timeout 120 + exp_continue + } + -i ${victim} "powered by Lua" { } + timeout { return -code error "Timeout" } + } + # Catch nwf's system bootup, in case we're testing an existing system, + # rather than a blank firmware. + expect { + -i ${victim} -re "Reset delay!.*${::expectnmcu::core::promptstr}" { + send -i ${victim} "stop(true)\n" + expect -i ${victim} -ex ${::expectnmcu::core::promptstr} + } + -i ${victim} -ex ${::expectnmcu::core::promptstr} { } + timeout { return -code error "Timeout" } + } + + # Do a little more active synchronization with the DUT: send it a command + # and wait for the side-effect of that command to happen, thereby ensuring + # that the next prompt we see is after this point in the input. + send -i ${victim} "print(\"a\",\"z\")\n" + expect { + -i ${victim} -ex "a\tz" { } + } + expect { + -i ${victim} -ex ${::expectnmcu::core::promptstr} { } + timeout { return -code error "Timeout" } + } +} + +# Establish a serial connection to the device via socat. Takes +# -baud=N, -reboot=0/1/dontwait, -waitboot=0/1 optional parameters +proc ::expectnmcu::core::connect { dev args } { + set opts { + { baud.arg 115200 } + { reboot.arg 1 } + } + array set arg [::cmdline::getoptions args $opts] + + spawn "socat" "STDIO" "${dev},b${arg(baud)},raw,crnl" + close -onexec 1 -i ${spawn_id} + set victim ${spawn_id} + + # XXX? + set victimfd [open ${dev} ] + set mode [fconfigure ${victimfd} -mode ${arg(baud)},n,8,1 ] + + if { ${arg(reboot)} != 0 } { + ::expectnmcu::core::reboot ${dev} + if { ${arg(reboot)} != "dontwait" } { + ::expectnmcu::core::waitboot ${victim} + } + } + + close ${victimfd} + + return ${victim} +} + +# This one is somewhat "for experts only" -- it expects that you have either +# consumed whatever command you flung at the node or that you have some reason +# to not be concerned with its echo (and return) +proc ::expectnmcu::core::exp_prompt { sid } { + expect { + -i ${sid} -ex ${::expectnmcu::core::promptstr} { } + -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout" } + } +} + +proc ::expectnmcu::core::send_exp_prompt { sid cmd } { + send -i ${sid} -- "${cmd}\n" + expect { + -i ${sid} -ex "${cmd}" { } + timeout { return -code error "Timeout" } + } + expect { + -i ${sid} -ex ${::expectnmcu::core::promptstr} { } + -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout" } + } +} + +proc ::expectnmcu::core::send_exp_res_prompt { sid cmd res } { + send -i ${sid} -- "${cmd}\n" + expect { + -i ${sid} -ex "${cmd}" { } + timeout { return -code error "Timeout" } + } + expect { + -i ${sid} -re "${res}.*${::expectnmcu::core::promptstr}" { } + -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + -i ${sid} -ex ${::expectnmcu::core::promptstr} { return -code error "Prompt before expected response" } + timeout { return -code error "Timeout" } + } +} + +proc ::expectnmcu::core::send_exp_prompt_c { sid cmd } { + send -i ${sid} -- "${cmd}\n" + expect { + -i ${sid} -ex "${cmd}" { } + timeout { return -code error "Timeout" } + } + expect { + -i ${sid} -ex "\n>> " { } + -i ${sid} -ex ${::expectnmcu::core::promptstr} { return -code error "Non-continuation prompt" } + -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout" } + } +} + +package provide expectnmcu::core 1.0 diff --git a/tests/expectnmcu/pkgIndex.tcl b/tests/expectnmcu/pkgIndex.tcl new file mode 100644 index 00000000..64dbff69 --- /dev/null +++ b/tests/expectnmcu/pkgIndex.tcl @@ -0,0 +1,12 @@ +# Tcl package index file, version 1.1 +# This file is generated by the "pkg_mkIndex" command +# and sourced either when an application starts up or +# by a "package unknown" script. It invokes the +# "package ifneeded" command to set up package-related +# information so that packages will be loaded automatically +# in response to "package require" commands. When this +# script is sourced, the variable $dir must contain the +# full path name of this file's directory. + +package ifneeded expectnmcu::core 1.0 [list source [file join $dir core.tcl]] +package ifneeded expectnmcu::xfer 1.0 [list source [file join $dir xfer.tcl]] diff --git a/tests/expectnmcu/xfer.tcl b/tests/expectnmcu/xfer.tcl new file mode 100644 index 00000000..6c9b4139 --- /dev/null +++ b/tests/expectnmcu/xfer.tcl @@ -0,0 +1,148 @@ +namespace eval expectnmcu::xfer { +} + +package require expectnmcu::core + +# Open remote file `which` on `dev` in `mode` as Lua object `dfh` +proc ::expectnmcu::xfer::open { dev dfh which mode } { + ::expectnmcu::core::send_exp_prompt ${dev} "${dfh} = nil" + ::expectnmcu::core::send_exp_prompt ${dev} "${dfh} = file.open(\"${which}\",\"${mode}\")" + ::expectnmcu::core::send_exp_res_prompt ${dev} "=type(${dfh})" "userdata" +} + +# Close Lua file object `dfh` on `dev` +proc ::expectnmcu::xfer::close { dev dfh } { + ::expectnmcu::core::send_exp_prompt ${dev} "${dfh}:close()" +} + +# Write to `dfh` on `dev` at `where` `what`, using base64 as transport +# +# This does not split lines; write only short amounts of data. +proc ::expectnmcu::xfer::pwrite { dev dfh where what } { + send -i ${dev} -- [string cat \ + "do local d,e = encoder.fromBase64(\"[binary encode base64 -maxlen 0 ${what}]\");" \ + "${dfh}:seek(\"set\",${where});" \ + "print(${dfh}:write(d));" \ + "end\n" \ + ] + expect { + -i ${dev} -re "true\[\r\n\]+> " { } + -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + -i ${dev} -ex "\n> " { return -code error "Bad result from pwrite" } + timeout { return -code error "Timeout while waiting for pwrite" } + } +} + +# Read `howmuch` byetes from `dfh` on `dev` at `where`, using base64 +# as transport. This buffers the whole data and its base64 encoding +# in device RAM; read only short strings. +proc ::expectnmcu::xfer::pread { dev dfh where howmuch } { + send -i ${dev} -- "${dfh}:seek(\"set\",${where}); print(encoder.toBase64(${dfh}:read(${howmuch})))\n" + expect { + -i ${dev} -re "\\)\\)\\)\[\r\n\]+(\[^\r\n\]+)\[\r\n\]+> " { + return [binary decode base64 ${expect_out(1,string)}] + } + -i ${dev} -ex "\n> " { return -code error "No reply to pread" } + -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout while pread-ing" } + } +} + +# Check for pipeutils on the target device +proc ::expectnmcu::xfer::haspipeutils { dev } { + send -i ${dev} -- "local ok, pu = pcall(require, \"pipeutils\"); print(ok and type(pu) == \"table\" and pu.chunker and pu.debase64 and true or false)\n" + expect { + -i ${dev} -re "\[\r\n\]+false\[\r\n\]+> " { return 0 } + -i ${dev} -re "\[\r\n\]+true\[\r\n\]+> " { return 1 } + -i ${dev} -ex "\n> " { return -code error "No reply to pipeutils probe" } + -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout while probing for pipeutils" } + } +} + +# Send local file `lfn` to the remote filesystem on `dev` and name it `rfn`. +# Use `dfo` as the Lua handle to the remote file for the duration of writing, +# (and `nil` it out afterwards) +proc ::expectnmcu::xfer::sendfile { dev lfn rfn {dfo "xfo"} } { + package require sha256 + + set has_pipeutils [::expectnmcu::xfer::haspipeutils ${dev} ] + + set ltf [::open ${lfn} ] + fconfigure ${ltf} -translation binary + file stat ${lfn} lfstat + ::expectnmcu::xfer::open ${dev} ${dfo} "${rfn}.sf" "w+" + + if { ${has_pipeutils} } { + # Send over a loader program + ::expectnmcu::core::send_exp_prompt_c ${dev} "do" + ::expectnmcu::core::send_exp_prompt_c ${dev} " local pu = require \"pipeutils\"" + ::expectnmcu::core::send_exp_prompt_c ${dev} " local ch = pu.chunker(function(d) ${dfo}:write(d) end, 256)" + ::expectnmcu::core::send_exp_prompt_c ${dev} " local db = pu.debase64(ch.write, function(ed,ee)" + ::expectnmcu::core::send_exp_prompt_c ${dev} " if ed:match(\"^%.\[\\r\\n\]*$\") then ch.flush() print(\"F I N\")" + ::expectnmcu::core::send_exp_prompt_c ${dev} " else print(\"ABORT\", ee, ed) end" + ::expectnmcu::core::send_exp_prompt_c ${dev} " uart.on(\"data\") end)" + # TODO: make echo use CRC not full string; probably best add to crypto module + ::expectnmcu::core::send_exp_prompt_c ${dev} " uart.on(\"data\", \"\\n\", function(x) db.write(x); uart.write(0, \"OK: \", x) end, 0)" + ::expectnmcu::core::send_exp_prompt ${dev} "end" + set xln 90 + } else { + set xln 48 + } + + set lho [sha2::SHA256Init] + + set fpos 0 + while { 1 } { + send_user ">> xfer ${fpos} of ${lfstat(size)}\n" + set data [read ${ltf} ${xln}] + sha2::SHA256Update ${lho} ${data} + if { ${has_pipeutils} } { + set estr [binary encode base64 -maxlen 0 ${data}] + send -i ${dev} -- "${estr}\n" + expect { + -i ${dev} -ex "OK: ${estr}" { expect -i ${dev} -re "\[\r\n\]+" {} } + -i ${dev} -ex "\n> " { return -code error "Prompt while sending data" } + -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout while sending data" } + } + } else { + ::expectnmcu::xfer::pwrite ${dev} ${dfo} ${fpos} ${data} + } + set fpos [expr $fpos + ${xln}] + if { [string length ${data}] != ${xln} } { break } + } + + if { ${has_pipeutils} } { + send -i ${dev} -- ".\n" + expect { + -i ${dev} -re "F I N\[\r\n\]+" { } + -i ${dev} -ex "\n> " { return -code error "Prompt while awaiting acknowledgement" } + -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout while awaiting acknowledgement" } + } + } + + ::close ${ltf} + ::expectnmcu::xfer::close ${dev} ${dfo} + ::expectnmcu::core::send_exp_prompt ${dev} "${dfo} = nil" + + set exphash [sha2::Hex [sha2::SHA256Final ${lho}]] + + send -i ${dev} "=encoder.toHex(crypto.fhash(\"sha256\",\"${rfn}.sf\"))\n" + expect { + -i ${dev} -re "\[\r\n\]+(\[a-f0-9\]+)\[\r\n\]+> " { + if { ${expect_out(1,string)} != ${exphash} } { + return -code error \ + "Sendfile checksum mismatch: ${expect_out(1,string)} != ${exphash}" + } + } + -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" } + timeout { return -code error "Timeout while verifying checksum" } + } + + ::expectnmcu::core::send_exp_prompt ${dev} "file.remove(\"${rfn}\")" + ::expectnmcu::core::send_exp_res_prompt ${dev} "=file.rename(\"${rfn}.sf\", \"${rfn}\")" "true" +} + +package provide expectnmcu::xfer 1.0 diff --git a/tests/tap-driver.expect b/tests/tap-driver.expect new file mode 100755 index 00000000..eed53816 --- /dev/null +++ b/tests/tap-driver.expect @@ -0,0 +1,266 @@ +#!/usr/bin/env expect + +# Push a file to the device, run it, and watch the tests run +# +# A typical invocation looks like: +# TCLLIBPATH=./expectnmcu ./tap-driver.expect -serial /dev/ttyUSB3 ./mispec.lua ./mispec_file.lua +# +# For debugging the driver itself, it may be useful to invoke expect with -d, +# which will give a great deal of diagnostic information about the expect state +# machine's internals: +# +# TCLLIBPATH=./expectnmcu expect -d ./tap-driver.expect ... +# +# The -debug option will turn on some additional reporting from this driver program, as well. + + +package require expectnmcu::core +package require expectnmcu::xfer + +package require cmdline +set cmd_parameters { + { serial.arg "/dev/ttyUSB0" "Set the serial interface name" } + { tpfx.arg "TAP: " "Set the expected TAP test prefix" } + { lfs.arg "" "Flash a file to LFS" } + { noxfer "Do not send files, just run script" } + { runfunc "Last argument is function, not file" } + { notests "Don't run tests, just xfer files" } + { nontestshim "Don't shim NTest when testing" } + { debug "Enable debugging reporting" } +} +set cmd_usage "- A NodeMCU Lua-based-test runner" +if {[catch {array set cmdopts [cmdline::getoptions ::argv $cmd_parameters $cmd_usage]}]} { + send_user [cmdline::usage $cmd_parameters $cmd_usage] + send_user "\n Additional arguments should be files be transferred\n" + send_user " The last file transferred will be run with `dofile`\n" + exit 0 +} + +if { ${cmdopts(noxfer)} } { + if { [ llength ${::argv} ] > 1 } { + send_user "No point in more than one argument if noxfer given\n" + exit 1 + } +} { + set xfers ${::argv} + + if { ${cmdopts(runfunc)} } { + # Last argument is command, not file to xfer + set xfers [lreplace xfers end end] + } + + foreach arg ${xfers} { + if { ! [file exists ${arg}] } { + send_user "File ${arg} does not exist\n" + exit 1 + } + } +} + +if { ${cmdopts(lfs)} ne "" } { + if { ! [file exists ${cmdopts(lfs)}] } { + send_user "LFS file does not exist\n" + exit 1 + } +} + +proc sus { what } { send_user "\n===> ${what} <===\n" } +proc sui { what } { send_user "\n---> ${what} <---\n" } +proc sud { what } { + upvar 1 cmdopts cmdopts + if { ${cmdopts(debug)} } { send_user "\n~~~> ${what} <~~~\n" } +} + +set victim [::expectnmcu::core::connect ${cmdopts(serial)}] +sus "Machine has booted" + +if { ${cmdopts(lfs)} ne "" } { + ::expectnmcu::xfer::sendfile ${victim} ${cmdopts(lfs)} "tap-driver.lfs" + send -i ${victim} "=node.LFS.reload(\"tap-driver.lfs\")\n" + ::expectnmcu::core::waitboot ${victim} +} + +if { ! ${cmdopts(noxfer)} } { + foreach arg ${xfers} { + ::expectnmcu::xfer::sendfile ${victim} ${arg} [file tail ${arg}] + } +} + +set tfn [file tail [lindex ${::argv} end ] ] + +if { ${cmdopts(notests)} || ${tfn} eq "" } { + sus "No tests requested, and so operations are completed" + exit 0 +} + +sus "Files transferred; running ${tfn}" + +if { ! ${cmdopts(nontestshim)} } { + ::expectnmcu::core::send_exp_prompt_c ${victim} "function ntshim(...)" + ::expectnmcu::core::send_exp_prompt_c ${victim} " local test = (require \"NTest\")(...)" + ::expectnmcu::core::send_exp_prompt_c ${victim} " test.outputhandler = require\"NTestTapOut\"" + ::expectnmcu::core::send_exp_prompt_c ${victim} " return test" + ::expectnmcu::core::send_exp_prompt ${victim} "end" +} else { + sui "Not shimming NTest output; test must report its own TAP messages" +} + +# ntshim may be nil at this point if -nontestshim was given; that's fine +if { ${cmdopts(runfunc)} } { + send -i ${victim} "[ lindex ${::argv} end ](ntshim)\n" + expect -i ${victim} -re "\\(ntshim\\)\[\r\n\]+" { } +} else { + send -i ${victim} "assert(loadfile(\"${tfn}\"))(ntshim)\n" + expect -i ${victim} -re "assert\\(loadfile\\(\"${tfn}\"\\)\\)\\(ntshim\\)\[\r\n\]+" { } +} + +set tpfx ${cmdopts(tpfx)} +set toeol "\[^\n\]*(?=\n)" + +# Wait for the test to start and tell us how many +# success lines we should expect +set ntests 0 +set timeout 10 +expect { + -i ${victim} -re "${tpfx}1\\.\\.(\\d+)(?=\r?\n)" { + global ntests + set ntests $expect_out(1,string) + } + -i ${victim} -re "${tpfx}Bail out!${toeol}" { + sus "Bail out before start" + exit 2 + } + -i ${victim} -re ${::expectnmcu::core::panicre} { + sus "Panic!" + exit 2 + } + # A prefixed line other than a plan (1..N) or bailout means we've not got + # a plan. Leave ${ntests} at 0 and proceed to run the protocol. + -i ${victim} -notransfer -re "${tpfx}${toeol}" { } + # -i ${victim} -ex "\n> " { + # sus "Prompt before start!" + # exit 2 + # } + # Consume other outputs and discard as if they were comments + # This must come as the last pattern that looks at input + -i ${victim} -re "(?p).${toeol}" { exp_continue } + timeout { + send_user "Failure: time out getting started\n" + exit 2 + } +} + +if { ${ntests} == 0 } { + sus "System did not report plan; will look for summary at end" +} else { + sus "Expecting ${ntests} test results" +} + +set timeout 60 +set exitwith 0 +set failures 0 +for {set this 1} {${ntests} == 0 || ${this} <= ${ntests}} {incr this} { + expect { + -i ${victim} -re "${tpfx}#${toeol}" { + sud "Harness got comment: ${expect_out(buffer)}" + exp_continue + } + -i ${victim} -re "${tpfx}ok (\\d+)\\D${toeol}" { + sud "Harness acknowledge OK! ${this} ${expect_out(1,string)}" + set tid ${expect_out(1,string)} + if { ${tid} != "" && ${tid} != ${this} } { + sui "WARNING: Test reporting misaligned at ${this} (got ${tid})" + } + } + -i ${victim} -re "${tpfx}ok #${toeol}" { + sud "Harness acknowledge anonymous ok! ${this}" + } + -i ${victim} -re "${tpfx}not ok (\\d+)\\D${toeol}" { + sud "Failure in simulation after ${this} ${expect_out(1,string)}" + set tid ${expect_out(1,string)} + if { ${tid} != "" && ${tid} != ${this} } { + sui "WARNING: Test reporting misaligned at ${this}" + } + set exitwith [expr max(${exitwith},1)] + incr failures + } + -i ${victim} -re "${tpfx}not ok #${toeol}" { + sud "Failure (anonymous) in simulation after ${this}" + set exitwith [expr max(${exitwith},1)] + incr failures + } + -i ${victim} -re "${tpfx}Bail out!${toeol}" { + sus "Bail out after ${this} tests" + exit 2 + } + -i ${victim} -re "${tpfx}POST 1\\.\\.(\\d+)(?=\r?\n)" { + # A post-factual plan; this must be the end of testing + global ntests + set ntests ${expect_out(1,string)} + if { ${ntests} != ${this} } { + sus "Postfix plan claimed ${ntests} but we saw ${this}" + set exitwith [expr max(${exitwith},2)] + incr failures + } + # break out of for loop + set this ${ntests} + } + -i ${victim} -re "${tpfx}${toeol}" { + sus "TAP line not understood!" + exit 2 + } + # -i ${victim} -ex ${::expectnmcu::core::promptstr} { + # sus "Prompt while running tests!" + # exit 2 + # } + -i ${victim} -re ${::expectnmcu::core::panicre} { + sus "Panic!" + exit 2 + } + # Consume other outputs and discard as if they were comments + # This must come as the last pattern that looks at input + -re "(?p).${toeol}" { exp_continue } + timeout { + send_user "Failure: time out\n" + exit 2 + } + } +} + +# We think we're done running tests; send a final command for synchronization +send -i ${victim} "print(\"f\",\"i\",\"n\")\n" +expect -i ${victim} -re "print\\(\"f\",\"i\",\"n\"\\)\[\r\n\]+" { } +expect { + -i ${victim} -ex "f\ti\tn" { } + + -i ${victim} -re "${tpfx}#${toeol}" { + sud "Harness got comment: ${expect_out(buffer)}" + exp_continue + } + + -i ${victim} -re "${tpfx}Bail out!${toeol}" { + sus "Bail out after all tests finished" + exit 2 + } + -i ${victim} -re "${tpfx}${toeol}" { + sus "Unexpected TAP output after tests finished" + exit 2 + } + -i ${victim} -re ${::expectnmcu::core::panicre} { + sus "Panic!" + exit 2 + } + + -re "(?p).${toeol}" { exp_continue } + timeout { + send_user "Failure: time out\n" + exit 2 + } +} + +if { ${exitwith} == 0 } { + sus "All tests reported in OK" +} else { + sus "${failures} TEST FAILURES; REVIEW LOGS" +} +exit ${exitwith} diff --git a/tests/utils/NTestTapOut.lua b/tests/utils/NTestTapOut.lua new file mode 100644 index 00000000..6adfc674 --- /dev/null +++ b/tests/utils/NTestTapOut.lua @@ -0,0 +1,30 @@ +-- This is a NTest output handler that formats its output in a way that +-- resembles the Test Anything Protocol (though prefixed with "TAP: " so we can +-- more readily find it in comingled output streams). + +local nrun +return function(e, test, msg, err) + msg = msg or "" + err = err or "" + if e == "pass" then + print(("\nTAP: ok %d %s # %s"):format(nrun, test, msg)) + nrun = nrun + 1 + elseif e == "fail" then + print(("\nTAP: not ok %d %s # %s: %s"):format(nrun, test, msg, err)) + nrun = nrun + 1 + elseif e == "except" then + print(("\nTAP: not ok %d %s # exn; %s: %s"):format(nrun, test, msg, err)) + nrun = nrun + 1 + elseif e == "abort" then + print(("\nTAP: Bail out! %d %s # exn; %s: %s"):format(nrun, test, msg, err)) + elseif e == "start" then + -- We don't know how many tests we plan to run, so emit a comment instead + print(("\nTAP: # STARTUP %s"):format(test)) + nrun = 1 + elseif e == "finish" then + -- Ah, now, here we go; we know how many tests we ran, so signal completion + print(("\nTAP: POST 1..%d"):format(nrun)) + elseif #msg ~= 0 or #err ~= 0 then + print(("\nTAP: # %s: %s: %s"):format(test, msg, err)) + end +end