Rewrite sntp in Lua with only a little C

app/modules/sntppkt.c

@@ -0,0 +1,403 @@
+/*
+ * Copyright 2015 Dius Computing Pty Ltd. All rights reserved.
+ * Copyright 2020 Nathaniel Wesley Filardo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the
+ *   distribution.
+ * - Neither the name of the copyright holders nor the names of
+ *   its contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @author Johny Mattsson <jmattsson@dius.com.au>
+ * @author Nathaniel Wesley Filardo <nwfilardo@gmail.com>
+ */
+
+// Module for Simple Network Time Protocol (SNTP) packet processing;
+// see lua_modules/sntp/sntp.lua for the user-friendly bits of this.
+
+#include "module.h"
+#include "lauxlib.h"
+#include "os_type.h"
+#include "osapi.h"
+#include "lwip/udp.h"
+#include <stdlib.h>
+#include "user_modules.h"
+#include "lwip/dns.h"
+
+#define max(a,b) ((a < b) ? b : a)
+
+#define NTP_PORT 123
+#define NTP_ANYCAST_ADDR(dst)  IP4_ADDR(dst, 224, 0, 1, 1)
+
+#if 0
+# define sntppkt_dbg(...) printf(__VA_ARGS__)
+#else
+# define sntppkt_dbg(...)
+#endif
+
+typedef struct
+{
+  uint32_t sec;
+  uint32_t frac;
+} ntp_timestamp_t;
+
+static const uint32_t NTP_TO_UNIX_EPOCH = 2208988800ul;
+
+typedef struct
+{
+  uint8_t mode : 3;
+  uint8_t ver : 3;
+  uint8_t LI : 2;
+  uint8_t stratum;
+  uint8_t poll;
+  uint8_t precision;
+  uint32_t delta_r;
+  uint32_t epsilon_r;
+  uint32_t refid;
+  ntp_timestamp_t ref;
+  ntp_timestamp_t origin;
+  ntp_timestamp_t recv;
+  ntp_timestamp_t xmit;
+} __attribute__((packed)) ntp_frame_t;
+
+#define NTP_RESPONSE_METATABLE  "sntppkt.resp"
+typedef struct {
+    /* Copied from incoming packet */
+    uint32_t  delta_r;
+    uint32_t  epsilon_r;
+    uint8_t   LI;
+    uint8_t   stratum;
+
+    /* Computed as per RFC 5905; units are 2^(-32) seconds */
+    int64_t   theta;
+    int64_t   delta;
+
+    /* Local computation */
+    uint64_t  t_rx;
+    uint32_t  score;
+} ntp_response_t;
+
+static const uint32_t MICROSECONDS      = 1000000;
+
+static uint64_t
+sntppkt_div1m(uint64_t n) {
+  uint64_t q1 = (n >> 5) + (n >> 10);
+  uint64_t q2 = (n >> 12) + (q1 >> 1);
+  uint64_t q3 = (q2 >> 11) - (q2 >> 23);
+
+  uint64_t q = n + q1 + q2 - q3;
+
+  q = q >> 20;
+
+  // Ignore the error term -- it is measured in pico seconds
+  return q;
+}
+
+static uint32_t
+sntppkt_us_to_frac(uint64_t us) {
+  return sntppkt_div1m(us << 32);
+}
+
+static uint32_t
+sntppkt_long_s(uint64_t time) {
+  return time >> 32;
+}
+
+static uint32_t
+sntppkt_long_us(uint64_t time) {
+  return ((time & 0xFFFFFFFF) * MICROSECONDS) >> 32;
+}
+
+/*
+ * Convert sec/usec to a Lua string suitable for depositing into a SNTP packet
+ * buffer.  This is a little gross, but it's not the worst thing a C
+ * programmer's ever done, I'm sure.
+ */
+static int
+sntppkt_make_ts(lua_State *L) {
+  ntp_timestamp_t ts;
+
+  ts.sec  = htonl(luaL_checkinteger(L, 1) + NTP_TO_UNIX_EPOCH) ;
+
+  uint32_t usec = luaL_checkinteger(L, 2)  ;
+  ts.frac = htonl(sntppkt_us_to_frac(usec));
+
+  lua_pushlstring(L, (const char *)&ts, sizeof(ts));
+  return 1;
+}
+
+/*
+ * Convert ntp_timestamp_t to uint64_t
+ */
+static uint64_t
+sntppkt_ts2uint64(ntp_timestamp_t ts) {
+  return (((uint64_t)(ts.sec)) << 32) + (uint64_t)(ts.frac);
+}
+
+/*
+ * Process a SNTP packet as contained in a Lua string, given a cookie timestamp
+ * and local clock second*usecond pair.  Generates a ntp_response_t userdata
+ * for later processing or a string if the server is telling us to go away.
+ *
+ * :: string (packet)
+ * -> string (cookie)
+ * -> int (local clock, sec component)
+ * -> int (local clock, usec component)
+ * -> sntppkt.resp
+ *
+ */
+static int
+sntppkt_proc_pkt(lua_State *L) {
+  size_t       pkts_len;
+  ntp_frame_t  pktb;
+  const char  *pkts = lua_tolstring(L, 1, &pkts_len);
+
+  luaL_argcheck(L, pkts && pkts_len == sizeof(pktb), 1, "Bad packet");
+
+  // make sure we have an aligned copy to work from
+  memcpy (&pktb, pkts, sizeof(pktb));
+
+  uint32_t now_sec  = luaL_checkinteger(L, 3);
+  uint32_t now_usec = luaL_checkinteger(L, 4);
+
+  size_t           cookie_len;
+  ntp_timestamp_t *cookie = (ntp_timestamp_t*) lua_tolstring(L, 2, &cookie_len);
+
+  /* Bad *length* is clearly something bogus */
+  luaL_argcheck(L, cookie && cookie_len == sizeof(*cookie), 2, "Bad cookie");
+
+  /* But mismatching value might just be a packet caught in the crossfire */
+  if (memcmp((const char *)cookie, (const char *)&pktb.origin, sizeof (*cookie))) {
+    /* bad cookie; return nil */
+    return 0;
+  }
+
+  /* KOD?  Do this test *after* checking the cookie */
+  if (pktb.LI == 3) {
+    lua_pushlstring(L, (const char *)&pktb.refid, 4);
+    return 1;
+  }
+
+  ntp_response_t *ntpr = lua_newuserdata(L, sizeof(ntp_response_t));
+  luaL_getmetatable(L, NTP_RESPONSE_METATABLE);
+  lua_setmetatable(L, -2);
+
+  ntpr->LI        = pktb.LI;
+  ntpr->stratum   = pktb.stratum;
+
+  // NTP Short Format: 16 bit seconds, 16 bit fraction
+  ntpr->delta_r   = ntohl(pktb.delta_r);
+  ntpr->epsilon_r = ntohl(pktb.epsilon_r);
+
+  /* Heavy time lifting time */
+
+  // NTP Long Format: 32 bit seconds, 32 bit fraction
+  pktb.origin.sec  = ntohl(pktb.origin.sec);
+  pktb.origin.frac = ntohl(pktb.origin.frac);
+  pktb.recv.sec    = ntohl(pktb.recv.sec);
+  pktb.recv.frac   = ntohl(pktb.recv.frac);
+  pktb.xmit.sec    = ntohl(pktb.xmit.sec);
+  pktb.xmit.frac   = ntohl(pktb.xmit.frac);
+
+  uint64_t ntp_origin = sntppkt_ts2uint64(pktb.origin); // We sent it
+  uint64_t ntp_recv   = sntppkt_ts2uint64(pktb.recv);   // They got it
+  uint64_t ntp_xmit   = sntppkt_ts2uint64(pktb.xmit);   // They replied
+
+  // When we got it back (our clock)
+  uint64_t ntp_dest   = (((uint64_t) now_sec + NTP_TO_UNIX_EPOCH ) << 32)
+                        + sntppkt_us_to_frac(now_usec);
+
+  ntpr->t_rx = ntp_dest;
+
+              // |       outgoing offset           |   |        incoming offset         |
+  ntpr->theta = (int64_t)(ntp_recv - ntp_origin) / 2 + (int64_t)(ntp_xmit - ntp_dest) / 2;
+
+              // |       our clock delta           |   |       their clock delta        |
+  ntpr->delta = (int64_t)(ntp_dest - ntp_origin) / 2 + (int64_t)(ntp_xmit - ntp_recv) / 2;
+
+  /* Used by sntppkt_resp_pick; bias towards closer clocks */
+  ntpr->score = ntpr->delta_r * 2 + ntpr->delta;
+
+  sntppkt_dbg("SNTPPKT n_o=%llx n_r=%llx n_x=%llx n_d=%llx th=%llx d=%llx\n",
+    ntp_origin, ntp_recv, ntp_xmit, ntp_dest, ntpr->theta, ntpr->delta);
+
+  return 1;
+}
+
+/*
+ * Left-biased selector of a "preferred" NTP response.  Note that preference
+ * is rather subjective!
+ *
+ * Returns true iff we'd prefer the second response to the first.
+ *
+ * :: sntppkt.resp -> sntppkt.resp -> boolean
+ */
+
+static int
+sntppkt_resp_pick(lua_State *L) {
+  ntp_response_t *a = luaL_checkudata(L, 1, NTP_RESPONSE_METATABLE);
+  ntp_response_t *b = luaL_checkudata(L, 2, NTP_RESPONSE_METATABLE);
+  int biased = 0;
+
+  biased = lua_toboolean(L, 3);
+
+  /*
+   * If we're "biased", prefer the second structure if the score is less than
+   * 3/4ths of the score of the first.  An unbiased comparison just uses the
+   * raw score values.
+   */
+  if (biased) {
+    lua_pushboolean(L, a->score * 3 > b->score * 4);
+  } else {
+    lua_pushboolean(L, a->score     > b->score    );
+  }
+  return 1;
+}
+
+static void
+field_from_integer(lua_State *L, const char * field_name, lua_Integer value) {
+  lua_pushinteger(L, value);
+  lua_setfield(L, -2, field_name);
+}
+
+/*
+ * Inflate a NTP response into a Lua table
+ *
+ * :: sntppkt.resp -> { }
+ */
+static int
+sntppkt_resp_totable(lua_State *L) {
+  ntp_response_t *r = luaL_checkudata(L, 1, NTP_RESPONSE_METATABLE);
+
+  lua_createtable(L, 0, 6);
+
+  sntppkt_dbg("SNTPPKT READ th=%llx\n", r->theta);
+
+  /*
+   * The raw response ends up in here, too, so that we can pass the
+   * tabular view to user callbacks and still have the internal large integers
+   * for use in drift_compensate.
+   */
+  lua_pushvalue(L, 1);
+  lua_setfield(L, -2, "raw");
+
+  field_from_integer(L, "theta_s",  (int32_t)sntppkt_long_s (r->theta));
+  field_from_integer(L, "theta_us", sntppkt_long_us(r->theta));
+
+  field_from_integer(L, "delta", r->delta >> 16);
+  field_from_integer(L, "delta_r", r->delta_r);
+  field_from_integer(L, "epsilon_r", r->epsilon_r);
+
+  field_from_integer(L, "leapind", r->LI);
+  field_from_integer(L, "stratum", r->stratum);
+  field_from_integer(L, "rx_s"   , sntppkt_long_s (r->t_rx) - NTP_TO_UNIX_EPOCH);
+  field_from_integer(L, "rx_us"  , sntppkt_long_us(r->t_rx));
+
+  return 1;
+}
+
+/*
+ * Compute local RTC drift rate given a SNTP response, previous sample time,
+ * and error integral value (i.e., this is a PI controller).  Returns new rate
+ * and integral value.
+ *
+ * Likely only sensible if resp->theta is sufficiently small (so we require
+ * less than a quarter of a second) and the inter-sample duration must, of
+ * course, be positive.
+ *
+ * There's nothing magic about the constants of the PI loop here; someone with
+ * a better understanding of control theory is welcome to suggest improvements.
+ *
+ * :: sntppkt.resp (most recent sample)
+ * -> sntppkt.resp (prior sample)
+ * -> int (integral)
+ * -> int (rate), int (integral)
+ *
+ */
+static int
+sntppkt_resp_drift_compensate(lua_State *L) {
+  ntp_response_t *resp = luaL_checkudata(L, 1, NTP_RESPONSE_METATABLE);
+
+  int32_t tsec = sntppkt_long_s(resp->theta);
+  uint32_t tfrac = resp->theta & 0xFFFFFFFF;
+  if ((tsec != 0 && tsec != -1) ||
+      ((tsec ==  0) && (tfrac > 0x40000000UL)) ||
+      ((tsec == -1) && (tfrac < 0xC0000000UL))) {
+    return luaL_error(L, "Large deviation");
+  }
+
+  int32_t dsec = sntppkt_long_s(resp->delta);
+  if (dsec != 0 && dsec != -1) {
+    return luaL_error(L, "Large estimated error");
+  }
+
+  ntp_response_t *prior_resp = luaL_checkudata(L, 2, NTP_RESPONSE_METATABLE);
+
+  int64_t  isdur    = resp->t_rx - prior_resp->t_rx;
+  if (isdur <= 0) {
+    return luaL_error(L, "Negative time base");
+  }
+
+  int32_t  err_int  = luaL_checkinteger(L, 3);
+
+  /* P: Compute drift over 2, in parts per 2^32 as expected by the RTC. */
+  int32_t drift2 = (resp->theta << 31) / isdur;
+
+  /* PI: rate is drift/4 + integral error */
+  int32_t newrate = (drift2 >> 1) + err_int;
+
+  /* I: Adjust integral by drift/32, with a little bit of wind-up protection */
+  if ((newrate > ~0x40000) && (newrate < 0x40000)) {
+    err_int += (drift2 + 0xF) >> 4;
+  }
+
+  sntppkt_dbg("SNTPPKT drift: isdur=%llx drift2=%lx -> newrate=%lx err_int=%lx\n",
+              isdur, drift2, newrate, err_int);
+
+  lua_pushinteger(L, newrate);
+  lua_pushinteger(L, err_int);
+  return 2;
+}
+
+LROT_BEGIN(sntppkt_resp, NULL, LROT_MASK_INDEX)
+  LROT_TABENTRY( __index, sntppkt_resp )
+  LROT_FUNCENTRY( pick, sntppkt_resp_pick )
+  LROT_FUNCENTRY( totable, sntppkt_resp_totable )
+  LROT_FUNCENTRY( drift_compensate, sntppkt_resp_drift_compensate )
+LROT_END(sntppkt_resp, NULL, LROT_MASK_INDEX)
+
+static int
+sntppkt_init(lua_State *L)
+{
+  luaL_rometatable(L, NTP_RESPONSE_METATABLE, LROT_TABLEREF(sntppkt_resp));
+  return 0;
+}
+
+// Module function map
+LROT_BEGIN(sntppkt, NULL, 0)
+  LROT_FUNCENTRY( make_ts  , sntppkt_make_ts   )
+  LROT_FUNCENTRY( proc_pkt , sntppkt_proc_pkt  )
+LROT_END( sntppkt, NULL, 0 )
+
+NODEMCU_MODULE(SNTPPKT, "sntppkt", sntppkt, sntppkt_init);

docs/lua-modules/sntp.md

@@ -0,0 +1,239 @@
+# SNTP Module
+| Since  | Origin / Contributor  | Maintainer  | Source  |
+| :----- | :-------------------- | :---------- | :------ |
+| 2019-07-01 | [nwf](https://github.com/nwf) | [nwf](https://github.com/nwf) | [sntp.lua](../../lua_modules/sntp/sntp.lua) |
+
+This is a user-friendly, Lua wrapper around the `sntppkt` module to facilitate
+the use of SNTP.
+
+!!! note
+
+       This `sntp` module is expected to live in LFS, and so all the documentation
+       uses `node.flashindex()` to find the module.  If the module is instead
+       loaded as a file, use, for example, `(require "sntp").go()` instead, but
+       note that this may consume a large amount of heap.
+
+## Default Constructor Wrapper
+
+The simplest use case is to use the `go` function with no arguments like this:
+
+```lua
+node.flashindex("sntp")().go()
+```
+
+This will...
+
+  * use `rtctime` to interface with the local clock;
+
+  * use a default list of servers, including the server provided by the local
+    DHCP server, if any;
+
+  * re-synchronize the clock every half-hour in the steady state;
+
+  * attempt to discipline the local oscillator to improve timing accuracy (but
+    note that this may take a while to converge; advanced users may wish to
+    manually persist drift rate across reboots using the `rtctime` interface);
+    and
+
+  * rapidly re-discipline the local oscillator when a large deviance is
+    observed (e.g., at cold boot)
+
+The sntp object encapsulating the state machine is returned, but this may be
+safely ignored in simple use cases.  (This object is augmented with `tmr`, the
+timer used to manage resychronization, should you wish to dynamically start,
+stop, or delay synchronization.)
+
+You can provide your own list of servers, too, overriding the default:
+
+```lua
+node.flashindex("sntp")().go({ "ntp-server-1", "ntp-server-2" })
+```
+
+Or, to just use the server provided by DHCP and no others, when one is certain
+that the DHCP server will provide one,
+
+```lua
+do
+  local ifi = net.ifinfo(0)
+  local srv = ifi and ifi.dhcp and ifi.dhcp.ntp_server
+  if srv then node.flashindex("sntp")().go({srv}) end
+end
+```
+
+Similarly, the frequency of synchronization can be changed:
+
+```lua
+-- use default servers and synchronize every ten minutes
+node.flashindex("sntp")().go(nil, 600000)
+```
+
+Success and failure callbacks can be given as well, for advanced use or
+increased reporting:
+
+```lua
+node.flashindex("sntp")().go(nil, 1200000,
+  function(res, serv, self)
+    print("SNTP OK", serv, res.theta_s, res.theta_us, rtctime.get())
+  end,
+  function(err, srv, rply)
+    if     err == "all"    then print("SNTP FAIL", srv)
+    elseif err == "kod"    then print("SNTP server kiss of death", srv, rply)
+    elseif err == "goaway" then print("SNTP server rejected us", srv, rply)
+    else                        print("SNTP server unreachable", srv, err)
+    end
+  end)
+```
+
+Details of the parameters to the callbacks are given below.  The remainder of
+this document details increasingly internal details, and is likely of
+decreasing interest to general audiences.
+
+#### Syntax
+`node.flashindex("sntp")().go([servers, [frequency, [success_cb, [failure_cb]]]])`
+
+### Notes
+
+#### Interaction with the Garbage Collector
+
+As our network stack does not capture the time of received packets (nor does
+it know how to timestamp egressing NTP packets as dedicated hardware does),
+there is a fair bit of local processing delay, as we have to come into Lua
+to get the local timestamps.  For higher-precision time keeping, if
+possible, it may help to move the device to a (E)GC mode which has more slop
+than the default, which prioritizes prompt reclaim of memory.  Consider, for
+example, something like `node.egc.setmode(node.egc.ON_MEM_LIMIT, -4096)` to
+permit the SNTP logic to run (more often) without interference of the GC.
+(The `go` function above defaults to collecting garbage before triggering
+SNTP synchronization.)
+
+## SNTP Object Constructor
+```lua
+sntp = node.flashindex("sntp")().new(servers, success_cb, [failure_cb], [clock])
+```
+
+where
+
+* `servers` specifies the name(s) of the (S)NTP server(s) to use; it may be...
+
+    * a string, either a DNS name or an IPv4 address in dotted quad form,
+    * an array of the above
+    * `nil` to use any DHCP-provided NTP server and some default
+      `*.nodemcu.pool.ntp.org` servers.
+
+* `success_cb` is called back at the end of a synchronization when at least one
+  server replied to us.  It will be given three arguments:
+
+    * the preferred SNTP result converted to a a table; see `sntppkt.res.totable` below.
+    * the name of the server whence that result came
+    * the `sntp` object itself
+
+* `failure_cb` may be `nil` but, otherwise, is called back in two circumstances:
+
+    * at the end of a pass during which no server could be reached.  In this case,
+      the first argument will be the string "all", the second will be the
+      number of servers tried, and the third will be the `sntp` object itself.
+
+    * an individual server has failed in some way.  In this case, the first
+      argument will be one of:
+
+        * "dns" (if name resolution failed),
+        * "timeout" (if the server failed to reply in time),
+        * "goaway" (if the server refused to answer), or
+        * "kod" ("kiss of death", if the server told us to stop contacting it entirely).
+
+      In all cases, the name of the server is the second argument and the `sntp`
+      object itself is the third; in the "goaway" case, the fourth argument will
+      contain the refusal string (e.g., "RATE" for rate-limiting or "DENY" for
+      kiss-of-death warnings.
+
+      In the case of kiss-of-death packets, the server will be removed from all
+      subsequent syncs.  This may result in there eventually being no servers to
+      contact.  Paranoid applications should therefore monitor failures!
+
+* `clock`, if given, should return two values describing the local clock in
+  seconds and microseconds (between 0 and 1000000).  If not given, the module
+  will fall back on `rtctime.get`; if `rtctime` is not available, a clock must
+  be provided.  Using `function() return 0, 0 end` will result in the "clock
+  offset" (`theta`) reported by the success callback being a direct estimate of
+  the true time.
+
+## Other module methods
+
+The module contains some other utility functions beyond the SNTP object
+constructor and the `go` utility function detailed above.
+
+### update_rtc()
+Given a result from a SNTP `sync` pass, update the local RTC through `rtctime`.
+Attempting to use this function without `rtctime` support will raise an error.
+
+`sntpobj` is used to track state between syncs and should be the "sntp object"
+given in the success and failure callbacks.  (In principle, any Lua table will
+do, but that is the most convenient one.  All data is passed using fields whose
+keys are strings with prefix `rtc_`.)
+
+#### Syntax
+`node.flashindex("sntp")().update_rtc(res, sntpobj)`
+
+## SNTP object methods
+
+### sync()
+
+Run a pass through the specified servers and call back as described above.
+
+#### Syntax
+`sntpobj:sync()`
+
+### stop()
+
+Abort any pass in progress; no more callbacks will be called.  The current
+preferred response and server name (i.e., the arguments to the success
+callback, should the pass end now) are returned.
+
+#### Syntax
+`sntpobj:stop()`
+
+### servers
+The table of NTP servers currently being used.  Please treat this as
+read-only.  This may be investigated to see if kiss-of-death processing
+has removed any servers, but one is probably better off listening for
+the failure callback notifications.
+
+## Internal Details: sntppkt response object methods
+
+### sntppkt.resp.totable()
+
+Expose a `sntppkt.resp` result as a Lua table with the following fields:
+
+* `theta_s`: Local clock offset, seconds component
+* `theta_us`: Local clock offset, microseconds component
+
+* `delta`: An estimate of the error, in 65536ths of a second (i.e., approx 15.3 microseconds)
+* `delta_r`: The server's estimate of its error, in 65536ths of a second
+* `epsilon_r`: The server's estimate of its dispersion, in 65536ths of a second
+
+* `leapind`: The leap-second indicator
+* `stratum`: The server's stratum
+
+* `rx_s`: Packet reception timestamp, seconds component
+* `rx_us`: Packet reception timestamp, microseconds component
+
+* `raw`: The `sntppkt.resp` itself, so that we can pass this table around to
+  user Lua code and still retain access to the raw internals in, for example,
+  `drift_compensate`, below.  See the use in `update_rtc`.
+
+Note that negative offsets will be represented with a negative `theta_s` and
+a *positive* `theta_us`.  For example, -200 microseconds would be -1 seconds
+and 999800 microseconds.
+
+#### Syntax
+`res:totable()`
+
+### sntppkt.resp.pick()
+
+Used internally to select among multiple responses; see source for usage.
+
+### sntppkt.resp.drift_compensate()
+
+Encapsulates a Proportional-Integral (PI) controller update step for use in
+disciplining the local oscillator.  Used internally to `update_rtc`; see source
+for usage.

lua_modules/sntp/sntp.lua

@@ -0,0 +1,294 @@
+-- Constructor
+--   sc and fc are our Success and Failure Callbacks, resp.
+local new = function(serv, sc, fc, now)
+
+  if     type(serv) == "string" then serv = {serv}
+  elseif serv == nil            then serv =
+    {
+      nil,
+      "1.nodemcu.pool.ntp.org",
+      "2.nodemcu.pool.ntp.org",
+    }
+    local ni = net.ifinfo(0)
+    ni = ni and ni.dhcp
+    serv[1] = ni.ntp_server or "0.nodemcu.pool.ntp.org"
+  elseif type(serv) ~= "table"  then error "Bad server table"
+  end
+
+  if type(sc) ~= "function" then error "Bad success callback type" end
+  if fc ~= nil and type(fc) ~= "function" then error "Bad failure callback type" end
+  if now ~= nil and type(now) ~= "function" then error "Bad clock type" end
+
+  now = now or (rtctime and rtctime.get)
+  if now == nil then error "Need clock function" end
+
+  local _self = {servers = serv}
+
+  local _tmr -- contains the currently running timer, if any
+  local _udp -- the socket we're using to talk to the world
+
+  local _kod -- kiss of death flags accumulated accoss syncs
+  local _pbest -- best server from prior pass
+
+  local _res -- the best result we've got so far this pass
+  local _best -- best server this pass, for updating _pbest
+
+  local _six -- index of the server in serv to whom we are speaking
+  local _sat -- number of times we've tried to reach this server
+
+  -- Shut down the state machine
+  --
+  -- upvals: _tmr, _udp, _six, _sat, _res, _best
+  local function _stop()
+    -- stop any time-based callbacks and drop _tmr
+    _tmr = _tmr and _tmr:unregister()
+
+    _six, _sat, _res, _best = nil, nil, nil, nil
+
+    -- stop any UDP callbacks and drop the socket; to be safe against
+    -- knots tied in the registry, explicitly unregister callbacks first
+    if _udp then
+      _udp:on("receive", nil)
+      _udp:on("sent"   , nil)
+      _udp:on("dns"    , nil)
+      _udp:close()
+      _udp = nil
+    end
+
+    -- Count down _kod entries
+    if _kod then
+      for k,v in pairs(_kod) do _kod[k] = (v > 0) and (v - 1) or nil end
+      if #_kod == 0 then _kod = nil end
+    end
+  end
+
+  local nextServer
+  local doserver
+
+  -- Try communicating with the current server
+  --
+  -- upvals: now, _tmr, _udp, _best, _kod, _pbest, _res, _six
+  local function hail(ip)
+    _tmr:alarm(5000 --[[const param: SNTP_TIMEOUT]], tmr.ALARM_SINGLE, function()
+      _udp:on("sent", nil)
+      _udp:on("receive", nil)
+      return doserver("timeout")
+    end)
+
+    local txts = sntppkt.make_ts(now())
+
+    _udp:on("receive",
+     -- upvals: now, ip, txts, _tmr, _best, _kod, _pbest, _res, _six
+     function(skt, d, port, rxip)
+      -- many things constitute bad packets; drop with tmr running
+      if rxip ~= ip and ip ~= "224.0.1.1" then return end   -- wrong peer (unless multicast)
+      if port ~= 123 then return end                        -- wrong port
+      if #d   <   48 then return end                        -- too short
+
+      local pok, pkt = pcall(sntppkt.proc_pkt, d, txts, now())
+
+      if not pok or pkt == nil then
+        -- sntppkt can also reject the packet for having a bad cookie;
+        -- this is important to prevent processing spurious or delayed responses
+        return
+      end
+
+      _tmr:unregister()
+      skt:on("receive", nil) -- skt == _udp
+      skt:on("sent", nil)
+
+      if type(pkt) == "string" then
+        if pkt == "DENY" then -- KoD packet
+
+          if _kod and _kod[rxip] then
+            -- There was already a strike against this IP address, and now
+            -- it's permanent.  We can't directly remove the IP from rotation,
+            -- but we can remove the DNS that's resolving to it, which isn't
+            -- great, but isn't the worst either.
+            if fc then fc("kod", serv[_six], _self) end
+            _kod[rxip] = nil
+            table.remove(serv, _six)
+            _six = _six - 1 -- nextServer will add one
+          else
+            _kod = _kod or {}
+            _kod[rxip] = 2
+            if fc then fc("goaway", serv[_six], _self, pkt) end
+          end
+        else
+          if fc then fc("goaway", serv[_six], _self, pkt) end
+        end
+        return nextServer()
+      end
+
+      if _pbest == serv[_six] then
+        -- this was our favorite server last time; if we don't have a
+        -- result or if we'd rather this one than the result we have...
+        if not _res or not pkt:pick(_res, true) then
+          _res = pkt
+          _best = _pbest
+        end
+      else
+        -- this was not our favorite server; take this result if we have no
+        -- other option or if it compares favorably to the one we have, which
+        -- might be from our favorite from last pass.
+        if not _res or _res:pick(pkt, _pbest == _best) then
+          _res = pkt
+          _best = serv[_six]
+        end
+      end
+
+      return nextServer()
+     end)
+
+    return _udp:send(123, ip,
+      -- '#' == 0x23: version 4, mode 3 (client), no LI
+      "#\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
+      .. txts)
+  end
+
+  -- upvals: _sat, _six, _udp, hail, _self
+  function doserver(err)
+    if _sat == 2 --[[const param: MAX_SERVER_ATTEMPTS]] then
+      if fc then fc(err, serv[_six], _self) end
+      return nextServer()
+    end
+    _sat = _sat + 1
+
+    return _udp:dns(serv[_six], function(skt, ip)
+      skt:on("dns", nil) -- skt == _udp
+      if ip == nil then return doserver("dns") else return hail(ip) end
+    end)
+  end
+
+  -- Move on to the next server or finish a pass
+  --
+  -- upvals: fc, serv, sc, _best, _pbest, _res, _sat, _six
+  function nextServer()
+    if _six >= #serv then
+     if _res then
+       _pbest = _best
+       local res = _res
+       local best = _best
+       _stop()
+       return sc(res:totable(), best, _self)
+     else
+       _stop()
+       if fc then return fc("all", #serv, _self) else return end
+     end
+    end
+
+    _six = _six + 1
+    _sat = 0
+    return doserver()
+  end
+
+  -- Poke all the servers and invoke the user's callbacks
+  --
+  -- upvals: _stop, _udp, _ENV, _tmr, _six, nextServer
+  function _self.sync()
+    _stop()
+    _udp = net.createUDPSocket()
+    _tmr = tmr.create()
+    _udp:listen() -- on random port
+    _six = 0
+    nextServer()
+  end
+
+  function _self.stop()
+    local res, best = _res, _best
+    _stop()
+    return res and res:totable(), best
+  end
+
+  return _self
+
+end
+
+-- A utility function which applies a result to the rtc
+local update_rtc = function(res, obj)
+  local rate = nil
+  if obj.rtc_last ~= nil then
+    -- adjust drift compensation.  We have three pieces of information:
+    --
+    --   our idea of time at rx (res.rx_*),
+    --   our idea of time at the last sync (obj.rtc_last.rx_*)
+    --   the measured theta now (res.theta_us)
+    --
+    -- We're going to integrate the theta signal over time and use
+    -- that to mediate the rate we set, making this a PI controller,
+    -- but we might take big steps if theta gets too bad.
+    local ok, err_int
+    local raw = res.raw
+    ok, rate, err_int = pcall(raw.drift_compensate, raw, obj.rtc_last,
+                              obj.rtc_err_int or 0)
+    if not ok then
+      rate = nil -- don't set the rate this time
+      obj.rtc_last = nil -- or next time
+    else
+      obj.rtc_last = res.raw
+      obj.rtc_err_int = err_int
+    end
+  else
+    obj.rtc_last = res.raw
+  end
+
+  if rate == nil then
+    -- update time (and cut rate, in case it's gotten out of hand)
+    local now_s, now_us, now_r = rtctime.get()
+    local new_s, new_us = now_s + res.theta_s, now_us + res.theta_us
+    if new_us > 1000000 then
+      new_s  = new_s  + 1
+      new_us = new_us - 1000000
+    end
+    rtctime.set(new_s, new_us, now_r / 2)
+  else
+    -- just change the rate
+    rtctime.set(nil, nil, rate)
+  end
+
+  return rate ~= nil
+end
+
+-- Default operation
+--
+-- upvals: new, update_rtc
+local go = function(servs, period, sc, fc)
+  local sntpobj = new(servs,
+    -- wrap the success callback with a utility function for managing the rtc
+    -- and polling frequency
+    function(res, serv, self)
+      local ok = update_rtc(res, self)
+
+      -- if the rate estimator thinks it has this under control, only poll
+      -- the server occasionally.  Otherwise, bother it more frequently,
+      -- in a "bursty" way
+      if ok and ((self.rtc_burst or 0) == 0)
+       then self.tmr:interval(period or 1800000)
+            self.rtc_burst = nil
+       else self.tmr:interval(30000)
+            self.rtc_burst = (ok and self.rtc_burst or 40) - 1
+      end
+
+      -- invoke the user's callback
+      if sc then return sc(res, serv, self) end
+    end,
+    fc)
+
+  local t = tmr.create()
+  sntpobj.tmr = t
+  t:alarm(60000, tmr.ALARM_AUTO, function() collectgarbage() sntpobj.sync() end)
+  sntpobj.sync()
+
+  return sntpobj
+end
+
+-- from sntppkt
+-- luacheck: ignore
+local _lfs_strings = "theta_s", "theta_us", "delta", "delta_r", "epsilon_r",
+  "leapind", "stratum", "rx_s", "rx_us"
+
+return {
+update_rtc = update_rtc,
+new        = new,
+go         = go,
+}

mkdocs.yml

@@ -67,6 +67,7 @@ pages:
       - 'mcp23008': 'lua-modules/mcp23008.md'
       - 'mcp23017': 'lua-modules/mcp23017.md'
       - 'redis': 'lua-modules/redis.md'
+      - 'sntp' : 'lua-modules/sntp.md'
       - 'telnet': 'lua-modules/telnet.md'
       - 'yeelink': 'lua-modules/yeelink.md'
   - C Modules:
@@ -120,6 +121,7 @@ pages:
       - 'sigma delta': 'modules/sigma-delta.md'
       - 'sjson': 'modules/sjson.md'
       - 'sntp': 'modules/sntp.md'
+      # sntppkt deliberately not documented
       - 'softuart': 'modules/softuart.md'
       - 'somfy': 'modules/somfy.md'
       - 'spi': 'modules/spi.md'

tools/luacheck_config.lua

@@ -619,6 +619,12 @@ stds.nodemcu_libs = {
         sync = empty
       }
     },
+    sntppkt = {
+      fields = {
+        make_ts = empty,
+        proc_pkt = empty,
+      }
+    },
     somfy = {
       fields = {
         DOWN = empty,