#589 WIP reboot recovery

supervisor/facility.lua

@@ -80,7 +80,7 @@ function facility.new(config)
         -- process control
         recovery = RCV_STATE.INACTIVE,  ---@type RECOVERY_STATE
         recovery_boot_state = nil,      ---@type sv_control_state|nil
-        last_unit_states = nil,         ---@type boolean[]
+        last_unit_states = {},          ---@type boolean[]
         units_ready = false,
         mode = PROCESS.INACTIVE,        ---@type PROCESS
         last_mode = PROCESS.INACTIVE,   ---@type PROCESS
@@ -166,6 +166,13 @@ function facility.new(config)
         table.insert(self.test_tone_states, false)
     end
 
+    -- init next boot state
+    settings.set("LastProcessState", PROCESS.INACTIVE)
+    settings.set("LastUnitStates", self.last_unit_states)
+    if not settings.save("/supervisor.settings") then
+        log.warning("FAC: failed to save initial control state into supervisor settings file")
+    end
+
     --#endregion
 
     -- PRIVATE FUNCTIONS --
@@ -316,19 +323,24 @@ function facility.new(config)
         if self.recovery == RCV_STATE.RUNNING then
             -- try to start auto control
             if self.recovery_boot_state.mode ~= nil and self.units_ready then
+                if self.recovery_boot_state.mode ~= PROCESS.INACTIVE and self.recovery_boot_state.mode ~= PROCESS.SYSTEM_ALARM_IDLE then
+                    self.mode = self.mode_set
+                    log.info("FAC: process startup resume initiated")
+                end
+
                 self.recovery_boot_state.mode = nil
-                self.mode = self.mode_set
-                log.info("FAC: process startup resume initiated")
             end
 
-            local recovered = self.recovery_boot_state.mode == nil
+            local recovered = self.recovery_boot_state.mode == nil or self.recovery_boot_state.mode == PROCESS.INACTIVE
 
             -- restore manual control reactors
             for i = 1, #self.units do
+                local u = self.units[i]
+
                 if self.recovery_boot_state.unit_states[i] and self.group_map[i] == AUTO_GROUP.MANUAL then
                     recovered = false
 
-                    if self.units[i].get_control_inf().ready then
+                    if u.get_control_inf().ready then
                         local plc_s = svsessions.get_reactor_session(i)
                         if plc_s ~= nil then
                             plc_s.in_queue.push_command(plc.PLC_S_CMDS.ENABLE)
@@ -344,7 +356,7 @@ function facility.new(config)
             if recovered then
                 self.recovery = RCV_STATE.STOPPED
                 self.recovery_boot_state = nil
-                log.info("FAC: startup resume complete")
+                log.info("FAC: startup resume sequence completed")
             end
         end
 
@@ -378,29 +390,45 @@ function facility.new(config)
 
     --#region Startup Recovery
 
+    -- on exit, use this to clear the boot state so we don't resume when exiting cleanly
+    function public.clear_boot_state()
+        settings.unset("LastProcessState")
+        settings.unset("LastUnitStates")
+
+        local saved = settings.save("/supervisor.settings")
+        if not saved then
+            log.warning("facility.clear_boot_state(): failed to save supervisor settings file")
+        else
+            log.debug("FAC: cleared boot state on exit")
+        end
+    end
+
+    -- initialize startup recovery
     ---@param state sv_control_state
     function public.startup_recovery_init(state)
         if self.recovery == RCV_STATE.INACTIVE then
             self.recovery_boot_state = state
             self.recovery = RCV_STATE.PRIMED
+            log.info("FAC: startup resume ready")
         end
     end
 
     -- attempt startup recovery
     ---@param auto_cfg start_auto_config configuration
     function public.startup_recovery_start(auto_cfg)
-        if self.recovery == RCV_STATE.PRIMED and self.recovery_boot_state and
+        if self.recovery == RCV_STATE.PRIMED then
-         self.recovery_boot_state.mode ~= PROCESS.INACTIVE and self.recovery_boot_state.mode ~= PROCESS.SYSTEM_ALARM_IDLE then
             self.recovery = util.trinary(_auto_check_and_save(auto_cfg), RCV_STATE.RUNNING, RCV_STATE.STOPPED)
-            log.info(util.c("FAC: startup resume ", util.trinary(self.recovery == RCV_STATE.RUNNING, "ready", "failed")))
+            log.info(util.c("FAC: startup resume ", util.trinary(self.recovery == RCV_STATE.RUNNING, "started", "failed")))
         else self.recovery = RCV_STATE.STOPPED end
     end
 
     -- used on certain coordinator commands to end reboot recovery (remain in current operational state)
     function public.cancel_recovery()
-        self.recovery = RCV_STATE.STOPPED
+        if self.recovery == RCV_STATE.RUNNING then
-        self.recovery_boot_state = nil
+            self.recovery = RCV_STATE.STOPPED
-        log.info("FAC: process startup resume cancelled by user operation")
+            self.recovery_boot_state = nil
+            log.info("FAC: process startup resume cancelled by user operation")
+        end
     end
 
     --#endregion

supervisor/facility_update.lua

@@ -817,9 +817,9 @@ function update.unit_mgmt()
             need_emcool = true
         end
 
-        -- check for control state changes to save
+        -- check for enabled state changes to save
-        if self.last_unit_states[i] ~= u.get_control_state() then
+        if self.last_unit_states[i] ~= u.is_reactor_enabled() then
-            self.last_unit_states[i] = u.get_control_state()
+            self.last_unit_states[i] = u.is_reactor_enabled()
             write_state = true
         end
     end
@@ -828,8 +828,7 @@ function update.unit_mgmt()
 
     if write_state then
         settings.set("LastUnitStates", self.last_unit_states)
-        local saved = settings.save("/supervisor.settings")
+        if not settings.save("/supervisor.settings") then
-        if not saved then
             log.warning("facility_update.unit_mgmt(): failed to save supervisor settings file")
         end
     end

supervisor/startup.lua

@@ -22,7 +22,7 @@ local supervisor = require("supervisor.supervisor")
 
 local svsessions = require("supervisor.session.svsessions")
 
-local SUPERVISOR_VERSION = "v1.6.2"
+local SUPERVISOR_VERSION = "v1.6.3"
 
 local println = util.println
 local println_ts = util.println_ts
@@ -240,6 +240,8 @@ local function main()
         end
     end
 
+    sv_facility.clear_boot_state()
+
     renderer.close_ui()
 
     util.println_ts("exited")

supervisor/unit.lua

@@ -840,6 +840,12 @@ function unit.new(reactor_id, num_boilers, num_turbines, ext_idle)
         return false
     end
 
+    -- check the active state of the reactor (if connected)
+    ---@nodiscard
+    function public.is_reactor_enabled()
+        if self.plc_i ~= nil then return self.plc_i.get_status().status else return false end
+    end
+
     -- check if the reactor is connected, is stopped, the RPS is not tripped, and no alarms are active
     ---@nodiscard
     function public.is_safe_idle()
@@ -917,12 +923,6 @@ function unit.new(reactor_id, num_boilers, num_turbines, ext_idle)
         return status
     end
 
-    -- check the commanded control state of the reactor (if connected)
-    ---@nodiscard
-    function public.get_control_state()
-        if self.plc_i ~= nil then return self.plc_i.get_db().control_state else return false end
-    end
-
     -- get the current burn rate (actual rate)
     ---@nodiscard
     function public.get_burn_rate()