Merge pull request #603 from MikaylaFischler/589-reboot-recovery

589 reboot recovery

coordinator/coordinator.lua

@@ -380,6 +380,18 @@ function coordinator.comms(version, nic, sv_watchdog)
         _send_sv(PROTOCOL.SCADA_MGMT, MGMT_TYPE.CLOSE, {})
     end
 
+    -- send the resume ready state to the supervisor
+    ---@param mode PROCESS process control mode
+    ---@param burn_target number burn rate target
+    ---@param charge_target number charge level target
+    ---@param gen_target number generation rate target
+    ---@param limits number[] unit burn rate limits
+    function public.send_ready(mode, burn_target, charge_target, gen_target, limits)
+        _send_sv(PROTOCOL.SCADA_CRDN, CRDN_TYPE.PROCESS_READY, {
+            mode, burn_target, charge_target, gen_target, limits
+        })
+    end
+
     -- send a facility command
     ---@param cmd FAC_COMMAND command
     ---@param option any? optional option options for the optional options (like waste mode)

coordinator/process.lua

@@ -139,6 +139,11 @@ function process.init(iocontrol, coord_comms)
 
         log.info("PROCESS: loaded priority groups settings")
     end
+
+    -- report to the supervisor all initial configuration data has been sent
+    -- startup resume can occur if needed
+    local p = ctl_proc
+    pctl.comms.send_ready(p.mode, p.burn_target, p.charge_target, p.gen_target, p.limits)
 end
 
 -- create a handle to process control for usage of commands that get acknowledgements

coordinator/startup.lua

@@ -19,7 +19,7 @@ local renderer    = require("coordinator.renderer")
 local sounder     = require("coordinator.sounder")
 local threads     = require("coordinator.threads")
 
-local COORDINATOR_VERSION = "v1.6.4"
+local COORDINATOR_VERSION = "v1.6.5"
 
 local CHUNK_LOAD_DELAY_S = 30.0
 

reactor-plc/plc.lua

@@ -23,8 +23,7 @@ local AUTO_ACK = comms.PLC_AUTO_ACK
 
 local RPS_LIMITS = const.RPS_LIMITS
 
--- I sure hope the devs don't change this error message, not that it would have safety implications
+-- specific errors thrown when scram/start is used that still count as success
--- I wish they didn't change it to be like this
 local PCALL_SCRAM_MSG = "Scram requires the reactor to be active."
 local PCALL_START_MSG = "Reactor is already active."
 

reactor-plc/startup.lua

@@ -18,7 +18,7 @@ local plc       = require("reactor-plc.plc")
 local renderer  = require("reactor-plc.renderer")
 local threads   = require("reactor-plc.threads")
 
-local R_PLC_VERSION = "v1.8.14"
+local R_PLC_VERSION = "v1.8.15"
 
 local println = util.println
 local println_ts = util.println_ts
@@ -169,12 +169,12 @@ local function main()
     -- PLC init<br>
     --- EVENT_CONSUMER: this function consumes events
     local function init()
-        -- just booting up, no fission allowed (neutrons stay put thanks)
+        -- scram on boot if networked, otherwise leave the reactor be
-        if (not plc_state.no_reactor) and plc_state.reactor_formed and smem_dev.reactor.getStatus() then
+        if __shared_memory.networked and (not plc_state.no_reactor) and plc_state.reactor_formed and smem_dev.reactor.getStatus() then
             smem_dev.reactor.scram()
         end
 
-        -- front panel time!
+        -- setup front panel
         if not renderer.ui_ready() then
             local message
             plc_state.fp_ok, message = renderer.try_start_ui(config.FrontPanelTheme, config.ColorMode)

scada-common/comms.lua

@@ -17,7 +17,7 @@ local max_distance = nil
 local comms = {}
 
 -- protocol/data versions (protocol/data independent changes tracked by util.lua version)
-comms.version = "3.0.4"
+comms.version = "3.0.5"
 comms.api_version = "0.0.9"
 
 ---@enum PROTOCOL
@@ -60,18 +60,19 @@ local MGMT_TYPE = {
 ---@enum CRDN_TYPE
 local CRDN_TYPE = {
     INITIAL_BUILDS = 0,  -- initial, complete builds packet to the coordinator
-    FAC_BUILDS = 1,      -- facility RTU builds
+    PROCESS_READY = 1,   -- process init is complete + last set of info for supervisor startup recovery
-    FAC_STATUS = 2,      -- state of facility and facility devices
+    FAC_BUILDS = 2,      -- facility RTU builds
-    FAC_CMD = 3,         -- faility command
+    FAC_STATUS = 3,      -- state of facility and facility devices
-    UNIT_BUILDS = 4,     -- build of each reactor unit (reactor + RTUs)
+    FAC_CMD = 4,         -- faility command
-    UNIT_STATUSES = 5,   -- state of each of the reactor units
+    UNIT_BUILDS = 5,     -- build of each reactor unit (reactor + RTUs)
-    UNIT_CMD = 6,        -- command a reactor unit
+    UNIT_STATUSES = 6,   -- state of each of the reactor units
-    API_GET_FAC = 7,     -- API: get the facility general data
+    UNIT_CMD = 7,        -- command a reactor unit
-    API_GET_FAC_DTL = 8, -- API: get (detailed) data for the facility app
+    API_GET_FAC = 8,     -- API: get the facility general data
-    API_GET_UNIT = 9,    -- API: get reactor unit data
+    API_GET_FAC_DTL = 9, -- API: get (detailed) data for the facility app
-    API_GET_CTRL = 10,   -- API: get data for the control app
+    API_GET_UNIT = 10,   -- API: get reactor unit data
-    API_GET_PROC = 11,   -- API: get data for the process app
+    API_GET_CTRL = 11,   -- API: get data for the control app
-    API_GET_WASTE = 12   -- API: get data for the waste app
+    API_GET_PROC = 12,   -- API: get data for the process app
+    API_GET_WASTE = 13   -- API: get data for the waste app
 }
 
 ---@enum ESTABLISH_ACK

supervisor/facility.lua

@@ -31,6 +31,17 @@ local START_STATUS = {
     BLADE_MISMATCH = 2
 }
 
+---@enum RECOVERY_STATE
+local RCV_STATE = {
+    INACTIVE = 0,
+    PRIMED = 1,
+    RUNNING = 2,
+    STOPPED = 3
+}
+
+local CHARGE_SCALER = 1000000   -- convert MFE to FE
+local GEN_SCALER    = 1000      -- convert kFE to FE
+
 ---@class facility_management
 local facility = {}
 
@@ -41,7 +52,7 @@ function facility.new(config)
     ---@class _facility_self
     local self = {
         units = {},     ---@type reactor_unit[]
-        types = { AUTO_SCRAM = AUTO_SCRAM, START_STATUS = START_STATUS },
+        types = { AUTO_SCRAM = AUTO_SCRAM, START_STATUS = START_STATUS, RCV_STATE = RCV_STATE },
         status_text = { "START UP", "initializing..." },
         all_sys_ok = false,
         allow_testing = false,
@@ -66,12 +77,15 @@ function facility.new(config)
         -- redstone I/O control
         io_ctl = nil,   ---@type rs_controller
         -- process control
+        recovery = RCV_STATE.INACTIVE,  ---@type RECOVERY_STATE
+        recovery_boot_state = nil,      ---@type sv_boot_state|nil
+        last_unit_states = {},          ---@type boolean[]
         units_ready = false,
-        mode = PROCESS.INACTIVE,
+        mode = PROCESS.INACTIVE,        ---@type PROCESS
-        last_mode = PROCESS.INACTIVE,
+        last_mode = PROCESS.INACTIVE,   ---@type PROCESS
-        return_mode = PROCESS.INACTIVE,
+        return_mode = PROCESS.INACTIVE, ---@type PROCESS
-        mode_set = PROCESS.MAX_BURN,
+        mode_set = PROCESS.MAX_BURN,    ---@type PROCESS
-        start_fail = START_STATUS.OK,
+        start_fail = START_STATUS.OK,   ---@type START_STATUS
         max_burn_combined = 0.0,        -- maximum burn rate to clamp at
         burn_target = 0.1,              -- burn rate target for aggregate burn mode
         charge_setpoint = 0,            -- FE charge target setpoint
@@ -101,8 +115,8 @@ function facility.new(config)
         last_error = 0.0,
         last_time = 0.0,
         -- waste processing
-        waste_product = WASTE.PLUTONIUM,
+        waste_product = WASTE.PLUTONIUM,         ---@type WASTE_PRODUCT
-        current_waste_product = WASTE.PLUTONIUM,
+        current_waste_product = WASTE.PLUTONIUM, ---@type WASTE_PRODUCT
         pu_fallback = false,
         sps_low_power = false,
         disabled_sps = false,
@@ -126,14 +140,16 @@ function facility.new(config)
         imtx_faulted_times = { 0, 0, 0 }
     }
 
+    --#region SETUP
+
     -- provide self to facility update functions
     local f_update = fac_update(self)
 
     -- create units
     for i = 1, config.UnitCount do
-        table.insert(self.units,
+        table.insert(self.units, unit.new(i, self.cooling_conf.r_cool[i].BoilerCount, self.cooling_conf.r_cool[i].TurbineCount, config.ExtChargeIdling))
-            unit.new(i, self.cooling_conf.r_cool[i].BoilerCount, self.cooling_conf.r_cool[i].TurbineCount, config.ExtChargeIdling))
         table.insert(self.group_map, AUTO_GROUP.MANUAL)
+        table.insert(self.last_unit_states, false)
     end
 
     -- list for RTU session management
@@ -149,6 +165,70 @@ function facility.new(config)
         table.insert(self.test_tone_states, false)
     end
 
+    -- init next boot state
+    settings.set("LastProcessState", PROCESS.INACTIVE)
+    settings.set("LastUnitStates", self.last_unit_states)
+    if not settings.save("/supervisor.settings") then
+        log.warning("FAC: failed to save initial control state into supervisor settings file")
+    end
+
+    --#endregion
+
+    -- PRIVATE FUNCTIONS --
+
+    -- check an auto process control configuration and save it if its valid (does not start the process)
+    ---@param auto_cfg start_auto_config configuration
+    ---@return boolean ready, number[] unit_limits
+    local function _auto_check_and_save(auto_cfg)
+        local ready = false
+
+        -- load up current limits
+        local limits = {}
+        for i = 1, config.UnitCount do
+            limits[i] = self.units[i].get_control_inf().lim_br100 * 100
+        end
+
+        -- only allow changes if not running
+        if self.mode == PROCESS.INACTIVE then
+            if (type(auto_cfg.mode) == "number") and (auto_cfg.mode > PROCESS.INACTIVE) and (auto_cfg.mode <= PROCESS.GEN_RATE) then
+                self.mode_set = auto_cfg.mode
+            end
+
+            if (type(auto_cfg.burn_target) == "number") and auto_cfg.burn_target >= 0.1 then
+                self.burn_target = auto_cfg.burn_target
+            end
+
+            if (type(auto_cfg.charge_target) == "number") and auto_cfg.charge_target >= 0 then
+                self.charge_setpoint = auto_cfg.charge_target * CHARGE_SCALER
+            end
+
+            if (type(auto_cfg.gen_target) == "number") and auto_cfg.gen_target >= 0 then
+                self.gen_rate_setpoint = auto_cfg.gen_target * GEN_SCALER
+            end
+
+            if (type(auto_cfg.limits) == "table") and (#auto_cfg.limits == config.UnitCount) then
+                for i = 1, config.UnitCount do
+                    local limit = auto_cfg.limits[i]
+
+                    if (type(limit) == "number") and (limit >= 0.1) then
+                        limits[i] = limit
+                        self.units[i].set_burn_limit(limit)
+                    end
+                end
+            end
+
+            ready = self.mode_set > 0
+
+            if ((self.mode_set == PROCESS.CHARGE) and (self.charge_setpoint <= 0)) or
+               ((self.mode_set == PROCESS.GEN_RATE) and (self.gen_rate_setpoint <= 0)) or
+               ((self.mode_set == PROCESS.BURN_RATE) and (self.burn_target < 0.1)) then
+                ready = false
+            end
+        end
+
+        return ready, limits
+    end
+
     -- PUBLIC FUNCTIONS --
 
     ---@class facility
@@ -239,6 +319,9 @@ function facility.new(config)
 
     -- update (iterate) the facility management
     function public.update()
+        -- run reboot recovery routine if needed
+        f_update.boot_recovery()
+
         -- run process control and evaluate automatic SCRAM
         f_update.pre_auto()
         f_update.auto_control(config.ExtChargeIdling)
@@ -267,6 +350,50 @@ function facility.new(config)
 
     --#endregion
 
+    --#region Startup Recovery
+
+    -- on exit, use this to clear the boot state so we don't resume when exiting cleanly
+    function public.clear_boot_state()
+        settings.unset("LastProcessState")
+        settings.unset("LastUnitStates")
+
+        if not settings.save("/supervisor.settings") then
+            log.warning("facility.clear_boot_state(): failed to save supervisor settings file")
+        else
+            log.debug("FAC: cleared boot state on exit")
+        end
+    end
+
+    -- initialize facility resume boot recovery
+    ---@param state sv_boot_state|nil
+    function public.boot_recovery_init(state)
+        if self.recovery == RCV_STATE.INACTIVE and state then
+            self.recovery_boot_state = state
+            self.recovery = RCV_STATE.PRIMED
+            log.info("FAC: startup resume ready")
+        end
+    end
+
+    -- attempt facility resume boot recovery
+    ---@param auto_cfg start_auto_config configuration
+    function public.boot_recovery_start(auto_cfg)
+        if self.recovery == RCV_STATE.PRIMED then
+            self.recovery = util.trinary(_auto_check_and_save(auto_cfg), RCV_STATE.RUNNING, RCV_STATE.STOPPED)
+            log.info(util.c("FAC: startup resume ", util.trinary(self.recovery == RCV_STATE.RUNNING, "started", "failed")))
+        else self.recovery = RCV_STATE.STOPPED end
+    end
+
+    -- used on certain coordinator commands to end reboot recovery (remain in current operational state)
+    function public.cancel_recovery()
+        if self.recovery == RCV_STATE.RUNNING then
+            self.recovery = RCV_STATE.STOPPED
+            self.recovery_boot_state = nil
+            log.info("FAC: process startup resume cancelled by user operation")
+        end
+    end
+
+    --#endregion
+
     --#region Commands
 
     -- SCRAM all reactor units
@@ -290,59 +417,13 @@ function facility.new(config)
     function public.auto_stop() self.mode = PROCESS.INACTIVE end
 
     -- set automatic control configuration and start the process
-    ---@param auto_cfg sys_auto_config configuration
+    ---@param auto_cfg start_auto_config configuration
     ---@return table response ready state (successfully started) and current configuration (after updating)
     function public.auto_start(auto_cfg)
-        local charge_scaler = 1000000   -- convert MFE to FE
+        local ready, limits = _auto_check_and_save(auto_cfg)
-        local gen_scaler    = 1000      -- convert kFE to FE
-        local ready         = false
 
-        -- load up current limits
+        if ready and self.units_ready then
-        local limits = {}
+            self.mode = self.mode_set
-        for i = 1, config.UnitCount do
-            limits[i] = self.units[i].get_control_inf().lim_br100 * 100
-        end
-
-        -- only allow changes if not running
-        if self.mode == PROCESS.INACTIVE then
-            if (type(auto_cfg.mode) == "number") and (auto_cfg.mode > PROCESS.INACTIVE) and (auto_cfg.mode <= PROCESS.GEN_RATE) then
-                self.mode_set = auto_cfg.mode
-            end
-
-            if (type(auto_cfg.burn_target) == "number") and auto_cfg.burn_target >= 0.1 then
-                self.burn_target = auto_cfg.burn_target
-            end
-
-            if (type(auto_cfg.charge_target) == "number") and auto_cfg.charge_target >= 0 then
-                self.charge_setpoint = auto_cfg.charge_target * charge_scaler
-            end
-
-            if (type(auto_cfg.gen_target) == "number") and auto_cfg.gen_target >= 0 then
-                self.gen_rate_setpoint = auto_cfg.gen_target * gen_scaler
-            end
-
-            if (type(auto_cfg.limits) == "table") and (#auto_cfg.limits == config.UnitCount) then
-                for i = 1, config.UnitCount do
-                    local limit = auto_cfg.limits[i]
-
-                    if (type(limit) == "number") and (limit >= 0.1) then
-                        limits[i] = limit
-                        self.units[i].set_burn_limit(limit)
-                    end
-                end
-            end
-
-            ready = self.mode_set > 0
-
-            if ((self.mode_set == PROCESS.CHARGE) and (self.charge_setpoint <= 0)) or
-               ((self.mode_set == PROCESS.GEN_RATE) and (self.gen_rate_setpoint <= 0)) or
-               ((self.mode_set == PROCESS.BURN_RATE) and (self.burn_target < 0.1)) then
-                ready = false
-            end
-
-            ready = ready and self.units_ready
-
-            if ready then self.mode = self.mode_set end
         end
 
         log.debug(util.c("FAC: process start ", util.trinary(ready, "accepted", "rejected")))
@@ -351,8 +432,8 @@ function facility.new(config)
             ready,
             self.mode_set,
             self.burn_target,
-            self.charge_setpoint / charge_scaler,
+            self.charge_setpoint / CHARGE_SCALER,
-            self.gen_rate_setpoint / gen_scaler,
+            self.gen_rate_setpoint / GEN_SCALER,
             limits
         }
     end

supervisor/facility_update.lua

@@ -5,6 +5,9 @@ local rsio   = require("scada-common.rsio")
 local types      = require("scada-common.types")
 local util       = require("scada-common.util")
 
+local plc        = require("supervisor.session.plc")
+local svsessions = require("supervisor.session.svsessions")
+
 local qtypes     = require("supervisor.session.rtu.qtypes")
 
 local TONE           = audio.TONE
@@ -12,6 +15,7 @@ local TONE           = audio.TONE
 local ALARM          = types.ALARM
 local PRIO           = types.ALARM_PRIORITY
 local ALARM_STATE    = types.ALARM_STATE
+local AUTO_GROUP     = types.AUTO_GROUP
 local CONTAINER_MODE = types.CONTAINER_MODE
 local PROCESS        = types.PROCESS
 local PROCESS_NAMES  = types.PROCESS_NAMES
@@ -131,6 +135,54 @@ end
 
 --#region PUBLIC FUNCTIONS
 
+-- run reboot recovery routine if needed
+function update.boot_recovery()
+    local RCV_STATE = self.types.RCV_STATE
+
+    -- attempt reboot recovery if in progress
+    if self.recovery == RCV_STATE.RUNNING then
+        local was_inactive = self.recovery_boot_state.mode == PROCESS.INACTIVE or self.recovery_boot_state.mode == PROCESS.SYSTEM_ALARM_IDLE
+
+        -- try to start auto control
+        if self.recovery_boot_state.mode ~= nil and self.units_ready then
+            if not was_inactive then
+                self.mode = self.mode_set
+                log.info("FAC: process startup resume initiated")
+            end
+
+            self.recovery_boot_state.mode = nil
+        end
+
+        local recovered = self.recovery_boot_state.mode == nil or was_inactive
+
+        -- restore manual control reactors
+        for i = 1, #self.units do
+            local u = self.units[i]
+
+            if self.recovery_boot_state.unit_states[i] and self.group_map[i] == AUTO_GROUP.MANUAL then
+                recovered = false
+
+                if u.get_control_inf().ready then
+                    local plc_s = svsessions.get_reactor_session(i)
+                    if plc_s ~= nil then
+                        plc_s.in_queue.push_command(plc.PLC_S_CMDS.ENABLE)
+                        log.info("FAC: startup resume enabling manually controlled reactor unit #" .. i)
+
+                        -- only execute once
+                        self.recovery_boot_state.unit_states[i] = nil
+                    end
+                end
+            end
+        end
+
+        if recovered then
+            self.recovery = RCV_STATE.STOPPED
+            self.recovery_boot_state = nil
+            log.info("FAC: startup resume sequence completed")
+        end
+    end
+end
+
 -- automatic control pre-update logic
 function update.pre_auto()
     -- unlink RTU sessions if they are closed
@@ -243,6 +295,11 @@ function update.auto_control(ExtChargeIdling)
 
         log.debug(util.c("FAC: state changed from ", PROCESS_NAMES[self.last_mode + 1], " to ", PROCESS_NAMES[self.mode + 1]))
 
+        settings.set("LastProcessState", self.mode)
+        if not settings.save("/supervisor.settings") then
+            log.warning("facility_update.auto_control(): failed to save supervisor settings file")
+        end
+
         if (self.last_mode == PROCESS.INACTIVE) or (self.last_mode == PROCESS.GEN_RATE_FAULT_IDLE) then
             self.start_fail = START_STATUS.OK
 
@@ -642,15 +699,16 @@ function update.auto_safety()
             self.ascram_reason = AUTO_SCRAM.NONE
 
             -- reset PLC RPS trips if we should
-            for i = 1, #self.units do
+            for i = 1, #self.prio_defs do
-                local u = self.units[i]
+                for _, u in pairs(self.prio_defs[i]) do
                     u.auto_cond_rps_reset()
                 end
             end
         end
+    end
 end
 
--- update last mode and set next mode
+-- update last mode, set next mode, and update saved state as needed
 function update.post_auto()
     self.last_mode = self.mode
     self.mode = next_mode
@@ -792,6 +850,7 @@ end
 function update.unit_mgmt()
     local insufficent_po_rate = false
     local need_emcool = false
+    local write_state = false
 
     for i = 1, #self.units do
         local u = self.units[i]
@@ -807,6 +866,21 @@ function update.unit_mgmt()
         if (self.cooling_conf.fac_tank_mode > 0) and u.is_emer_cool_tripped() and (self.cooling_conf.fac_tank_defs[i] == 2) then
             need_emcool = true
         end
+
+        -- check for enabled state changes to save
+        if self.last_unit_states[i] ~= u.is_reactor_enabled() then
+            self.last_unit_states[i] = u.is_reactor_enabled()
+            write_state = true
+        end
+    end
+
+    -- record unit control states
+
+    if write_state then
+        settings.set("LastUnitStates", self.last_unit_states)
+        if not settings.save("/supervisor.settings") then
+            log.warning("facility_update.unit_mgmt(): failed to save supervisor settings file")
+        end
     end
 
     -- update waste product

supervisor/session/coordinator.lua

@@ -234,6 +234,23 @@ function coordinator.new_session(id, s_addr, i_seq_num, in_queue, out_queue, tim
             if pkt.type == CRDN_TYPE.INITIAL_BUILDS then
                 -- acknowledgement to coordinator receiving builds
                 self.acks.builds = true
+            elseif pkt.type == CRDN_TYPE.PROCESS_READY then
+                if pkt.length == 5 then
+                    -- coordinator has sent all initial process data, power-on recovery is now possible
+
+                    ---@type start_auto_config
+                    local config = {
+                        mode = pkt.data[1],
+                        burn_target = pkt.data[2],
+                        charge_target = pkt.data[3],
+                        gen_target = pkt.data[4],
+                        limits = pkt.data[5]
+                    }
+
+                    facility.boot_recovery_start(config)
+                else
+                    log.debug(log_tag .. "CRDN process ready packet length mismatch")
+                end
             elseif pkt.type == CRDN_TYPE.FAC_BUILDS then
                 -- acknowledgement to coordinator receiving builds
                 self.acks.fac_builds = true
@@ -243,8 +260,11 @@ function coordinator.new_session(id, s_addr, i_seq_num, in_queue, out_queue, tim
 
                     if cmd == FAC_COMMAND.SCRAM_ALL then
                         facility.scram_all()
+                        facility.cancel_recovery()
                         _send(CRDN_TYPE.FAC_CMD, { cmd, true })
                     elseif cmd == FAC_COMMAND.STOP then
+                        facility.cancel_recovery()
+
                         local was_active = facility.auto_is_active()
 
                         if was_active then
@@ -253,15 +273,16 @@ function coordinator.new_session(id, s_addr, i_seq_num, in_queue, out_queue, tim
 
                         _send(CRDN_TYPE.FAC_CMD, { cmd, was_active })
                     elseif cmd == FAC_COMMAND.START then
+                        facility.cancel_recovery()
+
                         if pkt.length == 6 then
-                            ---@type sys_auto_config
+                            ---@class start_auto_config
----@diagnostic disable-next-line: missing-fields
                             local config = {
-                                mode = pkt.data[2],
+                                mode = pkt.data[2],          ---@type PROCESS
-                                burn_target = pkt.data[3],
+                                burn_target = pkt.data[3],   ---@type number
-                                charge_target = pkt.data[4],
+                                charge_target = pkt.data[4], ---@type number
-                                gen_target = pkt.data[5],
+                                gen_target = pkt.data[5],    ---@type number
-                                limits = pkt.data[6]
+                                limits = pkt.data[6]         ---@type number[]
                             }
 
                             _send(CRDN_TYPE.FAC_CMD, { cmd, table.unpack(facility.auto_start(config)) })
@@ -313,8 +334,11 @@ function coordinator.new_session(id, s_addr, i_seq_num, in_queue, out_queue, tim
                         local manual = facility.get_group(uid) == AUTO_GROUP.MANUAL
 
                         if cmd == UNIT_COMMAND.SCRAM then
+                            facility.cancel_recovery()
                             out_queue.push_data(SV_Q_DATA.SCRAM, data)
                         elseif cmd == UNIT_COMMAND.START then
+                            facility.cancel_recovery()
+
                             if manual then
                                 out_queue.push_data(SV_Q_DATA.START, data)
                             else
@@ -324,6 +348,8 @@ function coordinator.new_session(id, s_addr, i_seq_num, in_queue, out_queue, tim
                         elseif cmd == UNIT_COMMAND.RESET_RPS then
                             out_queue.push_data(SV_Q_DATA.RESET_RPS, data)
                         elseif cmd == UNIT_COMMAND.SET_BURN then
+                            facility.cancel_recovery()
+
                             if pkt.length == 3 then
                                 if manual then
                                     out_queue.push_data(SV_Q_DATA.SET_BURN, data)
@@ -354,6 +380,8 @@ function coordinator.new_session(id, s_addr, i_seq_num, in_queue, out_queue, tim
                                 log.debug(log_tag .. "CRDN unit command reset alarm missing alarm id")
                             end
                         elseif cmd == UNIT_COMMAND.SET_GROUP then
+                            facility.cancel_recovery()
+
                             if (pkt.length == 3) and (type(pkt.data[3]) == "number") and
                                (pkt.data[3] >= AUTO_GROUP.MANUAL) and (pkt.data[3] <= AUTO_GROUP.BACKUP) then
                                 facility.set_group(unit.get_id(), pkt.data[3])

supervisor/session/plc.lua

@@ -53,15 +53,15 @@ local PERIODICS = {
 ---@param in_queue mqueue in message queue
 ---@param out_queue mqueue out message queue
 ---@param timeout number communications timeout
+---@param initial_reset boolean[] initial PLC reset on timeout flags, indexed by reactor_id
 ---@param fp_ok boolean if the front panel UI is running
-function plc.new_session(id, s_addr, i_seq_num, reactor_id, in_queue, out_queue, timeout, fp_ok)
+function plc.new_session(id, s_addr, i_seq_num, reactor_id, in_queue, out_queue, timeout, initial_reset, fp_ok)
     -- print a log message to the terminal as long as the UI isn't running
     local function println(message) if not fp_ok then util.println_ts(message) end end
 
     local log_tag = "plc_session(" .. id .. "): "
 
     local self = {
-        commanded_state = false,
         commanded_burn_rate = 0.0,
         auto_cmd_token = 0,
         ramping_rate = false,
@@ -72,6 +72,7 @@ function plc.new_session(id, s_addr, i_seq_num, reactor_id, in_queue, out_queue,
         connected = true,
         received_struct = false,
         received_status_cache = false,
+        received_rps_status = false,
         conn_watchdog = util.new_watchdog(timeout),
         last_rtt = 0,
         -- periodic messages
@@ -381,6 +382,16 @@ function plc.new_session(id, s_addr, i_seq_num, reactor_id, in_queue, out_queue,
                     local status = pcall(_copy_rps_status, pkt.data)
                     if status then
                         -- copied in RPS status data OK
+                        self.received_rps_status = true
+
+                        -- try initial reset if needed
+                        if initial_reset[reactor_id] then
+                            initial_reset[reactor_id] = false
+                            if self.sDB.rps_trip_cause == "timeout" then
+                                _send(RPLC_TYPE.RPS_AUTO_RESET, {})
+                                log.debug(log_tag .. "initial RPS reset on timeout status sent")
+                            end
+                        end
                     else
                         -- error copying RPS status data
                         log.error(log_tag .. "failed to parse RPS status packet data")
@@ -394,6 +405,16 @@ function plc.new_session(id, s_addr, i_seq_num, reactor_id, in_queue, out_queue,
                     local status = pcall(_copy_rps_status, { true, table.unpack(pkt.data) })
                     if status then
                         -- copied in RPS status data OK
+                        self.received_rps_status = true
+
+                        -- try initial reset if needed
+                        if initial_reset[reactor_id] then
+                            initial_reset[reactor_id] = false
+                            if self.sDB.rps_trip_cause == "timeout" then
+                                _send(RPLC_TYPE.RPS_AUTO_RESET, {})
+                                log.debug(log_tag .. "initial RPS reset on timeout alarm sent")
+                            end
+                        end
                     else
                         -- error copying RPS status data
                         log.error(log_tag .. "failed to parse RPS alarm status data")
@@ -487,6 +508,10 @@ function plc.new_session(id, s_addr, i_seq_num, reactor_id, in_queue, out_queue,
     ---@nodiscard
     function public.get_db() return self.sDB end
 
+    -- check if the reactor structure, status, and RPS status have been received
+    ---@nodiscard
+    function public.check_received_all_data() return self.received_struct and self.received_status_cache and self.received_rps_status end
+
     -- check if ramping is completed by first verifying auto command token ack
     ---@nodiscard
     function public.is_ramp_complete()

supervisor/session/svsessions.lua

@@ -45,6 +45,7 @@ local self = {
     fp_ok = false,
     config = nil,           ---@type svr_config
     facility = nil,         ---@type facility|nil
+    plc_ini_reset = {},
     -- lists of connected sessions
 ---@diagnostic disable: missing-fields
     sessions = {
@@ -391,6 +392,7 @@ function svsessions.init(nic, fp_ok, config, facility)
             conns.tanks[1] = true
         end
 
+        self.plc_ini_reset[i] = true
         self.dev_dbg.connected.units[i] = conns
     end
 end
@@ -486,7 +488,7 @@ function svsessions.establish_plc_session(source_addr, i_seq_num, for_reactor, v
 
         local id = self.next_ids.plc
 
-        plc_s.instance = plc.new_session(id, source_addr, i_seq_num, for_reactor, plc_s.in_queue, plc_s.out_queue, self.config.PLC_Timeout, self.fp_ok)
+        plc_s.instance = plc.new_session(id, source_addr, i_seq_num, for_reactor, plc_s.in_queue, plc_s.out_queue, self.config.PLC_Timeout, self.plc_ini_reset, self.fp_ok)
         table.insert(self.sessions.plc, plc_s)
 
         local units = self.facility.get_units()

supervisor/startup.lua

@@ -22,7 +22,7 @@ local supervisor = require("supervisor.supervisor")
 
 local svsessions = require("supervisor.session.svsessions")
 
-local SUPERVISOR_VERSION = "v1.6.2"
+local SUPERVISOR_VERSION = "v1.6.3"
 
 local println = util.println
 local println_ts = util.println_ts
@@ -147,6 +147,9 @@ local function main()
     -- halve the rate heartbeat LED flash
     local heartbeat_toggle = true
 
+    -- init startup recovery
+    sv_facility.boot_recovery_init(supervisor.boot_state)
+
     -- event loop
     while true do
         local event, param1, param2, param3, param4, param5 = util.pull_event()
@@ -237,6 +240,8 @@ local function main()
         end
     end
 
+    sv_facility.clear_boot_state()
+
     renderer.close_ui()
 
     util.println_ts("exited")

supervisor/supervisor.lua

@@ -19,10 +19,24 @@ local config = {}
 
 supervisor.config = config
 
--- load the supervisor configuration
+-- control state from last unexpected shutdown
+supervisor.boot_state = nil ---@type sv_boot_state|nil
+
+-- load the supervisor configuration and startup state
 function supervisor.load_config()
     if not settings.load("/supervisor.settings") then return false end
 
+    ---@class sv_boot_state
+    local boot_state = {
+        mode = settings.get("LastProcessState"),     ---@type PROCESS
+        unit_states = settings.get("LastUnitStates") ---@type boolean[]
+    }
+
+    -- only record boot state if likely valid
+    if type(boot_state.mode) == "number" and type(boot_state.unit_states) == "table" then
+        supervisor.boot_state = boot_state
+    end
+
     config.UnitCount = settings.get("UnitCount")
     config.CoolingConfig = settings.get("CoolingConfig")
     config.FacilityTankMode = settings.get("FacilityTankMode")

supervisor/unit.lua

@@ -840,6 +840,12 @@ function unit.new(reactor_id, num_boilers, num_turbines, ext_idle)
         return false
     end
 
+    -- check the active state of the reactor (if connected)
+    ---@nodiscard
+    function public.is_reactor_enabled()
+        if self.plc_i ~= nil then return self.plc_i.get_status().status else return false end
+    end
+
     -- check if the reactor is connected, is stopped, the RPS is not tripped, and no alarms are active
     ---@nodiscard
     function public.is_safe_idle()

supervisor/unitlogic.lua

@@ -67,11 +67,10 @@ function logic.update_annunciator(self)
         local plc_db = self.plc_i.get_db()
 
         -- update ready state
-        --  - can't be tripped
+        --  - must be connected to a formed reactor
-        --  - must have received status at least once
+        --  - can't have a tripped RPS
-        --  - must have received struct at least once
+        --  - must have received status, struct, and RPS status at least once
-        plc_ready = plc_db.formed and (not plc_db.no_reactor) and (not plc_db.rps_tripped) and
+        plc_ready = plc_db.formed and (not plc_db.no_reactor) and (not plc_db.rps_tripped) and self.plc_i.check_received_all_data()
-                    (next(self.plc_i.get_status()) ~= nil) and (next(self.plc_i.get_struct()) ~= nil)
 
         -- update auto control limit
         if (plc_db.mek_struct.max_burn > 0) and ((self.db.control.lim_br100 / 100) > plc_db.mek_struct.max_burn) then