From f32855084e972c82408ec60a863fa1f55e8bc7f2 Mon Sep 17 00:00:00 2001 From: Mikayla Fischler Date: Sat, 8 Feb 2025 22:20:00 -0500 Subject: [PATCH] #589 WIP reboot recovery --- supervisor/facility.lua | 52 ++++++++++++++++++++++++++-------- supervisor/facility_update.lua | 9 +++--- supervisor/startup.lua | 4 ++- supervisor/unit.lua | 12 ++++---- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/supervisor/facility.lua b/supervisor/facility.lua index 6525d84..354ceb5 100644 --- a/supervisor/facility.lua +++ b/supervisor/facility.lua @@ -80,7 +80,7 @@ function facility.new(config) -- process control recovery = RCV_STATE.INACTIVE, ---@type RECOVERY_STATE recovery_boot_state = nil, ---@type sv_control_state|nil - last_unit_states = nil, ---@type boolean[] + last_unit_states = {}, ---@type boolean[] units_ready = false, mode = PROCESS.INACTIVE, ---@type PROCESS last_mode = PROCESS.INACTIVE, ---@type PROCESS @@ -166,6 +166,13 @@ function facility.new(config) table.insert(self.test_tone_states, false) end + -- init next boot state + settings.set("LastProcessState", PROCESS.INACTIVE) + settings.set("LastUnitStates", self.last_unit_states) + if not settings.save("/supervisor.settings") then + log.warning("FAC: failed to save initial control state into supervisor settings file") + end + --#endregion -- PRIVATE FUNCTIONS -- @@ -316,19 +323,24 @@ function facility.new(config) if self.recovery == RCV_STATE.RUNNING then -- try to start auto control if self.recovery_boot_state.mode ~= nil and self.units_ready then + if self.recovery_boot_state.mode ~= PROCESS.INACTIVE and self.recovery_boot_state.mode ~= PROCESS.SYSTEM_ALARM_IDLE then + self.mode = self.mode_set + log.info("FAC: process startup resume initiated") + end + self.recovery_boot_state.mode = nil - self.mode = self.mode_set - log.info("FAC: process startup resume initiated") end - local recovered = self.recovery_boot_state.mode == nil + local recovered = self.recovery_boot_state.mode == nil or self.recovery_boot_state.mode == PROCESS.INACTIVE -- restore manual control reactors for i = 1, #self.units do + local u = self.units[i] + if self.recovery_boot_state.unit_states[i] and self.group_map[i] == AUTO_GROUP.MANUAL then recovered = false - if self.units[i].get_control_inf().ready then + if u.get_control_inf().ready then local plc_s = svsessions.get_reactor_session(i) if plc_s ~= nil then plc_s.in_queue.push_command(plc.PLC_S_CMDS.ENABLE) @@ -344,7 +356,7 @@ function facility.new(config) if recovered then self.recovery = RCV_STATE.STOPPED self.recovery_boot_state = nil - log.info("FAC: startup resume complete") + log.info("FAC: startup resume sequence completed") end end @@ -378,29 +390,45 @@ function facility.new(config) --#region Startup Recovery + -- on exit, use this to clear the boot state so we don't resume when exiting cleanly + function public.clear_boot_state() + settings.unset("LastProcessState") + settings.unset("LastUnitStates") + + local saved = settings.save("/supervisor.settings") + if not saved then + log.warning("facility.clear_boot_state(): failed to save supervisor settings file") + else + log.debug("FAC: cleared boot state on exit") + end + end + + -- initialize startup recovery ---@param state sv_control_state function public.startup_recovery_init(state) if self.recovery == RCV_STATE.INACTIVE then self.recovery_boot_state = state self.recovery = RCV_STATE.PRIMED + log.info("FAC: startup resume ready") end end -- attempt startup recovery ---@param auto_cfg start_auto_config configuration function public.startup_recovery_start(auto_cfg) - if self.recovery == RCV_STATE.PRIMED and self.recovery_boot_state and - self.recovery_boot_state.mode ~= PROCESS.INACTIVE and self.recovery_boot_state.mode ~= PROCESS.SYSTEM_ALARM_IDLE then + if self.recovery == RCV_STATE.PRIMED then self.recovery = util.trinary(_auto_check_and_save(auto_cfg), RCV_STATE.RUNNING, RCV_STATE.STOPPED) - log.info(util.c("FAC: startup resume ", util.trinary(self.recovery == RCV_STATE.RUNNING, "ready", "failed"))) + log.info(util.c("FAC: startup resume ", util.trinary(self.recovery == RCV_STATE.RUNNING, "started", "failed"))) else self.recovery = RCV_STATE.STOPPED end end -- used on certain coordinator commands to end reboot recovery (remain in current operational state) function public.cancel_recovery() - self.recovery = RCV_STATE.STOPPED - self.recovery_boot_state = nil - log.info("FAC: process startup resume cancelled by user operation") + if self.recovery == RCV_STATE.RUNNING then + self.recovery = RCV_STATE.STOPPED + self.recovery_boot_state = nil + log.info("FAC: process startup resume cancelled by user operation") + end end --#endregion diff --git a/supervisor/facility_update.lua b/supervisor/facility_update.lua index 9526596..9b5f6fd 100644 --- a/supervisor/facility_update.lua +++ b/supervisor/facility_update.lua @@ -817,9 +817,9 @@ function update.unit_mgmt() need_emcool = true end - -- check for control state changes to save - if self.last_unit_states[i] ~= u.get_control_state() then - self.last_unit_states[i] = u.get_control_state() + -- check for enabled state changes to save + if self.last_unit_states[i] ~= u.is_reactor_enabled() then + self.last_unit_states[i] = u.is_reactor_enabled() write_state = true end end @@ -828,8 +828,7 @@ function update.unit_mgmt() if write_state then settings.set("LastUnitStates", self.last_unit_states) - local saved = settings.save("/supervisor.settings") - if not saved then + if not settings.save("/supervisor.settings") then log.warning("facility_update.unit_mgmt(): failed to save supervisor settings file") end end diff --git a/supervisor/startup.lua b/supervisor/startup.lua index c2686e0..0b5c1e6 100644 --- a/supervisor/startup.lua +++ b/supervisor/startup.lua @@ -22,7 +22,7 @@ local supervisor = require("supervisor.supervisor") local svsessions = require("supervisor.session.svsessions") -local SUPERVISOR_VERSION = "v1.6.2" +local SUPERVISOR_VERSION = "v1.6.3" local println = util.println local println_ts = util.println_ts @@ -240,6 +240,8 @@ local function main() end end + sv_facility.clear_boot_state() + renderer.close_ui() util.println_ts("exited") diff --git a/supervisor/unit.lua b/supervisor/unit.lua index 8bbac07..01d0d3e 100644 --- a/supervisor/unit.lua +++ b/supervisor/unit.lua @@ -840,6 +840,12 @@ function unit.new(reactor_id, num_boilers, num_turbines, ext_idle) return false end + -- check the active state of the reactor (if connected) + ---@nodiscard + function public.is_reactor_enabled() + if self.plc_i ~= nil then return self.plc_i.get_status().status else return false end + end + -- check if the reactor is connected, is stopped, the RPS is not tripped, and no alarms are active ---@nodiscard function public.is_safe_idle() @@ -917,12 +923,6 @@ function unit.new(reactor_id, num_boilers, num_turbines, ext_idle) return status end - -- check the commanded control state of the reactor (if connected) - ---@nodiscard - function public.get_control_state() - if self.plc_i ~= nil then return self.plc_i.get_db().control_state else return false end - end - -- get the current burn rate (actual rate) ---@nodiscard function public.get_burn_rate()