1 #ifndef _ots_GatewaySupervisor_h
2 #define _ots_GatewaySupervisor_h
4 #include <condition_variable>
7 #include "otsdaq/CoreSupervisors/ConfigurationSupervisorBase.h"
8 #include "otsdaq/CoreSupervisors/CorePropertySupervisorBase.h"
9 #include "otsdaq/FiniteStateMachine/RunControlStateMachine.h"
10 #include "otsdaq/FiniteStateMachine/RunInfoVInterface.h"
11 #include "otsdaq/GatewaySupervisor/Iterator.h"
12 #include "otsdaq/SOAPUtilities/SOAPMessenger.h"
13 #include "otsdaq/SupervisorInfo/AllSupervisorInfo.h"
14 #include "otsdaq/WebUsersUtilities/WebUsers.h"
15 #include "otsdaq/WorkLoopManager/WorkLoopManager.h"
17 #include "otsdaq/CodeEditor/CodeEditor.h"
18 #include "otsdaq/TablePlugins/DesktopIconTable.h"
20 #include "otsdaq/NetworkUtilities/TransceiverSocket.h"
22 #pragma GCC diagnostic push
23 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
24 #include <xdaq/Application.h>
25 #pragma GCC diagnostic pop
26 #include "otsdaq/Macros/XDAQApplicationMacros.h"
28 #include <toolbox/task/WorkLoop.h>
29 #include <xdata/String.h>
30 #include <xgi/Method.h>
31 #include "otsdaq/GatewaySupervisor/PixelHistoPicGen.h"
64 class ConfigurationManager;
66 class WorkLoopManager;
82 static const std::string COMMAND_PARAM_LOG_ENTRY_PREAMBLE;
83 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_PREAMBLE;
84 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_OVERRIDE_PREAMBLE;
85 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_CONTEXT_PREAMBLE;
86 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_CONTEXT_OVERRIDE_PREAMBLE;
87 static const std::string COMMAND_PARAM_ITERATION_INDEX_PREAMBLE;
88 static const std::string COMMAND_PARAM_MIN_EVENT_GEN_START_ITERATION_PREAMBLE;
98 void Default (xgi::Input* in, xgi::Output* out);
100 void loginRequest (xgi::Input* in, xgi::Output* out);
101 void request (xgi::Input* in, xgi::Output* out);
102 void tooltipRequest (xgi::Input* in, xgi::Output* out);
103 void XGI_Turtle (xgi::Input* in, xgi::Output* out);
105 void addStateMachineStatusToXML (
HttpXmlDocument& xmlOut,
const std::string& fsmName,
bool getRunNumber =
true);
106 void addFilteredConfigAliasesToXML (
HttpXmlDocument& xmlOut,
const std::string& fsmName);
107 void addRequiredFsmLogInputToXML (
HttpXmlDocument& xmlOut,
const std::string& fsmName);
108 static std::string getGlobalFieldsString (
ConfigurationManager* cfgMgr,
const std::map<std::string, TableVersion>& memberMap = {});
111 void stateMachineXgiHandler(xgi::Input* in, xgi::Output* out);
112 void stateMachineIterationBreakpoint(xgi::Input* in, xgi::Output* out);
114 static std::string getIconHeaderString(
void);
115 static bool handleAddDesktopIconRequest(
const std::string& author, cgicc::Cgicc& cgiIn,
HttpXmlDocument& xmlOut, std::vector<DesktopIconTable::DesktopIcon>* newIcons =
nullptr);
116 static void handleGetApplicationIdRequest(
AllSupervisorInfo* applicationInfo, cgicc::Cgicc& cgiIn,
HttpXmlDocument& xmlOut, std::map<std::string , std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>* portTranslationMap =
nullptr);
118 xoap::MessageReference stateMachineXoapHandler(xoap::MessageReference msg);
120 bool stateMachineThread(toolbox::task::WorkLoop* workLoop);
123 void statusRequest(xgi::Input* in, xgi::Output* out);
124 void infoRequestResultHandler(xgi::Input* in, xgi::Output* out);
125 bool infoRequestThread(toolbox::task::WorkLoop* workLoop);
128 xoap::MessageReference supervisorCookieCheck(xoap::MessageReference msg);
129 xoap::MessageReference supervisorGetActiveUsers(xoap::MessageReference msg);
130 xoap::MessageReference supervisorSystemMessage(xoap::MessageReference msg);
131 xoap::MessageReference supervisorGetUserInfo(xoap::MessageReference msg);
132 xoap::MessageReference supervisorSystemLogbookEntry(xoap::MessageReference msg);
133 xoap::MessageReference supervisorLastTableGroupRequest(xoap::MessageReference msg);
141 void inError(toolbox::fsm::FiniteStateMachine& fsm)
override;
143 void transitionConfiguring(toolbox::Event::Reference e)
override;
144 void transitionHalting(toolbox::Event::Reference e)
override;
145 void transitionInitializing(toolbox::Event::Reference e)
override;
146 void transitionPausing(toolbox::Event::Reference e)
override;
147 void transitionResuming(toolbox::Event::Reference e)
override;
148 void transitionStarting(toolbox::Event::Reference e)
override;
149 void transitionStopping(toolbox::Event::Reference e)
override;
150 void transitionShuttingDown(toolbox::Event::Reference e)
override;
151 void transitionStartingUp(toolbox::Event::Reference e)
override;
152 void enteringError(toolbox::Event::Reference e)
override;
154 void makeSystemLogEntry(
const std::string& entryText,
const std::string& subjectText =
"",
bool skipFooter =
false);
155 static void addSystemMessage(std::string toUserCSV, std::string message);
157 void checkForAsyncError(
void);
165 unsigned int getNextRunNumber (
const std::string& fsmName =
"");
166 void setNextRunNumber (
unsigned int runNumber,
const std::string& fsmName =
"");
167 std::string getLastLogEntry (
const std::string& logType,
const std::string& fsmName =
"");
168 void setLastLogEntry (
const std::string& logType,
const std::string& logEntry,
const std::string& fsmName =
"");
169 void writeRunInfoTransition (RunInfoVInterface::RunTransitionType transitionType,
const std::string& comment);
172 static xoap::MessageReference lastTableGroupRequestHandler (
const SOAPParameters& parameters);
174 static void launchStartOneServerCommand (
const std::string& command,
ConfigurationManager* cfgMgr,
const std::string& contextName);
177 xoap::MessageReference TRACESupervisorRequest (xoap::MessageReference message);
180 static void AppStatusWorkLoop (
GatewaySupervisor* supervisorPtr,
const bool doDisconnected =
false);
183 std::ostringstream* out,
184 const std::string& command,
185 const std::string& fsmName,
186 const std::string& fsmWindowName,
187 const std::string& username,
188 const std::vector<std::string>& parameters,
189 std::string logEntry =
"");
190 void broadcastMessage (xoap::MessageReference msg);
191 void broadcastMessageToRemoteGateways (
const xoap::MessageReference msg,
unsigned int iteration = 0);
192 void broadcastMessageToRemoteGatewaysComplete (
const xoap::MessageReference msg,
unsigned int iterationIndex = 0);
193 void signalAndWaitForBroadcastThreads (
unsigned int numberOfThreads);
195 struct BroadcastMessageIterationsDoneStruct
201 ~BroadcastMessageIterationsDoneStruct()
203 for (
auto& arr : iterationsDone_)
205 iterationsDone_.clear();
209 void push(
const unsigned int& size)
211 iterationsDone_.push_back(
new bool[size]);
212 arraySizes_.push_back(size);
215 for (
unsigned int i = 0; i < size; ++i)
216 iterationsDone_[iterationsDone_.size() - 1][i] =
false;
219 bool* operator[](
unsigned int i) {
return iterationsDone_[i]; }
220 const bool* operator[](
unsigned int i)
const {
return iterationsDone_[i]; }
221 unsigned int size(
unsigned int i = -1)
223 if (i == (
unsigned int)-1)
224 return iterationsDone_.size();
225 return arraySizes_[i];
229 std::vector<bool*> iterationsDone_;
230 std::vector<unsigned int> arraySizes_;
233 struct BroadcastThreadStruct
236 BroadcastThreadStruct()
246 BroadcastThreadStruct(BroadcastThreadStruct &&b)
247 : threadIndex_(b.threadIndex_)
248 , exitThread_(b.exitThread_.load())
249 , working_(b.working_.load())
250 , workToDo_(b.workToDo_.load())
251 , error_(b.error_.load())
260 xoap::MessageReference message,
261 const std::string& command,
262 const unsigned int& iteration,
263 bool& iterationsDone,
264 std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner)
268 , iteration_(iteration)
269 , iterationsDone_(iterationsDone)
270 , iterationsDoneOwner_(iterationsDoneOwner)
275 xoap::MessageReference message_;
276 const std::string command_;
277 const unsigned int iteration_;
278 bool& iterationsDone_;
282 std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner_;
289 xoap::MessageReference message,
290 const std::string& command,
291 const unsigned int& iteration,
292 bool& iterationsDone,
293 std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner)
297 appInfo, message, command, iteration, iterationsDone, iterationsDoneOwner));
301 const SupervisorInfo& getAppInfo() {
return messages_[0].appInfo_; }
302 xoap::MessageReference getMessage() {
return messages_[0].message_; }
303 const std::string& getCommand() {
return messages_[0].command_; }
304 const unsigned int& getIteration() {
return messages_[0].iteration_; }
305 std::string& getReply() {
return messages_[0].reply_; }
306 bool& getIterationsDone() {
return messages_[0].iterationsDone_; }
309 std::mutex threadMutex_;
310 unsigned int threadIndex_;
311 std::atomic<bool> exitThread_, working_, workToDo_, error_;
313 std::vector<BroadcastThreadStruct::BroadcastMessageStruct> messages_;
316 static void broadcastMessageThread(
318 std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct> threadStruct);
319 bool handleBroadcastMessageTarget(
const SupervisorInfo& appInfo,
320 xoap::MessageReference message,
321 const std::string& command,
322 const unsigned int& iteration,
324 unsigned int threadIndex = 0,
325 const std::atomic<bool>* exitFlag =
nullptr);
330 bool supervisorGuiHasBeenLoaded_;
331 static WebUsers theWebUsers_;
332 std::map<std::string , std::map<std::string ,
336 WorkLoopManager stateMachineWorkLoopManager_;
337 toolbox::BSem stateMachineSemaphore_;
339 std::string activeStateMachineName_;
340 std::string activeStateMachineWindowName_;
341 std::string activeStateMachineDumpFormatOnRun_, activeStateMachineDumpFormatOnConfigure_;
342 std::string activeStateMachineSystemDumpOnRun_, activeStateMachineSystemDumpOnConfigure_;
343 bool activeStateMachineSystemDumpOnRunEnable_, activeStateMachineSystemDumpOnConfigureEnable_;
344 std::string activeStateMachineSystemDumpOnRunFilename_, activeStateMachineSystemDumpOnConfigureFilename_;
345 bool activeStateMachineRequireUserLogOnRun_, activeStateMachineRequireUserLogOnConfigure_;
346 std::string activeStateMachineRunInfoPluginType_;
347 std::map<std::string , std::string >
348 stateMachineConfigureLogEntry_, stateMachineStartLogEntry_, stateMachineStopLogEntry_;
349 std::string activeStateMachineRawStartComment_, activeStateMachineRawStopComment_;
350 std::string activeStateMachineRunNumber_, activeStateMachineRunAlias_, activeStateMachineConfigurationAlias_;
351 bool activeStateMachineRollOverLogOnConfigure_, activeStateMachineRollOverLogOnStart_;
352 std::chrono::steady_clock::time_point
353 activeStateMachineRunStartTime;
354 time_t activeStateMachineRunWallClockStartTime_ = 0;
355 int activeStateMachineRunDuration_ms;
356 bool activeStateMachineWriteToEcl_ =
true;
357 bool activeStateMachineDiscardRun_ =
false;
358 unsigned int activeStateMachineConfigureConditionID_, activeStateMachineRunConditionID_;
359 unsigned int minReadyForEventGenerationStartIteration_ = 0;
360 std::string activeStateMachineSubsystemCommonList_, activeStateMachineSubsystemCommonOverrideList_;
361 std::string activeSubsystemCommonContextList_, activeSubsystemCommonContextOverrideList_;
362 std::string appliedContextCommonList_, appliedContextCommonOverrideList_;
363 std::mutex contextCommonMutex_;
365 std::string cachedSubsystemCommonBackboneKey_;
366 std::string cachedSubsystemCommonList_, cachedSubsystemCommonOverrideList_;
367 std::string cachedSubsystemCommonContextList_, cachedSubsystemCommonContextOverrideList_;
369 std::mutex systemStatusMutex_;
370 std::string lastLogbookEntry_;
371 time_t lastLogbookEntryTime_ = 0;
373 std::string lastConsoleErr_, lastConsoleWarn_, lastConsoleInfo_, lastConsoleErrTime_, lastConsoleWarnTime_, lastConsoleInfoTime_;
374 std::string firstConsoleErr_, firstConsoleWarn_, firstConsoleInfo_, firstConsoleErrTime_, firstConsoleWarnTime_, firstConsoleInfoTime_;
375 size_t systemConsoleErrCount_ = 0, systemConsoleWarnCount_ = 0, systemConsoleInfoCount_ = 0;
377 std::pair<std::string , TableGroupKey>
378 theConfigurationTableGroup_;
379 std::string stateMachineTransitionUsername_;
381 Iterator theIterator_;
382 std::mutex stateMachineAccessMutex_;
384 std::string stateMachineLastCommandInput_;
390 CodeEditor codeEditor_;
392 std::mutex broadcastCommandMessageIndexMutex_;
393 unsigned int broadcastCommandMessageIndex_;
394 std::atomic<bool> broadcastIterationsDone_{
true};
395 std::mutex broadcastIterationBreakpointMutex_;
396 unsigned int broadcastIterationBreakpoint_;
398 std::mutex broadcastCommandStatusUpdateMutex_;
399 std::string broadcastCommandStatus_;
401 std::mutex remoteIterationMutex_;
402 std::condition_variable remoteIterationCV_;
403 unsigned int remoteIterationIndex_ = 0;
404 std::atomic<bool> isRemoteSubsystemIteration_{
false};
405 std::atomic<bool> remoteSubsystemErrorReceived_{
false};
407 static std::vector<std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct>> broadcastThreadStructs_;
409 std::string securityType_;
413 unsigned int conditionID_;
430 time_t errorTimestamp = 0;
432 void setError(
const std::string& err) { error = err; errorTimestamp = time(0); }
433 void clearError() { error =
""; errorTimestamp = 0; }
434 void copyError(
const RemoteGatewayInfo& r) { error = r.error; errorTimestamp = r.errorTimestamp; }
435 const std::string& getError()
const {
return error; }
438 std::string config_dump;
445 size_t consoleErrCount = 0, consoleWarnCount = 0;
447 std::string fullName;
455 std::string usernameWithLock;
464 bool fsm_included =
true;
466 std::string getFsmMode()
const {
469 case FSM_ModeTypes::Follow_FSM:
return "Follow FSM";
472 default:
return "Impossible";
476 std::string getConfigDumpType()
const {
477 switch(config_dump_type)
479 case ConfigDumpTypes::Text:
return "Text";
480 case ConfigDumpTypes::JSON_all:
return "JSON all";
481 default:
return "Unknown";
485 std::map<std::string, SupervisorInfo::SubappInfo>
subapps;
489 std::string activeContextGroupName, activeConfigGroupName;
493 std::string selectedConfigGroupName;
495 bool doNotHaltWasCommandedHalt =
false;
498 std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteGatewayApps_;
499 std::mutex remoteGatewayAppsMutex_;
500 std::map<std::string ,
501 bool > appLastStatusGood_;
502 std::mutex dualStatusThreadMutex_;
510 std::mutex latestGatewayIconsMutex_;
519 std::string cachedGlobalFieldsString_;
520 std::pair<std::string, TableGroupKey> cachedGlobalFieldsGroup_;
522 static void CheckRemoteGatewayStatus (
GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp,
const std::unique_ptr<TransceiverSocket>& remoteGatewaySocket,
const std::string& ipForReverseLoginOverUDP,
int portForReverseLoginOverUDP,
const std::string& contextCommonList =
"",
const std::string& contextCommonOverrideList =
"");
524 static void applyContextCommonTables (
GatewaySupervisor* supervisor,
const std::string& contextCommonList,
const std::string& contextCommonOverrideList);
526 void loadRemoteGatewaySettings (std::vector<GatewaySupervisor::RemoteGatewayInfo>& remoteGateways,
bool onlyNotFound =
false)
const;
527 void saveRemoteGatewaySettings (
void)
const;
528 static std::string translateURLForRequestOrigin (
const std::string& url,
const std::string& requestOrigin, std::map<std::string , std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>& portTranslationMap);
529 static std::string translateRemoteIconStringForRequestOrigin (
const std::string& iconString,
const std::string& requestOrigin, std::map<std::string , std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>& portTranslationMap);
friend class GatewaySupervisor
for access to indicateOtsAlive()
std::string latestGatewayRemoteIconsString_
cached string of remote gateway icons for quick access
virtual void setSupervisorPropertyDefaults(void) override
override to control supervisor specific defaults
void stateHalted(toolbox::fsm::FiniteStateMachine &fsm) override
virtual void forceSupervisorPropertyValues(void) override
override to force supervisor property values (and ignore user settings)
std::vector< DesktopIconTable::DesktopIcon > latestGatewayIcons_
used to track the latest desktop icons (which are defined by the active context but allowed to change...
void statePaused(toolbox::fsm::FiniteStateMachine &fsm) override
std::string ipAddressForStateChangesOverUDP_
IP used for UDP reverse-login propagation to remote gateways.
void stateRunning(toolbox::fsm::FiniteStateMachine &fsm) override
int portForReverseLoginOverUDP_
UDP port for reverse-login; 0 = disabled.
std::map< unsigned int, SupervisorInfo > localAllSupervisorInfo_
only use in main thread, stable copy of app status
std::pair< std::string, TableGroupKey > latestGatewayIconsContextGroup_
used to track the table group key for the latest desktop icons
void stateConfigured(toolbox::fsm::FiniteStateMachine &fsm) override
void stateInitial(toolbox::fsm::FiniteStateMachine &fsm) override
std::pair< std::string, TableGroupKey > latestGatewayRemoteIconsContextGroup_
used to track the table group key for the latest remote desktop icons
defines used also by OtsConfigurationWizardSupervisor
ConfigDumpTypes
<FSM Modes: 'Follow FSM,' 'Do not Halt' (artdaq), or 'Only Configure' (DCS/DQM)
FSM_ModeTypes
<FSM Modes: 'Follow FSM,' 'Do not Halt' (artdaq), or 'Only Configure' (DCS/DQM)
@ OnlyConfigure
(e.g. for DCS/DQM)
@ DoNotHalt
(e.g. for artdaq)
TableGroupKey activeConfigGroupKey
selected_config_alias resolved to a group name+key by the remote subsystem itself (against its own ac...
std::string permissionThresholdString
used for desktop icons
time_t relaunchTime
timestamp of last relaunch via gatewayLaunchOTSInstance
std::set< std::string > config_aliases
used for remote gateway subapp control
std::string selected_config_alias
used for remote gateway subapp control
FSM_ModeTypes fsm_mode
used for remote gateway subapp control
std::string fsmName
when not "", need to send
bool iterationsDone
tracks per-gateway iteration completion during FSM transitions
std::map< std::string, SupervisorInfo::SubappInfo > subapps
remote gateways can have subapps
size_t ignoreStatusCount
if non-zero, do not ask for status
std::string instanceUser
used for remote ots instance ssh launch
time_t commandSentTime
timestamp of last command send; suppresses stale status write-backs briefly
std::string user_data_path_record
used for remote gateway subapp control
static std::string getTimestampString(const std::string &linuxTimeInSeconds)