1 #ifndef _ots_GatewaySupervisor_h
2 #define _ots_GatewaySupervisor_h
4 #include <condition_variable>
7 #include "otsdaq/CoreSupervisors/ConfigurationSupervisorBase.h"
8 #include "otsdaq/CoreSupervisors/CorePropertySupervisorBase.h"
9 #include "otsdaq/FiniteStateMachine/RunControlStateMachine.h"
10 #include "otsdaq/GatewaySupervisor/Iterator.h"
11 #include "otsdaq/SOAPUtilities/SOAPMessenger.h"
12 #include "otsdaq/SupervisorInfo/AllSupervisorInfo.h"
13 #include "otsdaq/WebUsersUtilities/WebUsers.h"
14 #include "otsdaq/WorkLoopManager/WorkLoopManager.h"
16 #include "otsdaq/CodeEditor/CodeEditor.h"
17 #include "otsdaq/TablePlugins/DesktopIconTable.h"
19 #include "otsdaq/NetworkUtilities/TransceiverSocket.h"
21 #pragma GCC diagnostic push
22 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
23 #include <xdaq/Application.h>
24 #pragma GCC diagnostic pop
25 #include "otsdaq/Macros/XDAQApplicationMacros.h"
27 #include <toolbox/task/WorkLoop.h>
28 #include <xdata/String.h>
29 #include <xgi/Method.h>
30 #include "otsdaq/GatewaySupervisor/PixelHistoPicGen.h"
63 class ConfigurationManager;
65 class WorkLoopManager;
81 static const std::string COMMAND_PARAM_LOG_ENTRY_PREAMBLE;
82 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_PREAMBLE;
83 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_OVERRIDE_PREAMBLE;
84 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_CONTEXT_PREAMBLE;
85 static const std::string COMMAND_PARAM_SUBSYSTEM_COMMON_CONTEXT_OVERRIDE_PREAMBLE;
86 static const std::string COMMAND_PARAM_ITERATION_INDEX_PREAMBLE;
96 void Default (xgi::Input* in, xgi::Output* out);
98 void loginRequest (xgi::Input* in, xgi::Output* out);
99 void request (xgi::Input* in, xgi::Output* out);
100 void tooltipRequest (xgi::Input* in, xgi::Output* out);
101 void XGI_Turtle (xgi::Input* in, xgi::Output* out);
103 void addStateMachineStatusToXML (
HttpXmlDocument& xmlOut,
const std::string& fsmName,
bool getRunNumber =
true);
104 void addFilteredConfigAliasesToXML (
HttpXmlDocument& xmlOut,
const std::string& fsmName);
105 void addRequiredFsmLogInputToXML (
HttpXmlDocument& xmlOut,
const std::string& fsmName);
106 static std::string getGlobalFieldsString (
ConfigurationManager* cfgMgr,
const std::map<std::string, TableVersion>& memberMap = {});
109 void stateMachineXgiHandler(xgi::Input* in, xgi::Output* out);
110 void stateMachineIterationBreakpoint(xgi::Input* in, xgi::Output* out);
112 static std::string getIconHeaderString(
void);
113 static bool handleAddDesktopIconRequest(
const std::string& author, cgicc::Cgicc& cgiIn,
HttpXmlDocument& xmlOut, std::vector<DesktopIconTable::DesktopIcon>* newIcons =
nullptr);
114 static void handleGetApplicationIdRequest(
AllSupervisorInfo* applicationInfo, cgicc::Cgicc& cgiIn,
HttpXmlDocument& xmlOut, std::map<std::string , std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>* portTranslationMap =
nullptr);
116 xoap::MessageReference stateMachineXoapHandler(xoap::MessageReference msg);
118 bool stateMachineThread(toolbox::task::WorkLoop* workLoop);
121 void statusRequest(xgi::Input* in, xgi::Output* out);
122 void infoRequestResultHandler(xgi::Input* in, xgi::Output* out);
123 bool infoRequestThread(toolbox::task::WorkLoop* workLoop);
126 xoap::MessageReference supervisorCookieCheck(xoap::MessageReference msg);
127 xoap::MessageReference supervisorGetActiveUsers(xoap::MessageReference msg);
128 xoap::MessageReference supervisorSystemMessage(xoap::MessageReference msg);
129 xoap::MessageReference supervisorGetUserInfo(xoap::MessageReference msg);
130 xoap::MessageReference supervisorSystemLogbookEntry(xoap::MessageReference msg);
131 xoap::MessageReference supervisorLastTableGroupRequest(xoap::MessageReference msg);
139 void inError(toolbox::fsm::FiniteStateMachine& fsm)
override;
141 void transitionConfiguring(toolbox::Event::Reference e)
override;
142 void transitionHalting(toolbox::Event::Reference e)
override;
143 void transitionInitializing(toolbox::Event::Reference e)
override;
144 void transitionPausing(toolbox::Event::Reference e)
override;
145 void transitionResuming(toolbox::Event::Reference e)
override;
146 void transitionStarting(toolbox::Event::Reference e)
override;
147 void transitionStopping(toolbox::Event::Reference e)
override;
148 void transitionShuttingDown(toolbox::Event::Reference e)
override;
149 void transitionStartingUp(toolbox::Event::Reference e)
override;
150 void enteringError(toolbox::Event::Reference e)
override;
152 void makeSystemLogEntry(
const std::string& entryText,
const std::string& subjectText =
"");
153 static void addSystemMessage(std::string toUserCSV, std::string message);
155 void checkForAsyncError(
void);
163 unsigned int getNextRunNumber (
const std::string& fsmName =
"");
164 void setNextRunNumber (
unsigned int runNumber,
const std::string& fsmName =
"");
165 std::string getLastLogEntry (
const std::string& logType,
const std::string& fsmName =
"");
166 void setLastLogEntry (
const std::string& logType,
const std::string& logEntry,
const std::string& fsmName =
"");
169 static xoap::MessageReference lastTableGroupRequestHandler (
const SOAPParameters& parameters);
171 static void launchStartOneServerCommand (
const std::string& command,
ConfigurationManager* cfgMgr,
const std::string& contextName);
174 xoap::MessageReference TRACESupervisorRequest (xoap::MessageReference message);
177 static void AppStatusWorkLoop (
GatewaySupervisor* supervisorPtr,
const bool doDisconnected =
false);
180 std::ostringstream* out,
181 const std::string& command,
182 const std::string& fsmName,
183 const std::string& fsmWindowName,
184 const std::string& username,
185 const std::vector<std::string>& parameters,
186 std::string logEntry =
"");
187 void broadcastMessage (xoap::MessageReference msg);
188 void broadcastMessageToRemoteGateways (
const xoap::MessageReference msg,
unsigned int iteration = 0);
189 void broadcastMessageToRemoteGatewaysComplete (
const xoap::MessageReference msg,
unsigned int iterationIndex = 0);
190 void signalAndWaitForBroadcastThreads (
unsigned int numberOfThreads);
192 struct BroadcastMessageIterationsDoneStruct
198 ~BroadcastMessageIterationsDoneStruct()
200 for (
auto& arr : iterationsDone_)
202 iterationsDone_.clear();
206 void push(
const unsigned int& size)
208 iterationsDone_.push_back(
new bool[size]);
209 arraySizes_.push_back(size);
212 for (
unsigned int i = 0; i < size; ++i)
213 iterationsDone_[iterationsDone_.size() - 1][i] =
false;
216 bool* operator[](
unsigned int i) {
return iterationsDone_[i]; }
217 const bool* operator[](
unsigned int i)
const {
return iterationsDone_[i]; }
218 unsigned int size(
unsigned int i = -1)
220 if (i == (
unsigned int)-1)
221 return iterationsDone_.size();
222 return arraySizes_[i];
226 std::vector<bool*> iterationsDone_;
227 std::vector<unsigned int> arraySizes_;
230 struct BroadcastThreadStruct
233 BroadcastThreadStruct()
243 BroadcastThreadStruct(BroadcastThreadStruct &&b)
244 : threadIndex_(b.threadIndex_)
245 , exitThread_(b.exitThread_.load())
246 , working_(b.working_.load())
247 , workToDo_(b.workToDo_.load())
248 , error_(b.error_.load())
257 xoap::MessageReference message,
258 const std::string& command,
259 const unsigned int& iteration,
260 bool& iterationsDone,
261 std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner)
265 , iteration_(iteration)
266 , iterationsDone_(iterationsDone)
267 , iterationsDoneOwner_(iterationsDoneOwner)
272 xoap::MessageReference message_;
273 const std::string command_;
274 const unsigned int iteration_;
275 bool& iterationsDone_;
279 std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner_;
286 xoap::MessageReference message,
287 const std::string& command,
288 const unsigned int& iteration,
289 bool& iterationsDone,
290 std::shared_ptr<BroadcastMessageIterationsDoneStruct> iterationsDoneOwner)
294 appInfo, message, command, iteration, iterationsDone, iterationsDoneOwner));
298 const SupervisorInfo& getAppInfo() {
return messages_[0].appInfo_; }
299 xoap::MessageReference getMessage() {
return messages_[0].message_; }
300 const std::string& getCommand() {
return messages_[0].command_; }
301 const unsigned int& getIteration() {
return messages_[0].iteration_; }
302 std::string& getReply() {
return messages_[0].reply_; }
303 bool& getIterationsDone() {
return messages_[0].iterationsDone_; }
306 std::mutex threadMutex_;
307 unsigned int threadIndex_;
308 std::atomic<bool> exitThread_, working_, workToDo_, error_;
310 std::vector<BroadcastThreadStruct::BroadcastMessageStruct> messages_;
313 static void broadcastMessageThread(
315 std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct> threadStruct);
316 bool handleBroadcastMessageTarget(
const SupervisorInfo& appInfo,
317 xoap::MessageReference message,
318 const std::string& command,
319 const unsigned int& iteration,
321 unsigned int threadIndex = 0,
322 const std::atomic<bool>* exitFlag =
nullptr);
327 bool supervisorGuiHasBeenLoaded_;
328 static WebUsers theWebUsers_;
329 std::map<std::string , std::map<std::string ,
333 WorkLoopManager stateMachineWorkLoopManager_;
334 toolbox::BSem stateMachineSemaphore_;
336 std::string activeStateMachineName_;
337 std::string activeStateMachineWindowName_;
338 std::string activeStateMachineDumpFormatOnRun_, activeStateMachineDumpFormatOnConfigure_;
339 std::string activeStateMachineSystemDumpOnRun_, activeStateMachineSystemDumpOnConfigure_;
340 bool activeStateMachineSystemDumpOnRunEnable_, activeStateMachineSystemDumpOnConfigureEnable_;
341 std::string activeStateMachineSystemDumpOnRunFilename_, activeStateMachineSystemDumpOnConfigureFilename_;
342 bool activeStateMachineRequireUserLogOnRun_, activeStateMachineRequireUserLogOnConfigure_;
343 std::string activeStateMachineRunInfoPluginType_;
344 std::map<std::string , std::string >
345 stateMachineConfigureLogEntry_, stateMachineStartLogEntry_, stateMachineStopLogEntry_;
346 std::string activeStateMachineRunNumber_, activeStateMachineRunAlias_, activeStateMachineConfigurationAlias_;
347 bool activeStateMachineRollOverLogOnConfigure_, activeStateMachineRollOverLogOnStart_;
348 std::chrono::steady_clock::time_point
349 activeStateMachineRunStartTime;
350 int activeStateMachineRunDuration_ms;
351 unsigned int activeStateMachineConfigureConditionID_, activeStateMachineRunConditionID_;
352 std::string activeStateMachineSubsystemCommonList_, activeStateMachineSubsystemCommonOverrideList_;
353 std::string activeSubsystemCommonContextList_, activeSubsystemCommonContextOverrideList_;
354 std::string appliedContextCommonList_, appliedContextCommonOverrideList_;
355 std::mutex contextCommonMutex_;
357 std::string cachedSubsystemCommonBackboneKey_;
358 std::string cachedSubsystemCommonList_, cachedSubsystemCommonOverrideList_;
359 std::string cachedSubsystemCommonContextList_, cachedSubsystemCommonContextOverrideList_;
361 std::mutex systemStatusMutex_;
362 std::string lastLogbookEntry_;
363 time_t lastLogbookEntryTime_ = 0;
365 std::string lastConsoleErr_, lastConsoleWarn_, lastConsoleInfo_, lastConsoleErrTime_, lastConsoleWarnTime_, lastConsoleInfoTime_;
366 std::string firstConsoleErr_, firstConsoleWarn_, firstConsoleInfo_, firstConsoleErrTime_, firstConsoleWarnTime_, firstConsoleInfoTime_;
367 size_t systemConsoleErrCount_ = 0, systemConsoleWarnCount_ = 0, systemConsoleInfoCount_ = 0;
369 std::pair<std::string , TableGroupKey>
370 theConfigurationTableGroup_;
371 std::string stateMachineTransitionUsername_;
373 Iterator theIterator_;
374 std::mutex stateMachineAccessMutex_;
376 std::string stateMachineLastCommandInput_;
382 CodeEditor codeEditor_;
384 std::mutex broadcastCommandMessageIndexMutex_;
385 unsigned int broadcastCommandMessageIndex_;
386 std::atomic<bool> broadcastIterationsDone_{
true};
387 std::mutex broadcastIterationBreakpointMutex_;
388 unsigned int broadcastIterationBreakpoint_;
390 std::mutex broadcastCommandStatusUpdateMutex_;
391 std::string broadcastCommandStatus_;
393 std::mutex remoteIterationMutex_;
394 std::condition_variable remoteIterationCV_;
395 unsigned int remoteIterationIndex_ = 0;
396 std::atomic<bool> isRemoteSubsystemIteration_{
false};
398 static std::vector<std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct>> broadcastThreadStructs_;
400 std::string securityType_;
404 unsigned int conditionID_;
421 time_t errorTimestamp = 0;
423 void setError(
const std::string& err) { error = err; errorTimestamp = time(0); }
424 void clearError() { error =
""; errorTimestamp = 0; }
425 void copyError(
const RemoteGatewayInfo& r) { error = r.error; errorTimestamp = r.errorTimestamp; }
426 const std::string& getError()
const {
return error; }
429 std::string config_dump;
434 size_t consoleErrCount = 0, consoleWarnCount = 0;
436 std::string fullName;
444 std::string usernameWithLock;
453 bool fsm_included =
true;
455 std::string getFsmMode()
const {
458 case FSM_ModeTypes::Follow_FSM:
return "Follow FSM";
461 default:
return "Impossible";
465 std::string getConfigDumpType()
const {
466 switch(config_dump_type)
468 case ConfigDumpTypes::Text:
return "Text";
469 case ConfigDumpTypes::JSON_all:
return "JSON all";
470 default:
return "Unknown";
474 std::map<std::string, SupervisorInfo::SubappInfo>
subapps;
478 std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteGatewayApps_;
479 std::mutex remoteGatewayAppsMutex_;
480 std::map<std::string ,
481 bool > appLastStatusGood_;
482 std::mutex dualStatusThreadMutex_;
490 std::mutex latestGatewayIconsMutex_;
499 std::string cachedGlobalFieldsString_;
500 std::pair<std::string, TableGroupKey> cachedGlobalFieldsGroup_;
502 static void CheckRemoteGatewayStatus (
GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp,
const std::unique_ptr<TransceiverSocket>& remoteGatewaySocket,
const std::string& ipForReverseLoginOverUDP,
int portForReverseLoginOverUDP,
const std::string& contextCommonList =
"",
const std::string& contextCommonOverrideList =
"");
504 static void applyContextCommonTables (
GatewaySupervisor* supervisor,
const std::string& contextCommonList,
const std::string& contextCommonOverrideList);
506 void loadRemoteGatewaySettings (std::vector<GatewaySupervisor::RemoteGatewayInfo>& remoteGateways,
bool onlyNotFound =
false)
const;
507 void saveRemoteGatewaySettings (
void)
const;
508 static std::string translateURLForRequestOrigin (
const std::string& url,
const std::string& requestOrigin, std::map<std::string , std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>& portTranslationMap);
509 static std::string translateRemoteIconStringForRequestOrigin (
const std::string& iconString,
const std::string& requestOrigin, std::map<std::string , std::map<std::string /* requestUrlHostPort */, std::string /* translatedHostPort */>>& portTranslationMap);
friend class GatewaySupervisor
for access to indicateOtsAlive()
std::string latestGatewayRemoteIconsString_
cached string of remote gateway icons for quick access
virtual void setSupervisorPropertyDefaults(void) override
override to control supervisor specific defaults
void stateHalted(toolbox::fsm::FiniteStateMachine &fsm) override
virtual void forceSupervisorPropertyValues(void) override
override to force supervisor property values (and ignore user settings)
std::vector< DesktopIconTable::DesktopIcon > latestGatewayIcons_
used to track the latest desktop icons (which are defined by the active context but allowed to change...
void statePaused(toolbox::fsm::FiniteStateMachine &fsm) override
std::string ipAddressForStateChangesOverUDP_
IP used for UDP reverse-login propagation to remote gateways.
void stateRunning(toolbox::fsm::FiniteStateMachine &fsm) override
int portForReverseLoginOverUDP_
UDP port for reverse-login; 0 = disabled.
std::map< unsigned int, SupervisorInfo > localAllSupervisorInfo_
only use in main thread, stable copy of app status
std::pair< std::string, TableGroupKey > latestGatewayIconsContextGroup_
used to track the table group key for the latest desktop icons
void stateConfigured(toolbox::fsm::FiniteStateMachine &fsm) override
void stateInitial(toolbox::fsm::FiniteStateMachine &fsm) override
std::pair< std::string, TableGroupKey > latestGatewayRemoteIconsContextGroup_
used to track the table group key for the latest remote desktop icons
defines used also by OtsConfigurationWizardSupervisor
ConfigDumpTypes
<FSM Modes: 'Follow FSM,' 'Do not Halt' (artdaq), or 'Only Configure' (DCS/DQM)
FSM_ModeTypes
<FSM Modes: 'Follow FSM,' 'Do not Halt' (artdaq), or 'Only Configure' (DCS/DQM)
@ OnlyConfigure
(e.g. for DCS/DQM)
@ DoNotHalt
(e.g. for artdaq)
std::string permissionThresholdString
used for desktop icons
std::set< std::string > config_aliases
used for remote gateway subapp control
std::string selected_config_alias
used for remote gateway subapp control
FSM_ModeTypes fsm_mode
used for remote gateway subapp control
std::string fsmName
when not "", need to send
bool iterationsDone
tracks per-gateway iteration completion during FSM transitions
std::map< std::string, SupervisorInfo::SubappInfo > subapps
remote gateways can have subapps
size_t ignoreStatusCount
if non-zero, do not ask for status
std::string instanceUser
used for remote ots instance ssh launch
std::string user_data_path_record
used for remote gateway subapp control
static std::string getTimestampString(const std::string &linuxTimeInSeconds)