otsdaq-utilities  3.02.00
ConsoleSupervisor.cc
1 #include "otsdaq-utilities/Console/ConsoleSupervisor.h"
2 #include <xdaq/NamespaceURI.h>
3 #include "otsdaq/CgiDataUtilities/CgiDataUtilities.h"
4 #include "otsdaq/Macros/CoutMacros.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/NetworkUtilities/ReceiverSocket.h"
7 #include "otsdaq/XmlUtilities/HttpXmlDocument.h"
8 
9 #include <dirent.h> //for DIR
10 #include <sys/stat.h> //for mkdir
11 #include <fstream>
12 #include <iostream>
13 #include <string>
14 #include <thread> //for std::thread
15 
16 using namespace ots;
17 
22 XDAQ_INSTANTIATOR_IMPL(ConsoleSupervisor)
23 
24 #define USER_CONSOLE_PREF_PATH \
25  std::string(__ENV__("SERVICE_DATA_PATH")) + "/ConsolePreferences/"
26 #define USER_CONSOLE_SNAPSHOT_PATH \
27  std::string(__ENV__("SERVICE_DATA_PATH")) + "/ConsoleSnapshots/"
28 #define USERS_PREFERENCES_FILETYPE "pref"
29 #define CUSTOM_COUNT_LIST_FILENAME std::string("CustomCountList.dat")
30 
31 #define QUIET_CFG_FILE \
32  std::string(__ENV__("USER_DATA")) + \
33  "/MessageFacilityConfigurations/" \
34  "QuietForwarder.cfg"
35 
36 #define CONSOLE_SPECIAL_ERROR \
37  std::string("|30-Aug-2019 15:30:17 CDT|0|||Error|Console||-1||ConsoleSupervisor|") + \
38  std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
39  std::string("|")
40 #define CONSOLE_SPECIAL_WARNING \
41  std::string( \
42  "|30-Aug-2019 15:30:17 CDT|0|||Warning|Console||-1||ConsoleSupervisor|") + \
43  std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
44  std::string("|")
45 
46 #undef __MF_SUBJECT__
47 #define __MF_SUBJECT__ "Console"
48 
49 #define CONSOLE_MISSED_NEEDLE "Console missed * packet(s)"
50 
52 const std::set<std::string> ConsoleSupervisor::CUSTOM_TRIGGER_ACTIONS({"Count Only",
53  "System Message",
54  "Halt",
55  "Stop",
56  "Pause",
57  "Soft Error",
58  "Hard Error",
59  "Run Script"});
60 
61 const std::string ConsoleSupervisor::ConsoleMessageStruct::LABEL_TRACE = "TRACE";
62 const std::string ConsoleSupervisor::ConsoleMessageStruct::LABEL_TRACE_PLUS = "TRACE+";
63 const std::map<ConsoleSupervisor::ConsoleMessageStruct::FieldType,
64  std::string /* fieldNames */>
65  ConsoleSupervisor::ConsoleMessageStruct::fieldNames({
66  {FieldType::TIMESTAMP, "Timestamp"},
67  {FieldType::SEQID, "SequenceID"},
68  {FieldType::LEVEL, "Level"},
69  {FieldType::LABEL, "Label"},
70  {FieldType::SOURCEID, "SourceID"},
71  {FieldType::HOSTNAME, "Hostname"},
72  {FieldType::SOURCE, "Source"},
73  {FieldType::FILE, "File"},
74  {FieldType::LINE, "Line"},
75  {FieldType::MSG, "Msg"},
76  });
77 
78 //==============================================================================
79 ConsoleSupervisor::ConsoleSupervisor(xdaq::ApplicationStub* stub)
80  : CoreSupervisorBase(stub)
81  , messageCount_(0)
82  , maxMessageCount_(100000)
83  , maxClientMessageRequest_(500)
84 {
85  __SUP_COUT__ << "Constructor started." << __E__;
86 
87  INIT_MF("." /*directory used is USER_DATA/LOG/.*/);
88 
89  // attempt to make directory structure (just in case)
90  mkdir(((std::string)USER_CONSOLE_PREF_PATH).c_str(), 0755);
91  mkdir(((std::string)USER_CONSOLE_SNAPSHOT_PATH).c_str(), 0755);
92 
93  xoap::bind(
94  this, &ConsoleSupervisor::resetConsoleCounts, "ResetConsoleCounts", XDAQ_NS_URI);
95 
96  init();
97 
98  //test custom count list and always force the first one to be the "Console missed" !
99  //Note: to avoid recursive triggers, Label='Console' can not trigger, so this 1st one is the only Console source trigger!
100  // Custom Trigger Philosophy
101  // - only one global custom count priority list (too complicated to manage by user, would have to timeout logins, etc..)
102  // - the admin users can modify priority of items in prority list
103  // - actions can infer others ('Count' is always inferred, in addition 'System Message' is inferred for FSM actions)
104  // - the admin users can modify actions
105  // -- including to the 1st Console-missing-message custom count
106  loadCustomCountList();
107  if(priorityCustomTriggerList_.size() ==
108  0) //then create default starting list example for user
109  {
110  addCustomTriggeredAction(CONSOLE_MISSED_NEEDLE, "System Message");
111  // addCustomTriggeredAction("runtime*4",
112  // "Halt");
113  // addCustomTriggeredAction("runtime",
114  // "System Message");
115  } //end test and force custom count list
116 
117  __SUP_COUT__ << "Constructor complete." << __E__;
118 
119 } // end constructor()
120 
121 //==============================================================================
122 ConsoleSupervisor::~ConsoleSupervisor(void) { destroy(); }
123 //==============================================================================
124 void ConsoleSupervisor::init(void)
125 {
126  // start mf msg listener
127  std::thread(
128  [](ConsoleSupervisor* cs) {
129  ConsoleSupervisor::messageFacilityReceiverWorkLoop(cs);
130  },
131  this)
132  .detach();
133 } // end init()
134 
135 //==============================================================================
136 void ConsoleSupervisor::destroy(void)
137 {
138  // called by destructor
139 } // end destroy()
140 
141 //==============================================================================
142 xoap::MessageReference ConsoleSupervisor::resetConsoleCounts(
143  xoap::MessageReference /*message*/)
144 {
145  __COUT_INFO__ << "Resetting Console Error/Warn/Info counts and preparing for new "
146  "first messages."
147  << __E__;
148 
149  // lockout the messages array for the remainder of the scope
150  // this guarantees the reading thread can safely access the messages
151  std::lock_guard<std::mutex> lock(messageMutex_);
152  errorCount_ = 0;
153  firstErrorMessageTime_ = 0;
154  lastErrorMessageTime_ = 0;
155  warnCount_ = 0;
156  firstWarnMessageTime_ = 0;
157  lastWarnMessageTime_ = 0;
158  infoCount_ = 0;
159  firstInfoMessageTime_ = 0;
160  lastInfoMessageTime_ = 0;
161 
162  return SOAPUtilities::makeSOAPMessageReference("Done");
163 } // end resetConsoleCounts()
164 
165 //==============================================================================
169 void ConsoleSupervisor::messageFacilityReceiverWorkLoop(ConsoleSupervisor* cs)
170 try
171 {
172  __COUT__ << "Starting workloop based on config file: " << QUIET_CFG_FILE << __E__;
173 
174  std::string configFile = QUIET_CFG_FILE;
175  FILE* fp = fopen(configFile.c_str(), "r");
176  if(!fp)
177  {
178  __SS__ << "File with port info could not be loaded: " << QUIET_CFG_FILE << __E__;
179  __COUT__ << "\n" << ss.str();
180  __SS_THROW__;
181  }
182  char tmp[100];
183  fgets(tmp, 100, fp); // receive port (ignore)
184  fgets(tmp, 100, fp); // destination port *** used here ***
185  int myport;
186  sscanf(tmp, "%*s %d", &myport);
187 
188  fgets(tmp, 100, fp); // destination ip *** used here ***
189  char myip[100];
190  sscanf(tmp, "%*s %s", myip);
191  fclose(fp);
192 
193  ReceiverSocket rsock(myip, myport); // Take Port from Configuration
194  try
195  {
196  rsock.initialize(0x1400000 /*socketReceiveBufferSize*/);
197  }
198  catch(...)
199  {
200  // lockout the messages array for the remainder of the scope
201  // this guarantees the reading thread can safely access the messages
202  std::lock_guard<std::mutex> lock(cs->messageMutex_);
203 
204  // NOTE: if we do not want this to be fatal, do not throw here, just print out
205 
206  if(1) // generate special message and throw for failed socket
207  {
208  __SS__ << "FATAL Console error. Could not initialize socket on port "
209  << myport
210  << ". Perhaps the port is already in use? Check for multiple stale "
211  "instances of otsdaq processes, or notify admins."
212  << " Multiple instances of otsdaq on the same node should be "
213  "possible, but port numbers must be unique."
214  << __E__;
215  __SS_THROW__;
216  }
217 
218  // generate special message to indicate failed socket
219  __SS__ << "FATAL Console error. Could not initialize socket on port " << myport
220  << ". Perhaps it is already in use? Exiting Console receive loop."
221  << __E__;
222  __COUT__ << ss.str();
223 
224  cs->messages_.emplace_back(CONSOLE_SPECIAL_ERROR + ss.str(),
225  cs->messageCount_++,
226  cs->priorityCustomTriggerList_);
227  //force time to now for auto-generated message
228  cs->messages_.back().setTime(time(0));
229  if(cs->messages_.size() > cs->maxMessageCount_)
230  {
231  cs->messages_.erase(cs->messages_.begin());
232  }
233 
234  return;
235  }
236 
237  std::string buffer;
238  int i = 0;
239  int heartbeatCount = 0;
240  int selfGeneratedMessageCount = 0;
241 
242  std::map<unsigned int, unsigned int> sourceLastSequenceID; // map from sourceID to
243  // lastSequenceID to
244  // identify missed messages
245  long long newSourceId;
246  uint32_t newSequenceId;
247  unsigned int c;
248 
249  // force a starting message
250  __COUT__ << "DEBUG messages look like this." << __E__;
251 
252  while(1)
253  {
254  // if receive succeeds display message
255 
256  //__COUTV__(i);
257 
258  if(rsock.receive(
259  buffer, 1 /*timeoutSeconds*/, 0 /*timeoutUSeconds*/, false /*verbose*/) !=
260  -1)
261  {
262  // use 1-byte "ping" to keep socket alive
263  if(buffer.size() == 1)
264  {
265  // std::cout << "Ping!" << __E__;
266  continue;
267  }
268 
269  if(i != 200)
270  {
271  __COUT__ << "Console has first message." << __E__;
272  i = 200; // mark so things are good for all time. (this indicates things
273  // are configured to be sent here)
274 
275  __COUT_INFO__ << "INFO messages look like this." << __E__;
276  __COUT_WARN__ << "WARNING messages look like this." << __E__;
277  __COUT_ERR__ << "ERROR messages look like this." << __E__;
278 
279  // //to debug special packets
280  // __SS__ << "???";
281  // cs->messages_[cs->writePointer_].set(CONSOLE_SPECIAL_ERROR
282  //+ ss.str(),
283  // cs->messageCount_++);
284  //
285  // if(++cs->writePointer_ == cs->messages_.size()) //handle
286  // wrap-around cs->writePointer_ = 0;
287  }
288 
289  if(selfGeneratedMessageCount)
290  --selfGeneratedMessageCount; // decrement internal message count
291  else // reset heartbeat if external messages are coming through
292  heartbeatCount = 0;
293 
294  //__COUT__ << buffer << __E__;
295 
296  // lockout the messages array for the remainder of the scope
297  // this guarantees the reading thread can safely access the messages
298  std::lock_guard<std::mutex> lock(cs->messageMutex_);
299 
300  // handle message stacking in packet
301  c = 0;
302  while(c < buffer.size())
303  {
304  // std::cout << "CONSOLE " << c << " sz=" << buffer.size() << " len=" <<
305  // strlen(&(buffer.c_str()[c])) << __E__;
306  cs->messages_.emplace_back(&(buffer.c_str()[c]),
307  cs->messageCount_++,
308  cs->priorityCustomTriggerList_);
309  if(cs->messages_.back().hasCustomTriggerMatchAction())
310  cs->customTriggerActionQueue_.push(
311  cs->messages_.back().getCustomTriggerMatch());
312 
313  //update system status
314  if(cs->messages_.back().getLevel() == "Error")
315  {
316  if(cs->errorCount_ == 0)
317  {
318  cs->firstErrorMessageTime_ = time(0);
319  cs->firstErrorMessage_ = cs->messages_.back().getMsg();
320  }
321 
322  cs->lastErrorMessageTime_ = time(0);
323  ++cs->errorCount_;
324  cs->lastErrorMessage_ = cs->messages_.back().getMsg();
325  }
326  else if(cs->messages_.back().getLevel() == "Warning")
327  {
328  if(cs->warnCount_ == 0)
329  {
330  cs->firstWarnMessageTime_ = time(0);
331  cs->firstWarnMessage_ = cs->messages_.back().getMsg();
332  }
333  cs->lastWarnMessageTime_ = time(0);
334  ++cs->warnCount_;
335  cs->lastWarnMessage_ = cs->messages_.back().getMsg();
336  }
337  else if(cs->messages_.back().getLevel() == "Info")
338  {
339  if(cs->infoCount_ == 0)
340  {
341  cs->firstInfoMessageTime_ = time(0);
342  cs->firstInfoMessage_ = cs->messages_.back().getMsg();
343  }
344  cs->lastInfoMessageTime_ = time(0);
345  ++cs->infoCount_;
346  cs->lastInfoMessage_ = cs->messages_.back().getMsg();
347  }
348 
349  // check if sequence ID is out of order
350  newSourceId = cs->messages_.back().getSourceIDAsNumber();
351  newSequenceId = cs->messages_.back().getSequenceIDAsNumber();
352 
353  // std::cout << rsock.getLastIncomingIPAddress() << ":" <<
354  // rsock.getLastIncomingPort() << ":newSourceId: " << newSourceId <<
355  // ":" << newSequenceId << __E__;
356 
357  if( // newSequenceId%1000 == 0 || //for debugging missed!
358  (newSourceId != -1 &&
359  sourceLastSequenceID.find(newSourceId) !=
360  sourceLastSequenceID
361  .end() && // ensure not first packet received
362  ((newSequenceId == 0 && sourceLastSequenceID[newSourceId] !=
363  (uint32_t)-1) || // wrap around case
364  newSequenceId !=
365  sourceLastSequenceID[newSourceId] + 1)) // normal sequence case
366  )
367  {
368  // missed some messages!
369  std::stringstream missedSs;
370  missedSs << "Console missed "
371  << (newSequenceId - 1) -
372  (sourceLastSequenceID[newSourceId] + 1) + 1
373  << " packet(s) from " << cs->messages_.back().getSource()
374  << "!" << __E__;
375  __SS__ << missedSs.str();
376  std::cout << ss.str();
377 
378  // generate special message to indicate missed packets
379  cs->messages_.emplace_back(CONSOLE_SPECIAL_WARNING + missedSs.str(),
380  cs->messageCount_++,
381  cs->priorityCustomTriggerList_);
382  //force time to now for auto-generated message
383  cs->messages_.back().setTime(time(0));
384  //Force a custom count because Console Label are ignored! if(cs->messages_.back().hasCustomTriggerMatchAction())
385  if(cs->priorityCustomTriggerList_.size())
386  {
387  cs->customTriggerActionQueue_.push(
388  cs->priorityCustomTriggerList_
389  [0]); //push newest action to back
390  cs->priorityCustomTriggerList_[0]
391  .occurrences++; //increment occurrences
392  cs->customTriggerActionQueue_.back().triggeredMessageCountIndex =
393  cs->messages_.back().getCount();
394  cs->messages_.back().setCustomTriggerMatch(
395  cs->customTriggerActionQueue_.back());
396  }
397  }
398 
399  // save the new last sequence ID
400  sourceLastSequenceID[newSourceId] = newSequenceId;
401 
402  while(cs->messages_.size() > 0 &&
403  cs->messages_.size() > cs->maxMessageCount_)
404  {
405  cs->messages_.erase(cs->messages_.begin());
406  }
407 
408  c += strlen(&(buffer.c_str()[c])) + 1;
409  } // end handle message stacking in packet
410  } // end received packet handling
411  else // idle network handling
412  {
413  if(i < 120) // if nothing received for 120 seconds, then something is wrong
414  // with Console configuration
415  ++i;
416 
417  sleep(1); // sleep one second, if timeout
418 
419  // every 60 heartbeatCount (2 seconds each = 1 sleep and 1 timeout) print a
420  // heartbeat message
421  if(i != 200 || // show first message, if not already a message
422  (heartbeatCount < 60 * 5 &&
423  heartbeatCount % 60 == 59)) // every ~2 min for first 5 messages
424  {
425  ++selfGeneratedMessageCount; // increment internal message count
426  __COUT__ << "Console is alive and waiting... (if no messages, next "
427  "heartbeat is in two minutes)"
428  << __E__;
429  }
430  else if(heartbeatCount % (60 * 30) == 59) // approx every hour
431  {
432  ++selfGeneratedMessageCount; // increment internal message count
433  __COUT__ << "Console is alive and waiting a long time... (if no "
434  "messages, next heartbeat is in one hour)"
435  << __E__;
436  }
437 
438  ++heartbeatCount;
439  } // end idle network handling
440 
441  // if nothing received for 2 minutes seconds, then something is wrong with Console
442  // configuration after 5 seconds there is a self-send. Which will at least
443  // confirm configuration. OR if 5 generated messages and never cleared.. then
444  // the forwarding is not working.
445  if(i == 120 || selfGeneratedMessageCount == 5)
446  {
447  __COUTV__(i);
448  __COUTV__(selfGeneratedMessageCount);
449  __COUT__ << "No messages received at Console Supervisor. Exiting Console "
450  "messageFacilityReceiverWorkLoop"
451  << __E__;
452  break; // assume something wrong, and break loop
453  }
454 
455  if(cs->customTriggerActionQueue_.size() && !cs->customTriggerActionThreadExists_)
456  {
457  cs->customTriggerActionThreadExists_ = true;
458 
459  std::thread(
460  [](ConsoleSupervisor* c) {
461  ConsoleSupervisor::customTriggerActionThread(c);
462  },
463  cs)
464  .detach();
465  }
466 
467  } // end infinite loop
468 
469 } // end messageFacilityReceiverWorkLoop()
470 catch(const std::runtime_error& e)
471 {
472  __COUT_ERR__ << "Error caught at Console Supervisor thread: " << e.what() << __E__;
473 }
474 catch(...)
475 {
476  __COUT_ERR__ << "Unknown error caught at Console Supervisor thread." << __E__;
477 } // end messageFacilityReceiverWorkLoop() exception handling
478 
479 //==============================================================================
482 void ConsoleSupervisor::customTriggerActionThread(ConsoleSupervisor* cs)
483 try
484 {
485  __COUT__ << "Starting customTriggerActionThread" << __E__;
486  CustomTriggeredAction_t triggeredAction;
487  while(1) //infinite workloop
488  {
489  { //mutex scope:
490  // lockout the messages array for the remainder of the scope
491  // this guarantees the reading thread can safely access the action queue
492  std::lock_guard<std::mutex> lock(cs->messageMutex_);
493  if(cs->customTriggerActionQueue_.size())
494  {
495  triggeredAction = cs->customTriggerActionQueue_.front();
496  cs->customTriggerActionQueue_.pop(); //pop first/oldest element
497  }
498  } //end mutex scope
499 
500  if(triggeredAction.action.size())
501  {
502  __COUTS__(2) << "Handling action '" << triggeredAction.action
503  << "' on custom count search string: "
504  << StringMacros::vectorToString(triggeredAction.needleSubstrings,
505  {'*'})
506  << __E__;
507  cs->doTriggeredAction(triggeredAction);
508  }
509 
510  triggeredAction.action = ""; //clear action for next in queue
511  triggeredAction.triggeredMessageCountIndex =
512  -1; //clear triggered message ID for next in queue
513  sleep(2); //mostly sleep
514 
515  } //end infinite workloop
516 
517 } // end customTriggerActionThread()
518 catch(const std::runtime_error& e)
519 {
520  __COUT_ERR__ << "Error caught at Console Supervisor Action thread: " << e.what()
521  << __E__;
522 }
523 catch(...)
524 {
525  __COUT_ERR__ << "Unknown error caught at Console Supervisor Action thread." << __E__;
526 } // end customTriggerActionThread() exception handling
527 
528 //==============================================================================
529 void ConsoleSupervisor::doTriggeredAction(const CustomTriggeredAction_t& triggeredAction)
530 {
531  __SUP_COUT_INFO__ << "Launching Triggered Action '" << triggeredAction.action
532  << "' fired on custom count search string: "
533  << StringMacros::vectorToString(triggeredAction.needleSubstrings,
534  {'*'})
535  << __E__;
536 
537  //valid actions:
538  // Halt
539  // Stop
540  // Pause
541  // Soft Error
542  // Hard Error
543  // System Message
544  // Run Script
545 
546  if(CUSTOM_TRIGGER_ACTIONS.find(triggeredAction.action) ==
547  CUSTOM_TRIGGER_ACTIONS.end())
548  {
549  __SUP_SS__ << "Unrecognized triggered action '" << triggeredAction.action
550  << ",' valid actions are "
551  << StringMacros::setToString(CUSTOM_TRIGGER_ACTIONS) << __E__;
552  __SUP_SS_THROW__;
553  }
554 
555  //all FSM commands include a system message
556  if(triggeredAction.action != "Count Only")
557  theRemoteWebUsers_.sendSystemMessage(
558  "*" /* to all users*/,
559  "In the Console Supervisor, a custom count fired the action '" +
560  triggeredAction.action + "' on the search string '" +
561  StringMacros::vectorToString(triggeredAction.needleSubstrings, {'*'}) +
562  "'");
563 
564  if(triggeredAction.action == "Halt")
565  {
566  try
567  {
568  ConsoleSupervisor::sendAsyncExceptionToGateway(
569  "Console-triggered FSM Halt", 0, 0);
570  }
571  catch(...)
572  {
573  theRemoteWebUsers_.sendSystemMessage(
574  "*" /* to all users*/,
575  "FSM Halt from Console Supervisor Triggered Action has failed!");
576  }
577  __SUP_COUTV__("FSM Halt triggered from console");
578  }
579  else if(triggeredAction.action == "Pause")
580  {
581  try
582  {
583  ConsoleSupervisor::sendAsyncExceptionToGateway(
584  "Console-triggered FSM Pause", 1, 0);
585  }
586  catch(...)
587  {
588  theRemoteWebUsers_.sendSystemMessage(
589  "*" /* to all users*/,
590  "FSM Pause from Console Supervisor Triggered Action has failed!");
591  }
592  __SUP_COUTV__("FSM Pause triggered from console");
593  }
594  else if(triggeredAction.action == "Stop")
595  {
596  try
597  {
598  ConsoleSupervisor::sendAsyncExceptionToGateway(
599  "Console-triggered FSM Stop", 0, 1);
600  }
601  catch(const std::exception& e)
602  {
603  theRemoteWebUsers_.sendSystemMessage(
604  "*" /* to all users*/,
605  "FSM Stop from Console Supervisor Triggered Action has failed!");
606  }
607  __SUP_COUTV__("FSM Stop triggered from console");
608  }
609  else if(triggeredAction.action == "Run Script")
610  {
611  std::string triggerScriptPath = "";
612  std::string scriptResult = "";
613  try
614  {
615  triggerScriptPath = __ENV__("OTS_CUSTOM_TRIGGER_SCRIPT");
616  triggerScriptPath = "source " + triggerScriptPath;
617  scriptResult = StringMacros::exec(triggerScriptPath.c_str());
618  __COUT_INFO__ << "The Script " << triggerScriptPath
619  << " Was launched, here is the result " << scriptResult;
620  }
621  catch(...)
622  {
623  __SS__ << "Trigger script path not defined! Please use environment variable "
624  "'OTS_CUSTOM_TRIGGER_SCRIPT' or contact admins."
625  << __E__;
626  __SS_THROW__;
627  }
628  }
629 
630 } // end doTriggeredAction()
631 
632 //==============================================================================
634 void ConsoleSupervisor::addCustomTriggeredAction(const std::string& triggerNeedle,
635  const std::string& triggerAction,
636  uint32_t priority, /* = -1 */
637  uint32_t triggerOnCount,
638  bool doLoop,
639  bool isArmed)
640 {
641  __SUP_COUTV__("Adding custom triggered action");
642  __SUP_COUTV__(triggerNeedle);
643  __SUP_COUTV__(triggerAction);
644  __SUP_COUTV__(priority);
645  __SUP_COUTV__(triggerOnCount);
646  __SUP_COUTV__(doLoop);
647  __SUP_COUTV__(isArmed);
648 
649  bool allAsterisks = true;
650  for(const auto& c : triggerNeedle)
651  if(c != '*')
652  {
653  allAsterisks = false;
654  break;
655  }
656  if(allAsterisks)
657  {
658  __SUP_SS__ << "Illegal empty Search String value for the new Custom Count and "
659  "Action! Please enter a valid Search String (* wildcards are "
660  "allowed, e.g. \"value = * seconds\")."
661  << __E__;
662  __SUP_SS_THROW__;
663  }
664 
665  //check if triggerNeedle already exists
666  uint32_t currentPriority = -1;
667  for(const auto& customTrigger : priorityCustomTriggerList_)
668  {
669  ++currentPriority; //inc first to get to 0
670  if(StringMacros::vectorToString(customTrigger.needleSubstrings, {'*'}) ==
671  triggerNeedle)
672  {
673  __SUP_SS__ << "Failure! Can not add Custom Count Search String that already "
674  "exists. Found '"
675  << triggerNeedle
676  << "' already existing at priority = " << currentPriority << __E__;
677  __SUP_SS_THROW__;
678  }
679  } //end check if already exists
680 
681  if(priority >= priorityCustomTriggerList_.size())
682  priority = priorityCustomTriggerList_.size(); //place at end
683  if(priority == 0 && triggerNeedle != CONSOLE_MISSED_NEEDLE)
684  {
685  __SUP_SS__ << "Illegal priority position of '" << priority
686  << "' requested. Please enter a priority value greater than 0. "
687  "Position 0 is reserved for identifying missing messages at the "
688  "Console Supervisor. Note: the action for missing messages, at "
689  "priority 0, may be customized by the user."
690  << __E__;
691  __SUP_SS_THROW__;
692  }
693  __SUP_COUTV__(priority);
694 
695  //valid actions:
696  // Halt
697  // Stop
698  // Pause
699  // Soft Error
700  // Hard Error
701  // System Message
702  // Run Script
703 
704  if(CUSTOM_TRIGGER_ACTIONS.find(triggerAction) == CUSTOM_TRIGGER_ACTIONS.end())
705  {
706  __SUP_SS__ << "Unrecognized triggered action '" << triggerAction
707  << ",' valid actions are "
708  << StringMacros::setToString(CUSTOM_TRIGGER_ACTIONS) << __E__;
709  __SUP_SS_THROW__;
710  }
711 
712  //insert new custom count at priority position
713  priorityCustomTriggerList_.insert(priorityCustomTriggerList_.begin() + priority,
714  CustomTriggeredAction_t());
715  // priorityCustomTriggerList_.push_back(CustomTriggeredAction_t());
716  // priorityCustomTriggerList_.back()
717 
718  //break up on substring
719  StringMacros::getVectorFromString(
720  triggerNeedle,
721  priorityCustomTriggerList_[priority].needleSubstrings,
722  {'*'} /* delimiter */,
723  {} /* do not ignore whitespace */);
724  priorityCustomTriggerList_[priority].action = triggerAction;
725  priorityCustomTriggerList_[priority].triggerOnCount = triggerOnCount;
726  priorityCustomTriggerList_[priority].doLoop = doLoop;
727  priorityCustomTriggerList_[priority].isArmed = isArmed;
728 
729  __SUP_COUT__ << "Added custom count: '"
730  << StringMacros::vectorToString(
731  priorityCustomTriggerList_[priority].needleSubstrings, "*")
732  << "' at priority: " << priority
733  << " triggered every: " << triggerOnCount << " occurrences";
734  if(doLoop)
735  __SUP_COUT__ << " and will loop.";
736  else
737  __SUP_COUT__ << " and will not loop.";
738  __SUP_COUT__ << __E__;
739 
740 } // end addCustomTriggeredAction()
741 
742 //==============================================================================
746 uint32_t ConsoleSupervisor::modifyCustomTriggeredAction(const std::string& currentNeedle,
747  const std::string& modifyType,
748  const std::string& setNeedle,
749  const std::string& setAction,
750  uint32_t setPriority,
751  uint32_t setTriggerOnCount,
752  bool setDoLoop,
753  bool setIsArmed)
754 {
755  __SUP_COUTV__(currentNeedle);
756  __SUP_COUTV__(modifyType);
757  __SUP_COUTV__(setNeedle);
758  __SUP_COUTV__(setAction);
759  __SUP_COUTV__(setPriority);
760  __SUP_COUTV__(setTriggerOnCount);
761  __SUP_COUTV__(setDoLoop);
762  __SUP_COUTV__(setIsArmed);
763 
764  //find current priority position of currentNeedle
765  uint32_t currentPriority = -1;
766  bool found = false;
767  for(const auto& customTrigger : priorityCustomTriggerList_)
768  {
769  ++currentPriority; //inc first to get to 0, -1 indicates not found
770  if(StringMacros::vectorToString(customTrigger.needleSubstrings, {'*'}) ==
771  currentNeedle)
772  {
773  found = true;
774  break; //found
775  }
776  }
777 
778  __SUP_COUTV__(currentPriority);
779  if(!found)
780  {
781  __SUP_SS__ << "Attempt to modify Custom Count Search String failed. Could not "
782  "find specified Search String '"
783  << currentNeedle << "' in prioritized list." << __E__;
784  __SUP_SS_THROW__;
785  }
786 
787  if(modifyType == "Deletion")
788  {
789  if(currentPriority == 0)
790  {
791  __SUP_SS__ << "Illegal deletion requested of priority position 0. Position 0 "
792  "is reserved for identifying missing messages at the Console "
793  "Supervisor. Note: the action of priority 0 may be customized "
794  "by the user, but it can not be deleted."
795  << __E__;
796  __SUP_SS_THROW__;
797  }
798 
799  __SUP_COUT__ << "Deleting custom count: "
800  << StringMacros::vectorToString(
801  priorityCustomTriggerList_[currentPriority].needleSubstrings,
802  {'*'})
803  << " w/action: "
804  << priorityCustomTriggerList_[currentPriority].action
805  << " and priority: " << currentPriority << __E__;
806  priorityCustomTriggerList_.erase(priorityCustomTriggerList_.begin() +
807  currentPriority);
808  return -1;
809  }
810 
811  if(modifyType == "Priority" || modifyType == "All")
812  {
813  if(setPriority >= priorityCustomTriggerList_.size())
814  setPriority = priorityCustomTriggerList_.size(); //place at end
815  if(setPriority == 0 && setNeedle != CONSOLE_MISSED_NEEDLE)
816  {
817  __SUP_SS__ << "Illegal priority position of '" << setPriority
818  << "' requested. Position 0 is reserved for identifying missing "
819  "messages at the Console Supervisor. Note: the action of "
820  "priority 0 may be customized by the user."
821  << __E__;
822  __SUP_SS_THROW__;
823  }
824  }
825  else //keep existing
826  setPriority = currentPriority;
827 
828  if(modifyType == "Action" || modifyType == "All")
829  {
830  //valid actions:
831  // Halt
832  // Stop
833  // Pause
834  // Soft Error
835  // Hard Error
836  // System Message
837 
838  if(CUSTOM_TRIGGER_ACTIONS.find(setAction) == CUSTOM_TRIGGER_ACTIONS.end())
839  {
840  __SUP_SS__ << "Unrecognized custom count action '" << setAction
841  << ",' valid actions are "
842  << StringMacros::setToString(CUSTOM_TRIGGER_ACTIONS) << __E__;
843  __SUP_SS_THROW__;
844  }
845  //modify existing action
846  priorityCustomTriggerList_[currentPriority].action = setAction;
847  }
848 
849  if(modifyType == "Search String" || modifyType == "All")
850  {
851  //modify existing needle
852  priorityCustomTriggerList_[currentPriority].needleSubstrings.clear();
853  StringMacros::getVectorFromString(
854  setNeedle,
855  priorityCustomTriggerList_[currentPriority].needleSubstrings,
856  {'*'} /* delimiter */,
857  {} /* do not ignore whitespace */);
858  }
859 
860  if(modifyType == "Trigger on Count" || modifyType == "All")
861  {
862  //modify existing action
863  priorityCustomTriggerList_[currentPriority].triggerOnCount = setTriggerOnCount;
864  }
865  if(modifyType == "Do Loop" || modifyType == "All")
866  {
867  //modify existing action
868  priorityCustomTriggerList_[currentPriority].doLoop = setDoLoop;
869  }
870  if(modifyType == "Arm Trigger" || modifyType == "All")
871  {
872  //modify existing action
873  priorityCustomTriggerList_[currentPriority].isArmed = setIsArmed;
874  }
875 
876  if(currentPriority != setPriority) //then need to copy
877  {
878  //insert new custom count at priority position
879  priorityCustomTriggerList_.insert(
880  priorityCustomTriggerList_.begin() + setPriority,
881  priorityCustomTriggerList_[currentPriority]);
882 
883  //delete from old position
884  if(currentPriority >= setPriority) //then increment after insert
885  ++currentPriority;
886 
887  priorityCustomTriggerList_.erase(priorityCustomTriggerList_.begin() +
888  currentPriority);
889 
890  if(currentPriority < setPriority) //then decrement after delete
891  --setPriority;
892  }
893 
894  __SUP_COUT__ << "Modified '" << modifyType << "' custom count: "
895  << StringMacros::vectorToString(
896  priorityCustomTriggerList_[setPriority].needleSubstrings, {'*'})
897  << " now w/action: " << priorityCustomTriggerList_[setPriority].action
898  << " and at priority: " << setPriority << __E__;
899 
900  return setPriority;
901 } // end modifyCustomTriggeredAction()
902 
903 //==============================================================================
904 void ConsoleSupervisor::loadCustomCountList()
905 {
906  // TODO: Migrate to a more robust read/write method
907  __SUP_COUT__ << "loadCustomCountList() from "
908  << USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME << __E__;
909 
910  FILE* fp = fopen((USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME).c_str(), "r");
911  if(!fp)
912  {
913  __SUP_COUT__ << "Ignoring missing Custom Count list file at path: "
914  << (USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME) << __E__;
915  return;
916  }
917  priorityCustomTriggerList_.clear();
918 
919  char line[1000]; //do not allow larger than 1000 chars!
920  uint32_t i = 0;
921  std::string needle;
922  std::string action;
923  size_t priority = 0;
924  size_t triggerOnCount = 0;
925  bool doLoop = false;
926  bool isArmed = false;
927  while(fgets(line, 1000, fp))
928  {
929  ++i;
930 
931  //ignore new line
932  if(strlen(line))
933  line[strlen(line) - 1] = '\0';
934 
935  __SUP_COUTV__(i);
936  __SUP_COUTV__(line);
937 
938  if(i == 1) //needle
939  needle = line;
940  else if(i == 2)
941  action = line;
942  else if(i == 3)
943  priority = std::stoi(line);
944  else if(i == 4)
945  triggerOnCount = std::stoi(line);
946  else if(i == 5)
947  doLoop = std::stoi(line) > 0 ? true : false;
948  else if(i == 6) // last line
949  {
950  isArmed = std::stoi(line) > 0 ? true : false;
951  __SUP_COUTTV__(needle);
952  __SUP_COUTTV__(priority);
953  __SUP_COUTTV__(action);
954  __SUP_COUTTV__(triggerOnCount);
955  __SUP_COUTTV__(doLoop);
956  __SUP_COUTTV__(isArmed);
957  if(i == 1 &&
958  needle !=
959  CONSOLE_MISSED_NEEDLE) //then force missed Console message as priority 0
960  addCustomTriggeredAction(
961  CONSOLE_MISSED_NEEDLE, "System Message", 0, 1, false, true);
962  addCustomTriggeredAction(
963  needle, action, priority, triggerOnCount, doLoop, isArmed);
964  i = 0; //reset for next entry
965  }
966  }
967  fclose(fp);
968 
969 } // end loadCustomCountList()
970 
971 //==============================================================================
972 void ConsoleSupervisor::saveCustomCountList()
973 {
974  __SUP_COUT__ << "saveCustomCountList()" << __E__;
975 
976  FILE* fp = fopen((USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME).c_str(), "w");
977  if(!fp)
978  {
979  __SUP_SS__ << "Failed to create Custom Count list file at path: "
980  << (USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME) << __E__;
981  __SUP_SS_THROW__;
982  }
983  unsigned int priority = 0;
984  for(auto& customCount : priorityCustomTriggerList_)
985  {
986  fprintf(fp,
987  (StringMacros::vectorToString(customCount.needleSubstrings, {'*'}) + "\n")
988  .c_str());
989  fprintf(fp, "%s\n", customCount.action.c_str());
990  fprintf(fp, "%d\n", priority);
991  fprintf(fp, "%zu\n", customCount.triggerOnCount);
992  fprintf(fp, "%s\n", customCount.doLoop ? "1" : "0");
993  fprintf(fp, "%s\n", customCount.isArmed ? "1" : "0");
994  ++priority;
995  }
996  fclose(fp);
997 } // end saveCustomCountList()
998 
999 //==============================================================================
1000 void ConsoleSupervisor::defaultPage(xgi::Input* /*in*/, xgi::Output* out)
1001 {
1002  // __SUP_COUT__ << "ApplicationDescriptor LID="
1003  // << getApplicationDescriptor()->getLocalId() << __E__;
1004  *out << "<!DOCTYPE HTML><html lang='en'><frameset col='100%' row='100%'><frame "
1005  "src='/WebPath/html/Console.html?urn="
1006  << getApplicationDescriptor()->getLocalId() << "'></frameset></html>";
1007 } // end defaultPage()
1008 
1009 //==============================================================================
1012 void ConsoleSupervisor::forceSupervisorPropertyValues()
1013 {
1014  CorePropertySupervisorBase::setSupervisorProperty(
1015  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AutomatedRequestTypes,
1016  "GetConsoleMsgs");
1017 } // end forceSupervisorPropertyValues()
1018 
1019 //==============================================================================
1023 void ConsoleSupervisor::request(const std::string& requestType,
1024  cgicc::Cgicc& cgiIn,
1025  HttpXmlDocument& xmlOut,
1026  const WebUsers::RequestUserInfo& userInfo)
1027 {
1028  //__SUP_COUT__ << "requestType " << requestType << __E__;
1029 
1030  // Commands:
1031  // GetConsoleMsgs
1032  // PrependHistoricMessages
1033  // SaveUserPreferences
1034  // LoadUserPreferences
1035  // GetTraceLevels
1036  // SetTraceLevels
1037  // GetTriggerStatus
1038  // SetTriggerEnable
1039  // ResetTRACE
1040  // EnableTRACE
1041  // GetTraceSnapshot
1042  // GetCustomCountsAndActions
1043  // AddCustomCountsAndAction
1044  // ModifyCustomCountsAndAction
1045 
1046  // Note: to report to logbook admin status use
1047  // xmlOut.addTextElementToData(XML_ADMIN_STATUS,refreshTempStr_);
1048 
1049  if(requestType == "GetConsoleMsgs")
1050  {
1051  // lindex of -1 means first time and user just gets update lcount and lindex
1052  std::string lastUpdateCountStr = CgiDataUtilities::postData(cgiIn, "lcount");
1053 
1054  if(lastUpdateCountStr == "")
1055  {
1056  __SUP_COUT_ERR__ << "Invalid Parameters! lastUpdateCount="
1057  << lastUpdateCountStr << __E__;
1058  xmlOut.addTextElementToData("Error",
1059  "Error - Invalid parameters for GetConsoleMsgs.");
1060  return;
1061  }
1062 
1063  size_t lastUpdateCount = std::stoull(lastUpdateCountStr);
1064 
1065  // __SUP_COUT__ << "lastUpdateCount=" << lastUpdateCount << __E__;
1066 
1067  insertMessageRefresh(&xmlOut, lastUpdateCount);
1068  }
1069  else if(requestType == "PrependHistoricMessages")
1070  {
1071  size_t earliestOnhandMessageCount =
1072  CgiDataUtilities::postDataAsInt(cgiIn, "earlyCount");
1073  __SUP_COUTV__(earliestOnhandMessageCount);
1074  prependHistoricMessages(&xmlOut, earliestOnhandMessageCount);
1075  }
1076  else if(requestType == "SaveUserPreferences")
1077  {
1078  int colorIndex = CgiDataUtilities::postDataAsInt(cgiIn, "colorIndex");
1079  int showSideBar = CgiDataUtilities::postDataAsInt(cgiIn, "showSideBar");
1080  int noWrap = CgiDataUtilities::postDataAsInt(cgiIn, "noWrap");
1081  int messageOnly = CgiDataUtilities::postDataAsInt(cgiIn, "messageOnly");
1082  int hideLineNumers = CgiDataUtilities::postDataAsInt(cgiIn, "hideLineNumers");
1083 
1084  // __SUP_COUT__ << "requestType " << requestType << __E__;
1085  // __SUP_COUT__ << "colorIndex: " << colorIndex << __E__;
1086  // __SUP_COUT__ << "showSideBar: " << showSideBar << __E__;
1087  // __SUP_COUT__ << "noWrap: " << noWrap << __E__;
1088  // __SUP_COUT__ << "messageOnly: " << messageOnly << __E__;
1089  // __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << __E__;
1090 
1091  if(userInfo.username_ == "") // should never happen?
1092  {
1093  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_
1094  << __E__;
1095  xmlOut.addTextElementToData("Error",
1096  "Error - InvauserInfo.username_user found.");
1097  return;
1098  }
1099 
1100  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." +
1101  (std::string)USERS_PREFERENCES_FILETYPE;
1102 
1103  // __SUP_COUT__ << "Save preferences: " << fn << __E__;
1104  FILE* fp = fopen(fn.c_str(), "w");
1105  if(!fp)
1106  {
1107  __SS__;
1108  __THROW__(ss.str() + "Could not open file: " + fn);
1109  }
1110  fprintf(fp, "colorIndex %d\n", colorIndex);
1111  fprintf(fp, "showSideBar %d\n", showSideBar);
1112  fprintf(fp, "noWrap %d\n", noWrap);
1113  fprintf(fp, "messageOnly %d\n", messageOnly);
1114  fprintf(fp, "hideLineNumers %d\n", hideLineNumers);
1115  fclose(fp);
1116  }
1117  else if(requestType == "LoadUserPreferences")
1118  {
1119  // __SUP_COUT__ << "requestType " << requestType << __E__;
1120 
1121  unsigned int colorIndex, showSideBar, noWrap, messageOnly, hideLineNumers;
1122 
1123  if(userInfo.username_ == "") // should never happen?
1124  {
1125  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_
1126  << __E__;
1127  xmlOut.addTextElementToData("Error", "Error - Invalid user found.");
1128  return;
1129  }
1130 
1131  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." +
1132  (std::string)USERS_PREFERENCES_FILETYPE;
1133 
1134  // __SUP_COUT__ << "Load preferences: " << fn << __E__;
1135 
1136  FILE* fp = fopen(fn.c_str(), "r");
1137  if(!fp)
1138  {
1139  // return defaults
1140  __SUP_COUT__ << "Returning defaults." << __E__;
1141  xmlOut.addTextElementToData("colorIndex", "0");
1142  xmlOut.addTextElementToData("showSideBar", "0");
1143  xmlOut.addTextElementToData("noWrap", "1");
1144  xmlOut.addTextElementToData("messageOnly", "0");
1145  xmlOut.addTextElementToData("hideLineNumers", "1");
1146  return;
1147  }
1148  fscanf(fp, "%*s %u", &colorIndex);
1149  fscanf(fp, "%*s %u", &showSideBar);
1150  fscanf(fp, "%*s %u", &noWrap);
1151  fscanf(fp, "%*s %u", &messageOnly);
1152  fscanf(fp, "%*s %u", &hideLineNumers);
1153  fclose(fp);
1154  // __SUP_COUT__ << "colorIndex: " << colorIndex << __E__;
1155  // __SUP_COUT__ << "showSideBar: " << showSideBar << __E__;
1156  // __SUP_COUT__ << "noWrap: " << noWrap << __E__;
1157  // __SUP_COUT__ << "messageOnly: " << messageOnly << __E__;
1158  // __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << __E__;
1159 
1160  char tmpStr[20];
1161  sprintf(tmpStr, "%u", colorIndex);
1162  xmlOut.addTextElementToData("colorIndex", tmpStr);
1163  sprintf(tmpStr, "%u", showSideBar);
1164  xmlOut.addTextElementToData("showSideBar", tmpStr);
1165  sprintf(tmpStr, "%u", noWrap);
1166  xmlOut.addTextElementToData("noWrap", tmpStr);
1167  sprintf(tmpStr, "%u", messageOnly);
1168  xmlOut.addTextElementToData("messageOnly", tmpStr);
1169  sprintf(tmpStr, "%u", hideLineNumers);
1170  xmlOut.addTextElementToData("hideLineNumers", tmpStr);
1171  }
1172  else if(requestType == "GetTraceLevels")
1173  {
1174  __SUP_COUT__ << "requestType " << requestType << __E__;
1175 
1176  SOAPParameters txParameters; // params for xoap to send
1177  txParameters.addParameter("Request", "GetTraceLevels");
1178 
1179  SOAPParameters rxParameters; // params for xoap to recv
1180  rxParameters.addParameter("Command");
1181  rxParameters.addParameter("Error");
1182  rxParameters.addParameter("TRACEHostnameList");
1183  rxParameters.addParameter("TRACEList");
1184 
1185  traceMapToXDAQHostname_.clear(); // reset
1186 
1187  std::string traceList = "";
1188  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1189  for(const auto& appInfo : allTraceApps)
1190  {
1191  __SUP_COUT__ << "Supervisor hostname = " << appInfo.first << "/"
1192  << appInfo.second.getId()
1193  << " name = " << appInfo.second.getName()
1194  << " class = " << appInfo.second.getClass()
1195  << " hostname = " << appInfo.second.getHostname() << __E__;
1196  try
1197  {
1198  xoap::MessageReference retMsg =
1199  SOAPMessenger::sendWithSOAPReply(appInfo.second.getDescriptor(),
1200  "TRACESupervisorRequest",
1201  txParameters);
1202  SOAPUtilities::receive(retMsg, rxParameters);
1203  __SUP_COUT__ << "Received TRACE response: "
1204  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1205  << SOAPUtilities::translate(retMsg) << __E__;
1206 
1207  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1208  {
1209  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1210  "hostname = "
1211  << appInfo.first << "/" << appInfo.second.getId()
1212  << " name = " << appInfo.second.getName()
1213  << " class = " << appInfo.second.getClass()
1214  << " hostname = " << appInfo.second.getHostname() << __E__;
1215  __SUP_SS_THROW__;
1216  }
1217  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1218  {
1219  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1220  << __E__;
1221  __SUP_SS_THROW__;
1222  }
1223  }
1224  catch(const xdaq::exception::Exception& e)
1225  {
1226  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1227  << appInfo.second.getId()
1228  << " name = " << appInfo.second.getName() << ". \n\n"
1229  << e.what() << __E__;
1230  //do not throw exception, because unable to set levels when some Supervisors are down
1231  //__SUP_SS_THROW__;
1232  __SUP_COUT_ERR__ << ss.str();
1233  continue; //skip bad Supervisor
1234  }
1235 
1236  std::vector<std::string> traceHostnameArr;
1237  __COUTTV__(rxParameters.getValue("TRACEHostnameList"));
1238  StringMacros::getVectorFromString(
1239  rxParameters.getValue("TRACEHostnameList"), traceHostnameArr, {';'});
1240  for(const auto& traceHostname : traceHostnameArr)
1241  {
1242  if(traceHostname == "")
1243  continue; //skip blanks
1244  traceMapToXDAQHostname_[traceHostname] = appInfo.first;
1245  }
1246 
1247  // traceList += ";" + appInfo.first; //insert xdaq context version of
1248  // name
1249  // //FIXME and create mapp from user's typed in xdaq
1250  // context name to TRACE hostname resolution
1251 
1252  __COUTTV__(rxParameters.getValue("TRACEList"));
1253  traceList += rxParameters.getValue("TRACEList");
1254 
1255  } // end app get TRACE loop
1256  __SUP_COUT__ << "TRACE hostname map received: \n"
1257  << StringMacros::mapToString(traceMapToXDAQHostname_) << __E__;
1258  __SUP_COUT__ << "TRACE List received: \n" << traceList << __E__;
1259  xmlOut.addTextElementToData("traceList", traceList);
1260  } // end GetTraceLevels
1261  else if(requestType == "SetTraceLevels")
1262  {
1263  __SUP_COUT__ << "requestType " << requestType << __E__;
1264 
1265  std::string individualValues =
1266  CgiDataUtilities::postData(cgiIn, "individualValues");
1267  std::string hostLabelMap = CgiDataUtilities::postData(cgiIn, "hostLabelMap");
1268  std::string setMode = CgiDataUtilities::postData(cgiIn, "setMode");
1269  std::string setValueMSB = CgiDataUtilities::postData(cgiIn, "setValueMSB");
1270  std::string setValueLSB = CgiDataUtilities::postData(cgiIn, "setValueLSB");
1271 
1272  __SUP_COUTV__(individualValues);
1273  __SUP_COUTV__(setMode);
1274  // set modes: SLOW, FAST, TRIGGER
1275  __SUP_COUTV__(setValueMSB);
1276  __SUP_COUTV__(setValueLSB);
1277 
1278  std::map<std::string /*host*/, std::string /*labelArr*/> hostToLabelMap;
1279 
1280  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1281 
1282  SOAPParameters rxParameters; // params for xoap to recv
1283  rxParameters.addParameter("Command");
1284  rxParameters.addParameter("Error");
1285  rxParameters.addParameter("TRACEList");
1286 
1287  std::string modifiedTraceList = "";
1288  std::string xdaqHostname;
1289  StringMacros::getMapFromString(hostLabelMap, hostToLabelMap, {';'}, {':'});
1290  for(auto& hostLabelsPair : hostToLabelMap)
1291  {
1292  // identify artdaq hosts to go through ARTDAQ supervisor
1293  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1294  __SUP_COUTV__(hostLabelsPair.first);
1295  __SUP_COUTV__(hostLabelsPair.second);
1296 
1297  // use map to convert to xdaq host
1298  try
1299  {
1300  xdaqHostname = traceMapToXDAQHostname_.at(hostLabelsPair.first);
1301  }
1302  catch(...)
1303  {
1304  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1305  << hostLabelsPair.first << "' to xdaq Context hostname."
1306  << __E__;
1307  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1308  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1309  << __E__;
1310  __SUP_SS_THROW__;
1311  }
1312 
1313  __SUP_COUTV__(xdaqHostname);
1314 
1315  auto& appInfo = allTraceApps.at(xdaqHostname);
1316  __SUP_COUT__ << "Supervisor hostname = " << hostLabelsPair.first << "/"
1317  << xdaqHostname << ":" << appInfo.getId()
1318  << " name = " << appInfo.getName()
1319  << " class = " << appInfo.getClass()
1320  << " hostname = " << appInfo.getHostname() << __E__;
1321  try
1322  {
1323  SOAPParameters txParameters; // params for xoap to send
1324  txParameters.addParameter("Request", "SetTraceLevels");
1325  txParameters.addParameter("IndividualValues", individualValues);
1326  txParameters.addParameter("Host", hostLabelsPair.first);
1327  txParameters.addParameter("SetMode", setMode);
1328  txParameters.addParameter("Labels", hostLabelsPair.second);
1329  txParameters.addParameter("SetValueMSB", setValueMSB);
1330  txParameters.addParameter("SetValueLSB", setValueLSB);
1331 
1332  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1333  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1334  SOAPUtilities::receive(retMsg, rxParameters);
1335  __SUP_COUT__ << "Received TRACE response: "
1336  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1337  << SOAPUtilities::translate(retMsg) << __E__;
1338 
1339  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1340  {
1341  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1342  "hostname = "
1343  << hostLabelsPair.first << "/" << appInfo.getId()
1344  << " name = " << appInfo.getName()
1345  << " class = " << appInfo.getClass()
1346  << " hostname = " << appInfo.getHostname() << __E__;
1347  __SUP_SS_THROW__;
1348  }
1349  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1350  {
1351  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1352  << __E__;
1353  __SUP_SS_THROW__;
1354  }
1355  }
1356  catch(const xdaq::exception::Exception& e)
1357  {
1358  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1359  << appInfo.getId() << " name = " << appInfo.getName()
1360  << ". \n\n"
1361  << e.what() << __E__;
1362  __SUP_SS_THROW__;
1363  }
1364 
1365  modifiedTraceList +=
1366  ";" + hostLabelsPair.first; // insert xdaq context version of name
1367  // FIXME and create mapp from user's typed in xdaq
1368  // context name to TRACE hostname resolution
1369 
1370  modifiedTraceList += rxParameters.getValue("TRACEList");
1371 
1372  } // end host set TRACE loop
1373 
1374  __SUP_COUT__ << "mod'd TRACE List received: \n" << modifiedTraceList << __E__;
1375  xmlOut.addTextElementToData("modTraceList", modifiedTraceList);
1376  } // end SetTraceLevels
1377  else if(requestType == "GetTriggerStatus")
1378  {
1379  __SUP_COUT__ << "requestType " << requestType << __E__;
1380  SOAPParameters txParameters; // params for xoap to send
1381  txParameters.addParameter("Request", "GetTriggerStatus");
1382 
1383  SOAPParameters rxParameters; // params for xoap to recv
1384  rxParameters.addParameter("Command");
1385  rxParameters.addParameter("Error");
1386  rxParameters.addParameter("TRACETriggerStatus");
1387 
1388  std::string traceTriggerStatus = "";
1389  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1390  for(const auto& appInfo : allTraceApps)
1391  {
1392  __SUP_COUT__ << "Supervisor hostname = " << appInfo.first << "/"
1393  << appInfo.second.getId()
1394  << " name = " << appInfo.second.getName()
1395  << " class = " << appInfo.second.getClass()
1396  << " hostname = " << appInfo.second.getHostname() << __E__;
1397  try
1398  {
1399  xoap::MessageReference retMsg =
1400  SOAPMessenger::sendWithSOAPReply(appInfo.second.getDescriptor(),
1401  "TRACESupervisorRequest",
1402  txParameters);
1403  SOAPUtilities::receive(retMsg, rxParameters);
1404  __SUP_COUT__ << "Received TRACE response: "
1405  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1406  << SOAPUtilities::translate(retMsg) << __E__;
1407 
1408  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1409  {
1410  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1411  "hostname = "
1412  << appInfo.first << "/" << appInfo.second.getId()
1413  << " name = " << appInfo.second.getName()
1414  << " class = " << appInfo.second.getClass()
1415  << " hostname = " << appInfo.second.getHostname() << __E__;
1416  __SUP_SS_THROW__;
1417  }
1418  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1419  {
1420  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1421  << __E__;
1422  __SUP_SS_THROW__;
1423  }
1424  }
1425  catch(const xdaq::exception::Exception& e)
1426  {
1427  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1428  << appInfo.second.getId()
1429  << " name = " << appInfo.second.getName() << ". \n\n"
1430  << e.what() << __E__;
1431  __SUP_SS_THROW__;
1432  }
1433 
1434  traceTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1435 
1436  } // end app get TRACE loop
1437  __SUP_COUT__ << "TRACE Trigger Status received: \n"
1438  << traceTriggerStatus << __E__;
1439  xmlOut.addTextElementToData("traceTriggerStatus", traceTriggerStatus);
1440  } // end GetTriggerStatus
1441  else if(requestType == "SetTriggerEnable")
1442  {
1443  __SUP_COUT__ << "requestType " << requestType << __E__;
1444 
1445  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1446 
1447  __SUP_COUTV__(hostList);
1448 
1449  std::vector<std::string /*host*/> hosts;
1450 
1451  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1452 
1453  SOAPParameters rxParameters; // params for xoap to recv
1454  rxParameters.addParameter("Command");
1455  rxParameters.addParameter("Error");
1456  rxParameters.addParameter("TRACETriggerStatus");
1457 
1458  std::string modifiedTriggerStatus = "";
1459  std::string xdaqHostname;
1460  StringMacros::getVectorFromString(hostList, hosts, {';'});
1461  for(auto& host : hosts)
1462  {
1463  // identify artdaq hosts to go through ARTDAQ supervisor
1464  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1465  __SUP_COUTV__(host);
1466  if(host.size() < 3)
1467  continue; // skip bad hostnames
1468 
1469  // use map to convert to xdaq host
1470  try
1471  {
1472  xdaqHostname = traceMapToXDAQHostname_.at(host);
1473  }
1474  catch(...)
1475  {
1476  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1477  << host << "' to xdaq Context hostname." << __E__;
1478  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1479  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1480  << __E__;
1481  __SUP_SS_THROW__;
1482  }
1483 
1484  __SUP_COUTV__(xdaqHostname);
1485 
1486  auto& appInfo = allTraceApps.at(xdaqHostname);
1487  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1488  << appInfo.getId() << " name = " << appInfo.getName()
1489  << " class = " << appInfo.getClass()
1490  << " hostname = " << appInfo.getHostname() << __E__;
1491  try
1492  {
1493  SOAPParameters txParameters; // params for xoap to send
1494  txParameters.addParameter("Request", "SetTriggerEnable");
1495  txParameters.addParameter("Host", host);
1496 
1497  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1498  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1499  SOAPUtilities::receive(retMsg, rxParameters);
1500  __SUP_COUT__ << "Received TRACE response: "
1501  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1502  << SOAPUtilities::translate(retMsg) << __E__;
1503 
1504  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1505  {
1506  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1507  "hostname = "
1508  << host << "/" << appInfo.getId()
1509  << " name = " << appInfo.getName()
1510  << " class = " << appInfo.getClass()
1511  << " hostname = " << appInfo.getHostname() << __E__;
1512  __SUP_SS_THROW__;
1513  }
1514  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1515  {
1516  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1517  << __E__;
1518  __SUP_SS_THROW__;
1519  }
1520  }
1521  catch(const xdaq::exception::Exception& e)
1522  {
1523  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1524  << appInfo.getId() << " name = " << appInfo.getName()
1525  << ". \n\n"
1526  << e.what() << __E__;
1527  __SUP_SS_THROW__;
1528  }
1529 
1530  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1531  } // end host set TRACE loop
1532 
1533  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1534  << modifiedTriggerStatus << __E__;
1535  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1536  } // end SetTriggerEnable
1537  else if(requestType == "ResetTRACE")
1538  {
1539  __SUP_COUT__ << "requestType " << requestType << __E__;
1540 
1541  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1542 
1543  __SUP_COUTV__(hostList);
1544 
1545  std::vector<std::string /*host*/> hosts;
1546 
1547  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1548 
1549  SOAPParameters rxParameters; // params for xoap to recv
1550  rxParameters.addParameter("Command");
1551  rxParameters.addParameter("Error");
1552  rxParameters.addParameter("TRACETriggerStatus");
1553 
1554  std::string modifiedTriggerStatus = "";
1555  std::string xdaqHostname;
1556  StringMacros::getVectorFromString(hostList, hosts, {';'});
1557  for(auto& host : hosts)
1558  {
1559  // identify artdaq hosts to go through ARTDAQ supervisor
1560  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1561  __SUP_COUTV__(host);
1562  if(host.size() < 3)
1563  continue; // skip bad hostnames
1564 
1565  // use map to convert to xdaq host
1566  try
1567  {
1568  xdaqHostname = traceMapToXDAQHostname_.at(host);
1569  }
1570  catch(...)
1571  {
1572  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1573  << host << "' to xdaq Context hostname." << __E__;
1574  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1575  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1576  << __E__;
1577  __SUP_SS_THROW__;
1578  }
1579 
1580  __SUP_COUTV__(xdaqHostname);
1581 
1582  auto& appInfo = allTraceApps.at(xdaqHostname);
1583  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1584  << appInfo.getId() << " name = " << appInfo.getName()
1585  << " class = " << appInfo.getClass()
1586  << " hostname = " << appInfo.getHostname() << __E__;
1587  try
1588  {
1589  SOAPParameters txParameters; // params for xoap to send
1590  txParameters.addParameter("Request", "ResetTRACE");
1591  txParameters.addParameter("Host", host);
1592 
1593  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1594  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1595  SOAPUtilities::receive(retMsg, rxParameters);
1596  __SUP_COUT__ << "Received TRACE response: "
1597  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1598  << SOAPUtilities::translate(retMsg) << __E__;
1599 
1600  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1601  {
1602  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1603  "hostname = "
1604  << host << "/" << appInfo.getId()
1605  << " name = " << appInfo.getName()
1606  << " class = " << appInfo.getClass()
1607  << " hostname = " << appInfo.getHostname() << __E__;
1608  __SUP_SS_THROW__;
1609  }
1610  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1611  {
1612  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1613  << __E__;
1614  __SUP_SS_THROW__;
1615  }
1616  }
1617  catch(const xdaq::exception::Exception& e)
1618  {
1619  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1620  << appInfo.getId() << " name = " << appInfo.getName()
1621  << ". \n\n"
1622  << e.what() << __E__;
1623  __SUP_SS_THROW__;
1624  }
1625 
1626  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1627  } // end host set TRACE loop
1628 
1629  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1630  << modifiedTriggerStatus << __E__;
1631  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1632  } // end ResetTRACE
1633  else if(requestType == "EnableTRACE")
1634  {
1635  __SUP_COUT__ << "requestType " << requestType << __E__;
1636 
1637  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1638  std::string enable = CgiDataUtilities::postData(cgiIn, "enable");
1639 
1640  __SUP_COUTV__(hostList);
1641  __SUP_COUTV__(enable);
1642 
1643  std::vector<std::string /*host*/> hosts;
1644 
1645  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1646 
1647  SOAPParameters rxParameters; // params for xoap to recv
1648  rxParameters.addParameter("Command");
1649  rxParameters.addParameter("Error");
1650  rxParameters.addParameter("TRACETriggerStatus");
1651 
1652  std::string modifiedTriggerStatus = "";
1653  std::string xdaqHostname;
1654  StringMacros::getVectorFromString(hostList, hosts, {';'});
1655  for(auto& host : hosts)
1656  {
1657  // identify artdaq hosts to go through ARTDAQ supervisor
1658  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1659  __SUP_COUTV__(host);
1660  if(host.size() < 3)
1661  continue; // skip bad hostnames
1662 
1663  // use map to convert to xdaq host
1664  try
1665  {
1666  xdaqHostname = traceMapToXDAQHostname_.at(host);
1667  }
1668  catch(...)
1669  {
1670  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1671  << host << "' to xdaq Context hostname." << __E__;
1672  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1673  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1674  << __E__;
1675  __SUP_SS_THROW__;
1676  }
1677 
1678  __SUP_COUTV__(xdaqHostname);
1679 
1680  auto& appInfo = allTraceApps.at(xdaqHostname);
1681  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1682  << appInfo.getId() << " name = " << appInfo.getName()
1683  << " class = " << appInfo.getClass()
1684  << " hostname = " << appInfo.getHostname() << __E__;
1685  try
1686  {
1687  SOAPParameters txParameters; // params for xoap to send
1688  txParameters.addParameter("Request", "EnableTRACE");
1689  txParameters.addParameter("Host", host);
1690  txParameters.addParameter("SetEnable", enable);
1691 
1692  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1693  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1694  SOAPUtilities::receive(retMsg, rxParameters);
1695  __SUP_COUT__ << "Received TRACE response: "
1696  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1697  << SOAPUtilities::translate(retMsg) << __E__;
1698 
1699  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1700  {
1701  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1702  "hostname = "
1703  << host << "/" << appInfo.getId()
1704  << " name = " << appInfo.getName()
1705  << " class = " << appInfo.getClass()
1706  << " hostname = " << appInfo.getHostname() << __E__;
1707  __SUP_SS_THROW__;
1708  }
1709  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1710  {
1711  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1712  << __E__;
1713  __SUP_SS_THROW__;
1714  }
1715  }
1716  catch(const xdaq::exception::Exception& e)
1717  {
1718  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1719  << appInfo.getId() << " name = " << appInfo.getName()
1720  << ". \n\n"
1721  << e.what() << __E__;
1722  __SUP_SS_THROW__;
1723  }
1724 
1725  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1726  } // end host set TRACE loop
1727 
1728  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1729  << modifiedTriggerStatus << __E__;
1730  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1731  } // end EnableTRACE
1732  else if(requestType == "GetTraceSnapshot")
1733  {
1734  __SUP_COUT__ << "requestType " << requestType << __E__;
1735 
1736  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1737  std::string filterFor = CgiDataUtilities::postData(cgiIn, "filterFor");
1738  std::string filterOut = CgiDataUtilities::postData(cgiIn, "filterOut");
1739 
1740  __SUP_COUTV__(hostList);
1741  __SUP_COUTV__(filterFor);
1742  __SUP_COUTV__(filterOut);
1743 
1744  std::vector<std::string /*host*/> hosts;
1745 
1746  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1747 
1748  SOAPParameters rxParameters; // params for xoap to recv
1749  rxParameters.addParameter("Command");
1750  rxParameters.addParameter("Error");
1751  rxParameters.addParameter("TRACETriggerStatus");
1752  rxParameters.addParameter("TRACESnapshot");
1753 
1754  std::string modifiedTriggerStatus = "";
1755  std::string xdaqHostname;
1756  StringMacros::getVectorFromString(hostList, hosts, {';'});
1757  for(auto& host : hosts)
1758  {
1759  // identify artdaq hosts to go through ARTDAQ supervisor
1760  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1761  __SUP_COUTV__(host);
1762  if(host.size() < 3)
1763  continue; // skip bad hostnames
1764 
1765  // use map to convert to xdaq host
1766  try
1767  {
1768  xdaqHostname = traceMapToXDAQHostname_.at(host);
1769  }
1770  catch(...)
1771  {
1772  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1773  << host << "' to xdaq Context hostname." << __E__;
1774  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1775  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1776  << __E__;
1777  __SUP_SS_THROW__;
1778  }
1779 
1780  __SUP_COUTV__(xdaqHostname);
1781 
1782  auto& appInfo = allTraceApps.at(xdaqHostname);
1783  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1784  << appInfo.getId() << " name = " << appInfo.getName()
1785  << " class = " << appInfo.getClass()
1786  << " hostname = " << appInfo.getHostname() << __E__;
1787  try
1788  {
1789  SOAPParameters txParameters; // params for xoap to send
1790  txParameters.addParameter("Request", "GetSnapshot");
1791  txParameters.addParameter("Host", host);
1792  txParameters.addParameter("FilterForCSV", filterFor);
1793  txParameters.addParameter("FilterOutCSV", filterOut);
1794 
1795  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1796  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1797  SOAPUtilities::receive(retMsg, rxParameters);
1798  __SUP_COUT__ << "Received TRACE response: "
1799  << SOAPUtilities::translate(retMsg).getCommand() << __E__;
1800  //<< " ==> Bytes " << SOAPUtilities::translate(retMsg) << __E__;
1801 
1802  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1803  {
1804  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1805  "hostname = "
1806  << host << "/" << appInfo.getId()
1807  << " name = " << appInfo.getName()
1808  << " class = " << appInfo.getClass()
1809  << " hostname = " << appInfo.getHostname() << __E__;
1810  __SUP_SS_THROW__;
1811  }
1812  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1813  {
1814  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1815  << __E__;
1816  __SUP_SS_THROW__;
1817  }
1818  }
1819  catch(const xdaq::exception::Exception& e)
1820  {
1821  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1822  << appInfo.getId() << " name = " << appInfo.getName()
1823  << ". \n\n"
1824  << e.what() << __E__;
1825  __SUP_SS_THROW__;
1826  }
1827 
1828  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1829  xmlOut.addTextElementToData("host", host);
1830  std::string snapshot = rxParameters.getValue("TRACESnapshot");
1831  // if(snapshot.size() > 100000)
1832  // {
1833  // __SUP_COUT__ << "Truncating snapshot" << __E__;
1834  // snapshot.resize(100000);
1835  // }
1836  // xmlOut.addTextElementToData("hostSnapshot", snapshot);
1837 
1838  {
1839  std::string filename =
1840  USER_CONSOLE_SNAPSHOT_PATH + "snapshot_" + host + ".txt";
1841  __SUP_COUTV__(filename);
1842  FILE* fp = fopen(filename.c_str(), "w");
1843  if(!fp)
1844  {
1845  __SUP_SS__ << "Failed to create snapshot file: " << filename << __E__;
1846  __SUP_SS_THROW__;
1847  }
1848  fprintf(fp,
1849  "TRACE Snapshot taken at %s\n",
1850  StringMacros::getTimestampString().c_str());
1851 
1852  if(snapshot.size() > 5 && snapshot[2] != 'i')
1853  {
1854  // add header lines
1855  fprintf(
1856  fp,
1857  " idx us_tod delta pid tid cpu "
1858  " trcname lvl r msg \n");
1859  fprintf(fp,
1860  "----- ---------------- ----------- ------ ------ --- "
1861  "-------------------------------------- --- - "
1862  "--------------------------\n");
1863  }
1864  fprintf(fp, "%s", snapshot.c_str());
1865  fclose(fp);
1866  }
1867  } // end host set TRACE loop
1868 
1869  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1870  << modifiedTriggerStatus << __E__;
1871  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1872  } // end getTraceSnapshot
1873  else if(requestType == "GetCustomCountsAndActions" ||
1874  requestType == "AddCustomCountsAndAction" ||
1875  requestType == "ModifyCustomCountsAndAction")
1876  {
1877  __SUP_COUT__ << "requestType " << requestType
1878  << " size=" << priorityCustomTriggerList_.size() << __E__;
1879 
1880  //mutex scope:
1881  // lockout the messages array for the remainder of the scope
1882  // this guarantees can safely access the action queue
1883  std::lock_guard<std::mutex> lock(messageMutex_);
1884 
1885  if(requestType == "AddCustomCountsAndAction" ||
1886  requestType == "ModifyCustomCountsAndAction")
1887  {
1888  std::string needle = StringMacros::decodeURIComponent(
1889  CgiDataUtilities::postData(cgiIn, "needle"));
1890  uint32_t priority = CgiDataUtilities::postDataAsInt(cgiIn, "priority");
1891  std::string action = StringMacros::decodeURIComponent(
1892  CgiDataUtilities::postData(cgiIn, "action"));
1893  uint32_t triggerOnCount =
1894  CgiDataUtilities::postDataAsInt(cgiIn, "triggerOnCount");
1895  bool doLoop =
1896  CgiDataUtilities::postDataAsInt(cgiIn, "doLoop") > 0 ? true : false;
1897  bool isArmed =
1898  CgiDataUtilities::postDataAsInt(cgiIn, "isArmed") > 0 ? true : false;
1899 
1900  __SUP_COUTV__(needle);
1901  __SUP_COUTV__(priority);
1902  __SUP_COUTV__(action);
1903  __SUP_COUTV__(triggerOnCount);
1904  __SUP_COUTV__(doLoop);
1905  __SUP_COUTV__(isArmed);
1906 
1907  if(requestType == "ModifyCustomCountsAndAction")
1908  {
1909  std::string buttonDo = StringMacros::decodeURIComponent(
1910  CgiDataUtilities::postData(cgiIn, "buttonDo"));
1911  std::string currentNeedle =
1912  CgiDataUtilities::postData(cgiIn, "currentNeedle");
1913 
1914  //treat needle as CSV list and do in reverse order to maintain priority of group
1915  std::vector<std::string> csvNeedles =
1916  StringMacros::getVectorFromString(currentNeedle, {','});
1917  for(size_t i = csvNeedles.size() - 1; i < csvNeedles.size(); --i)
1918  {
1919  if(csvNeedles[i].size() == 0)
1920  continue; //skip empty entries
1921  //change the priority to the last placed priority entry to keep group order
1922  priority = modifyCustomTriggeredAction(
1923  StringMacros::decodeURIComponent(csvNeedles[i]),
1924  buttonDo,
1925  needle,
1926  action,
1927  priority,
1928  triggerOnCount,
1929  doLoop,
1930  isArmed);
1931  } //end csv needle list handling
1932  }
1933  else
1934  addCustomTriggeredAction(
1935  needle, action, priority, triggerOnCount, doLoop, isArmed);
1936 
1937  saveCustomCountList();
1938  } // end AddCustomCountsAndAction
1939 
1940  //always calculate untriggered count
1941  size_t untriggeredCount =
1942  messageCount_; //copy and then decrement "unique" incrementing ID for messages
1943 
1944  for(const auto& customCount : priorityCustomTriggerList_)
1945  {
1946  xercesc::DOMElement* customCountParent =
1947  xmlOut.addTextElementToData("customCount", "");
1948 
1949  if(customCount.occurrences < untriggeredCount)
1950  untriggeredCount -= customCount.occurrences;
1951  else
1952  {
1953  __SUP_SS__ << "Impossible custom count; notify admins! "
1954  << customCount.occurrences << " > " << untriggeredCount
1955  << " for "
1956  << StringMacros::vectorToString(customCount.needleSubstrings,
1957  {'*'})
1958  << __E__;
1959  __SUP_SS_THROW__;
1960  }
1961  xmlOut.addTextElementToParent(
1962  "needle",
1963  StringMacros::vectorToString(customCount.needleSubstrings, {'*'}),
1964  customCountParent);
1965  xmlOut.addTextElementToParent(
1966  "count", std::to_string(customCount.occurrences), customCountParent);
1967  xmlOut.addTextElementToParent(
1968  "action", customCount.action, customCountParent);
1969  xmlOut.addTextElementToParent("triggerOnCount",
1970  std::to_string(customCount.triggerOnCount),
1971  customCountParent);
1972  xmlOut.addTextElementToParent(
1973  "doLoop", std::to_string(customCount.doLoop), customCountParent);
1974  xmlOut.addTextElementToParent(
1975  "isArmed", std::to_string(customCount.isArmed), customCountParent);
1976  } //end adding custom counts to response xml loop
1977 
1978  //add untriggered always last
1979  xercesc::DOMElement* customCountParent =
1980  xmlOut.addTextElementToData("customCount", "");
1981  xmlOut.addTextElementToParent("needle", "< Untriggered >", customCountParent);
1982  xmlOut.addTextElementToParent(
1983  "count", std::to_string(untriggeredCount), customCountParent);
1984  xmlOut.addTextElementToParent("action", "Count Only", customCountParent);
1985  xmlOut.addTextElementToParent("triggerOnCount", "1", customCountParent);
1986  xmlOut.addTextElementToParent("doLoop", "1", customCountParent);
1987 
1988  } // end GetCustomCountsAndActions or AddCustomCountsAndAction
1989  else
1990  {
1991  __SUP_SS__ << "requestType Request, " << requestType << ", not recognized."
1992  << __E__;
1993  __SUP_SS_THROW__;
1994  }
1995 } // end request()
1996 
1997 //==============================================================================
2000 std::string ConsoleSupervisor::getStatusProgressDetail(void)
2001 {
2002  //Console Supervisor status detatil format is:
2003  // uptime, Err count, Warn count, Last Error msg, Last Warn msg
2004 
2005  //return uptime detail
2006  std::stringstream ss;
2007  ss << "Uptime: "
2008  << StringMacros::encodeURIComponent(StringMacros::getTimeDurationString(
2009  CorePropertySupervisorBase::getSupervisorUptime()));
2010 
2011  //return Err count, Warn count, Last Error msg, Last Warn msg, Last Info msg, Info count
2012 
2013  // size_t errorCount_ = 0, warnCount_ = 0;
2014  // std::string lastErrorMessage_, lastWarnMessage_;
2015  // time_t lastErrorMessageTime_ = 0, lastWarnMessageTime_ = 0;
2016 
2017  ss << ", Error #: " << errorCount_;
2018  ss << ", Warn #: " << warnCount_;
2019  ss << ", Last Error ("
2020  << (lastErrorMessageTime_ ? StringMacros::getTimestampString(lastErrorMessageTime_)
2021  : "0")
2022  << "): "
2023  << (lastErrorMessageTime_ ? StringMacros::encodeURIComponent(lastErrorMessage_)
2024  : "");
2025  ss << ", Last Warn ("
2026  << (lastWarnMessageTime_ ? StringMacros::getTimestampString(lastWarnMessageTime_)
2027  : "0")
2028  << "): "
2029  << (lastWarnMessageTime_ ? StringMacros::encodeURIComponent(lastWarnMessage_)
2030  : "");
2031  ss << ", Last Info ("
2032  << (lastInfoMessageTime_ ? StringMacros::getTimestampString(lastInfoMessageTime_)
2033  : "0")
2034  << "): "
2035  << (lastInfoMessageTime_ ? StringMacros::encodeURIComponent(lastInfoMessage_)
2036  : "");
2037  ss << ", Info #: " << infoCount_;
2038  ss << ", First Error ("
2039  << (firstErrorMessageTime_
2040  ? StringMacros::getTimestampString(firstErrorMessageTime_)
2041  : "0")
2042  << "): "
2043  << (firstErrorMessageTime_ ? StringMacros::encodeURIComponent(firstErrorMessage_)
2044  : "");
2045  ss << ", First Warn ("
2046  << (firstWarnMessageTime_ ? StringMacros::getTimestampString(firstWarnMessageTime_)
2047  : "0")
2048  << "): "
2049  << (firstWarnMessageTime_ ? StringMacros::encodeURIComponent(firstWarnMessage_)
2050  : "");
2051  ss << ", First Info ("
2052  << (firstInfoMessageTime_ ? StringMacros::getTimestampString(firstInfoMessageTime_)
2053  : "0")
2054  << "): "
2055  << (firstInfoMessageTime_ ? StringMacros::encodeURIComponent(firstInfoMessage_)
2056  : "");
2057 
2058  return ss.str();
2059 } // end getStatusProgressDetail()
2060 
2061 //==============================================================================
2084 void ConsoleSupervisor::insertMessageRefresh(HttpXmlDocument* xmlOut,
2085  const size_t lastUpdateCount)
2086 {
2087  //__SUP_COUT__ << __E__;
2088 
2089  if(messages_.size() == 0)
2090  return;
2091 
2092  // validate lastUpdateCount
2093  if(lastUpdateCount > messages_.back().getCount() && lastUpdateCount != (size_t)-1)
2094  {
2095  __SS__ << "Invalid lastUpdateCount: " << lastUpdateCount
2096  << " messagesArray size = " << messages_.back().getCount() << __E__;
2097  __SS_THROW__;
2098  }
2099 
2100  // lockout the messages array for the remainder of the scope
2101  // this guarantees the reading thread can safely access the messages
2102  std::lock_guard<std::mutex> lock(messageMutex_);
2103 
2104  xmlOut->addTextElementToData("last_update_count",
2105  std::to_string(messages_.back().getCount()));
2106 
2107  refreshParent_ = xmlOut->addTextElementToData("messages", "");
2108 
2109  bool requestOutOfSync = false;
2110  std::string requestOutOfSyncMsg;
2111 
2112  size_t refreshReadPointer = 0;
2113  if(lastUpdateCount != (size_t)-1)
2114  {
2115  while(refreshReadPointer < messages_.size() &&
2116  messages_[refreshReadPointer].getCount() <= lastUpdateCount)
2117  {
2118  ++refreshReadPointer;
2119  }
2120  }
2121 
2122  if(refreshReadPointer >= messages_.size())
2123  return;
2124 
2125  // limit number of catch-up messages
2126  if(messages_.size() - refreshReadPointer > maxClientMessageRequest_)
2127  {
2128  // __SUP_COUT__ << "Only sending latest " << maxClientMessageRequest_ << "
2129  // messages!";
2130 
2131  // auto oldrrp = refreshReadPointer;
2132  refreshReadPointer = messages_.size() - maxClientMessageRequest_;
2133 
2134  // __SS__ << "Skipping " << (refreshReadPointer - oldrrp)
2135  // << " messages because the web console has fallen behind!" << __E__;
2136  // __COUT__ << ss.str();
2137  // ConsoleMessageStruct msg(CONSOLE_SPECIAL_WARNING + ss.str(), lastUpdateCount);
2138  // auto it = messages_.begin();
2139  // std::advance(it, refreshReadPointer + 1);
2140  // messages_.insert(it, msg);
2141  }
2142 
2143  //return first_update_count, so that older messages could be retrieved later if desired by user
2144  xmlOut->addTextElementToData(
2145  "earliest_update_count",
2146  std::to_string(messages_[refreshReadPointer].getCount()));
2147 
2148  // output oldest to new
2149  for(; refreshReadPointer < messages_.size(); ++refreshReadPointer)
2150  {
2151  auto msg = messages_[refreshReadPointer];
2152  if(msg.getCount() < lastUpdateCount)
2153  {
2154  if(!requestOutOfSync) // record out of sync message once only
2155  {
2156  requestOutOfSync = true;
2157  __SS__ << "Request is out of sync! Message count should be more recent! "
2158  << msg.getCount() << " < " << lastUpdateCount << __E__;
2159  requestOutOfSyncMsg = ss.str();
2160  }
2161  // assume these messages are new (due to a system restart)
2162  // continue;
2163  }
2164 
2165  addMessageToResponse(xmlOut, msg);
2166 
2167  } //end main message add loop
2168 
2169  if(requestOutOfSync) // if request was out of sync, show message
2170  __SUP_COUT__ << requestOutOfSyncMsg;
2171 } // end insertMessageRefresh()
2172 
2173 //==============================================================================
2194 void ConsoleSupervisor::prependHistoricMessages(HttpXmlDocument* xmlOut,
2195  const size_t earliestOnhandMessageCount)
2196 {
2197  //__SUP_COUT__ << __E__;
2198 
2199  if(messages_.size() == 0)
2200  return;
2201 
2202  // validate earliestOnhandMessageCount
2203  if(earliestOnhandMessageCount >= messages_.back().getCount())
2204  {
2205  __SS__
2206  << "Invalid claim from user request of earliest onhand message sequence ID = "
2207  << earliestOnhandMessageCount
2208  << ". Latest existing sequence ID = " << messages_.back().getCount()
2209  << ". Was the Console Supervisor restarted?" << __E__;
2210  __SS_THROW__;
2211  }
2212 
2213  // lockout the messages array for the remainder of the scope
2214  // this guarantees the reading thread can safely access the messages
2215  std::lock_guard<std::mutex> lock(messageMutex_);
2216 
2217  refreshParent_ = xmlOut->addTextElementToData("messages", "");
2218 
2219  size_t refreshReadPointer = 0;
2220  size_t readCountStart = earliestOnhandMessageCount - maxClientMessageRequest_;
2221  if(readCountStart >= messages_.back().getCount()) //then wrapped around, so set to 0
2222  readCountStart = 0;
2223 
2224  //find starting read pointer
2225  while(refreshReadPointer < messages_.size() &&
2226  messages_[refreshReadPointer].getCount() < readCountStart)
2227  {
2228  ++refreshReadPointer;
2229  }
2230 
2231  if(refreshReadPointer >= messages_.size())
2232  return;
2233 
2234  xmlOut->addTextElementToData("earliest_update_count", //return new early onhand count
2235  std::to_string(readCountStart));
2236 
2237  //messages returned will be from readCountStart to earliestOnhandMessageCount-1
2238  // output oldest to new
2239  for(; refreshReadPointer < messages_.size(); ++refreshReadPointer)
2240  {
2241  auto msg = messages_[refreshReadPointer];
2242  if(messages_[refreshReadPointer].getCount() >= earliestOnhandMessageCount)
2243  break; //found last message
2244 
2245  addMessageToResponse(xmlOut, msg);
2246 
2247  } //end main message add loop
2248 
2249 } // end prependHistoricMessages()
2250 
2251 //==============================================================================
2252 void ConsoleSupervisor::addMessageToResponse(HttpXmlDocument* xmlOut,
2253  ConsoleSupervisor::ConsoleMessageStruct& msg)
2254 {
2255  // for all fields, give value
2256  for(auto& field : msg.fields)
2257  {
2258  if(field.first == ConsoleMessageStruct::FieldType::SOURCE)
2259  continue; // skip, not userful
2260  if(field.first == ConsoleMessageStruct::FieldType::SOURCEID)
2261  continue; // skip, not userful
2262  if(field.first == ConsoleMessageStruct::FieldType::SEQID)
2263  continue; // skip, not userful
2264  if(field.first == ConsoleMessageStruct::FieldType::TIMESTAMP) //use Time instead
2265  continue; // skip, not userful
2266  if(field.first ==
2267  ConsoleMessageStruct::FieldType::LEVEL) //use modified getLevel instead
2268  continue; // skip, not userful
2269 
2270  xmlOut->addTextElementToParent(
2271  "message_" + ConsoleMessageStruct::fieldNames.at(field.first),
2272  field.second,
2273  refreshParent_);
2274  } //end msg field loop
2275 
2276  // give modified level also
2277  xmlOut->addTextElementToParent("message_Level", msg.getLevel(), refreshParent_);
2278 
2279  // give timestamp also
2280  xmlOut->addTextElementToParent("message_Time", msg.getTime(), refreshParent_);
2281 
2282  // give global count index also
2283  xmlOut->addTextElementToParent(
2284  "message_Count", std::to_string(msg.getCount()), refreshParent_);
2285 
2286  //give Custom count label also (i.e., which search string this message matches, or blank "" for no match)
2287  xmlOut->addTextElementToParent(
2288  "message_Custom",
2289  StringMacros::vectorToString(msg.getCustomTriggerMatch().needleSubstrings, {'*'}),
2290  refreshParent_);
2291 
2292 } // end addMessageToResponse()
ots::CgiDataUtilities::postData
static std::string postData(cgicc::Cgicc &cgi, const std::string &needle)
ots::ConsoleSupervisor::forceSupervisorPropertyValues
virtual void forceSupervisorPropertyValues(void) override
Definition: ConsoleSupervisor.cc:1012
ots::ConsoleSupervisor::request
virtual void request(const std::string &requestType, cgicc::Cgicc &cgiIn, HttpXmlDocument &xmlOut, const WebUsers::RequestUserInfo &userInfo) override
Definition: ConsoleSupervisor.cc:1023
ots::ConsoleSupervisor::CUSTOM_TRIGGER_ACTIONS
static const std::set< std::string > CUSTOM_TRIGGER_ACTIONS
Count always happens, and System Message always happens for FSM commands.
Definition: ConsoleSupervisor.h:61
ots::ConsoleSupervisor::getStatusProgressDetail
virtual std::string getStatusProgressDetail(void) override
Definition: ConsoleSupervisor.cc:2000
ots::CoreSupervisorBase::sendAsyncExceptionToGateway
void sendAsyncExceptionToGateway(const std::string &errMsg, bool isPauseException, bool isStopException)
Parameters< std::string, std::string >::addParameter
void addParameter(const std::string name, const std::string value)
ots::XmlDocument::addTextElementToParent
xercesc::DOMElement * addTextElementToParent(const std::string &childName, const std::string &childText, xercesc::DOMElement *parent)
ots::INIT_MF
void INIT_MF(const char *name)
ots::ConsoleSupervisor::ConsoleMessageStruct::FieldType::SEQID
@ SEQID
sequence ID is incrementing number independent from each source
ots::StringMacros::getTimestampString
static std::string getTimestampString(const std::string &linuxTimeInSeconds)
ots::StringMacros::getVectorFromString
static void getVectorFromString(const std::string &inputString, std::vector< std::string > &listToReturn, const std::set< char > &delimiter={',', '|', '&'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'}, std::vector< char > *listOfDelimiters=0, bool decodeURIComponents=false)
ots::StringMacros::exec
static std::string exec(const char *cmd)
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
ots::StringMacros::vectorToString
static std::string vectorToString(const std::vector< T > &setToReturn, const std::string &delimeter=", ")
ots::StringMacros::mapToString
static std::string mapToString(const std::map< std::string, T > &mapToReturn, const std::string &primaryDelimeter=", ", const std::string &secondaryDelimeter=": ")
ots::StringMacros::getMapFromString
static void getMapFromString(const std::string &inputString, std::map< S, T > &mapToReturn, const std::set< char > &pairPairDelimiter={',', '|', '&'}, const std::set< char > &nameValueDelimiter={'=', ':'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'})
ots::StringMacros::getTimeDurationString
static std::string getTimeDurationString(const time_t durationInSeconds=time(0))
ots::StringMacros::decodeURIComponent
static std::string decodeURIComponent(const std::string &data)