otsdaq-utilities  3.02.00
ConsoleSupervisor.cc
1 #include "otsdaq-utilities/Console/ConsoleSupervisor.h"
2 #include <xdaq/NamespaceURI.h>
3 #include "otsdaq/CgiDataUtilities/CgiDataUtilities.h"
4 #include "otsdaq/Macros/CoutMacros.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/NetworkUtilities/ReceiverSocket.h"
7 #include "otsdaq/XmlUtilities/HttpXmlDocument.h"
8 
9 #include <dirent.h> //for DIR
10 #include <sys/stat.h> //for mkdir
11 #include <fstream>
12 #include <iostream>
13 #include <string>
14 #include <thread> //for std::thread
15 
16 using namespace ots;
17 
22 XDAQ_INSTANTIATOR_IMPL(ConsoleSupervisor)
23 
24 #define USER_CONSOLE_PREF_PATH \
25  std::string(__ENV__("SERVICE_DATA_PATH")) + "/ConsolePreferences/"
26 #define USER_CONSOLE_SNAPSHOT_PATH \
27  std::string(__ENV__("SERVICE_DATA_PATH")) + "/ConsoleSnapshots/"
28 #define USERS_PREFERENCES_FILETYPE "pref"
29 #define CUSTOM_COUNT_LIST_FILENAME std::string("CustomCountList.dat")
30 
31 #define QUIET_CFG_FILE \
32  std::string(__ENV__("USER_DATA")) + \
33  "/MessageFacilityConfigurations/" \
34  "QuietForwarder.cfg"
35 
36 #define CONSOLE_SPECIAL_ERROR \
37  std::string("|30-Aug-2019 15:30:17 CDT|0|||Error|Console||-1||ConsoleSupervisor|") + \
38  std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
39  std::string("|")
40 #define CONSOLE_SPECIAL_WARNING \
41  std::string( \
42  "|30-Aug-2019 15:30:17 CDT|0|||Warning|Console||-1||ConsoleSupervisor|") + \
43  std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
44  std::string("|")
45 
46 #undef __MF_SUBJECT__
47 #define __MF_SUBJECT__ "Console"
48 
49 #define CONSOLE_MISSED_NEEDLE "Console missed * packet(s)"
50 
52 const std::set<std::string> ConsoleSupervisor::CUSTOM_TRIGGER_ACTIONS({"Count Only",
53  "System Message",
54  "Halt",
55  "Stop",
56  "Pause",
57  "Soft Error",
58  "Hard Error",
59  "Run Script"});
60 
61 const std::string ConsoleSupervisor::ConsoleMessageStruct::LABEL_TRACE = "TRACE";
62 const std::string ConsoleSupervisor::ConsoleMessageStruct::LABEL_TRACE_PLUS = "TRACE+";
63 const std::map<ConsoleSupervisor::ConsoleMessageStruct::FieldType,
64  std::string /* fieldNames */>
65  ConsoleSupervisor::ConsoleMessageStruct::fieldNames({
66  {FieldType::TIMESTAMP, "Timestamp"},
67  {FieldType::SEQID, "SequenceID"},
68  {FieldType::LEVEL, "Level"},
69  {FieldType::LABEL, "Label"},
70  {FieldType::SOURCEID, "SourceID"},
71  {FieldType::HOSTNAME, "Hostname"},
72  {FieldType::SOURCE, "Source"},
73  {FieldType::FILE, "File"},
74  {FieldType::LINE, "Line"},
75  {FieldType::MSG, "Msg"},
76  });
77 
78 //==============================================================================
79 ConsoleSupervisor::ConsoleSupervisor(xdaq::ApplicationStub* stub)
80  : CoreSupervisorBase(stub)
81  , messageCount_(0)
82  , maxMessageCount_(100000)
83  , maxClientMessageRequest_(500)
84 {
85  __SUP_COUT__ << "Constructor started." << __E__;
86 
87  INIT_MF("." /*directory used is USER_DATA/LOG/.*/);
88 
89  // attempt to make directory structure (just in case)
90  mkdir(((std::string)USER_CONSOLE_PREF_PATH).c_str(), 0755);
91  mkdir(((std::string)USER_CONSOLE_SNAPSHOT_PATH).c_str(), 0755);
92 
93  xoap::bind(
94  this, &ConsoleSupervisor::resetConsoleCounts, "ResetConsoleCounts", XDAQ_NS_URI);
95 
96  init();
97 
98  //test custom count list and always force the first one to be the "Console missed" !
99  //Note: to avoid recursive triggers, Label='Console' can not trigger, so this 1st one is the only Console source trigger!
100  // Custom Trigger Philosophy
101  // - only one global custom count priority list (too complicated to manage by user, would have to timeout logins, etc..)
102  // - the admin users can modify priority of items in prority list
103  // - actions can infer others ('Count' is always inferred, in addition 'System Message' is inferred for FSM actions)
104  // - the admin users can modify actions
105  // -- including to the 1st Console-missing-message custom count
106  loadCustomCountList();
107  if(priorityCustomTriggerList_.size() ==
108  0) //then create default starting list example for user
109  {
110  addCustomTriggeredAction(CONSOLE_MISSED_NEEDLE, "System Message");
111  // addCustomTriggeredAction("runtime*4",
112  // "Halt");
113  // addCustomTriggeredAction("runtime",
114  // "System Message");
115  } //end test and force custom count list
116 
117  __SUP_COUT__ << "Constructor complete." << __E__;
118 
119 } // end constructor()
120 
121 //==============================================================================
122 ConsoleSupervisor::~ConsoleSupervisor(void) { destroy(); }
123 //==============================================================================
124 void ConsoleSupervisor::init(void)
125 {
126  // start mf msg listener
127  std::thread(
128  [](ConsoleSupervisor* cs) {
129  ConsoleSupervisor::messageFacilityReceiverWorkLoop(cs);
130  },
131  this)
132  .detach();
133 } // end init()
134 
135 //==============================================================================
136 void ConsoleSupervisor::destroy(void)
137 {
138  // called by destructor
139 } // end destroy()
140 
141 //==============================================================================
142 xoap::MessageReference ConsoleSupervisor::resetConsoleCounts(
143  xoap::MessageReference /*message*/)
144 {
145  __COUT_INFO__ << "Resetting Console Error/Warn/Info counts and preparing for new "
146  "first messages."
147  << __E__;
148 
149  // lockout the messages array for the remainder of the scope
150  // this guarantees the reading thread can safely access the messages
151  std::lock_guard<std::mutex> lock(messageMutex_);
152  errorCount_ = 0;
153  firstErrorMessageTime_ = 0;
154  lastErrorMessageTime_ = 0;
155  warnCount_ = 0;
156  firstWarnMessageTime_ = 0;
157  lastWarnMessageTime_ = 0;
158  infoCount_ = 0;
159  firstInfoMessageTime_ = 0;
160  lastInfoMessageTime_ = 0;
161 
162  return SOAPUtilities::makeSOAPMessageReference("Done");
163 } // end resetConsoleCounts()
164 
165 //==============================================================================
169 void ConsoleSupervisor::messageFacilityReceiverWorkLoop(ConsoleSupervisor* cs)
170 try
171 {
172  __COUT__ << "Starting workloop based on config file: " << QUIET_CFG_FILE << __E__;
173 
174  std::string configFile = QUIET_CFG_FILE;
175  FILE* fp = fopen(configFile.c_str(), "r");
176  if(!fp)
177  {
178  __SS__ << "File with port info could not be loaded: " << QUIET_CFG_FILE << __E__;
179  __COUT__ << "\n" << ss.str();
180  __SS_THROW__;
181  }
182  char tmp[100];
183  fgets(tmp, 100, fp); // receive port (ignore)
184  fgets(tmp, 100, fp); // destination port *** used here ***
185  int myport;
186  sscanf(tmp, "%*s %d", &myport);
187 
188  fgets(tmp, 100, fp); // destination ip *** used here ***
189  char myip[100];
190  sscanf(tmp, "%*s %s", myip);
191  fclose(fp);
192 
193  ReceiverSocket rsock(myip, myport); // Take Port from Configuration
194  try
195  {
196  rsock.initialize(0x1400000 /*socketReceiveBufferSize*/);
197  }
198  catch(...)
199  {
200  // lockout the messages array for the remainder of the scope
201  // this guarantees the reading thread can safely access the messages
202  std::lock_guard<std::mutex> lock(cs->messageMutex_);
203 
204  // NOTE: if we do not want this to be fatal, do not throw here, just print out
205 
206  if(1) // generate special message and throw for failed socket
207  {
208  __SS__ << "FATAL Console error. Could not initialize socket on port "
209  << myport
210  << ". Perhaps the port is already in use? Check for multiple stale "
211  "instances of otsdaq processes, or notify admins."
212  << " Multiple instances of otsdaq on the same node should be "
213  "possible, but port numbers must be unique."
214  << __E__;
215  __SS_THROW__;
216  }
217 
218  // generate special message to indicate failed socket
219  __SS__ << "FATAL Console error. Could not initialize socket on port " << myport
220  << ". Perhaps it is already in use? Exiting Console receive loop."
221  << __E__;
222  __COUT__ << ss.str();
223 
224  cs->messages_.emplace_back(CONSOLE_SPECIAL_ERROR + ss.str(),
225  cs->messageCount_++,
226  cs->priorityCustomTriggerList_);
227  //force time to now for auto-generated message
228  cs->messages_.back().setTime(time(0));
229  if(cs->messages_.size() > cs->maxMessageCount_)
230  {
231  cs->messages_.erase(cs->messages_.begin());
232  }
233 
234  return;
235  }
236 
237  std::string buffer;
238  int i = 0;
239  int heartbeatCount = 0;
240  int selfGeneratedMessageCount = 0;
241 
242  std::map<unsigned int, unsigned int> sourceLastSequenceID; // map from sourceID to
243  // lastSequenceID to
244  // identify missed messages
245  long long newSourceId;
246  uint32_t newSequenceId;
247  unsigned int c;
248 
249  // force a starting message
250  __COUT__ << "DEBUG messages look like this." << __E__;
251 
252  while(1)
253  {
254  // if receive succeeds display message
255 
256  //__COUTV__(i);
257 
258  if(rsock.receive(
259  buffer, 1 /*timeoutSeconds*/, 0 /*timeoutUSeconds*/, false /*verbose*/) !=
260  -1)
261  {
262  // use 1-byte "ping" to keep socket alive
263  if(buffer.size() == 1)
264  {
265  // std::cout << "Ping!" << __E__;
266  continue;
267  }
268 
269  if(i != 200)
270  {
271  __COUT__ << "Console has first message." << __E__;
272  i = 200; // mark so things are good for all time. (this indicates things
273  // are configured to be sent here)
274 
275  __COUT_INFO__ << "INFO messages look like this." << __E__;
276  __COUT_WARN__ << "WARNING messages look like this." << __E__;
277  __COUT_ERR__ << "ERROR messages look like this." << __E__;
278 
279  // //to debug special packets
280  // __SS__ << "???";
281  // cs->messages_[cs->writePointer_].set(CONSOLE_SPECIAL_ERROR
282  //+ ss.str(),
283  // cs->messageCount_++);
284  //
285  // if(++cs->writePointer_ == cs->messages_.size()) //handle
286  // wrap-around cs->writePointer_ = 0;
287  }
288 
289  if(selfGeneratedMessageCount)
290  --selfGeneratedMessageCount; // decrement internal message count
291  else // reset heartbeat if external messages are coming through
292  heartbeatCount = 0;
293 
294  //__COUT__ << buffer << __E__;
295 
296  // lockout the messages array for the remainder of the scope
297  // this guarantees the reading thread can safely access the messages
298  std::lock_guard<std::mutex> lock(cs->messageMutex_);
299 
300  // handle message stacking in packet
301  c = 0;
302  while(c < buffer.size())
303  {
304  // std::cout << "CONSOLE " << c << " sz=" << buffer.size() << " len=" <<
305  // strlen(&(buffer.c_str()[c])) << __E__;
306  cs->messages_.emplace_back(&(buffer.c_str()[c]),
307  cs->messageCount_++,
308  cs->priorityCustomTriggerList_);
309  if(cs->messages_.back().hasCustomTriggerMatchAction())
310  cs->customTriggerActionQueue_.push(
311  cs->messages_.back().getCustomTriggerMatch());
312 
313  //update system status
314  if(cs->messages_.back().getLevel() == "Error")
315  {
316  if(cs->errorCount_ == 0)
317  {
318  cs->firstErrorMessageTime_ = time(0);
319  cs->firstErrorMessage_ = cs->messages_.back().getMsg();
320  }
321 
322  cs->lastErrorMessageTime_ = time(0);
323  ++cs->errorCount_;
324  cs->lastErrorMessage_ = cs->messages_.back().getMsg();
325  }
326  else if(cs->messages_.back().getLevel() == "Warning")
327  {
328  if(cs->warnCount_ == 0)
329  {
330  cs->firstWarnMessageTime_ = time(0);
331  cs->firstWarnMessage_ = cs->messages_.back().getMsg();
332  }
333  cs->lastWarnMessageTime_ = time(0);
334  ++cs->warnCount_;
335  cs->lastWarnMessage_ = cs->messages_.back().getMsg();
336  }
337  else if(cs->messages_.back().getLevel() == "Info")
338  {
339  if(cs->infoCount_ == 0)
340  {
341  cs->firstInfoMessageTime_ = time(0);
342  cs->firstInfoMessage_ = cs->messages_.back().getMsg();
343  }
344  cs->lastInfoMessageTime_ = time(0);
345  ++cs->infoCount_;
346  cs->lastInfoMessage_ = cs->messages_.back().getMsg();
347  }
348 
349  // check if sequence ID is out of order
350  newSourceId = cs->messages_.back().getSourceIDAsNumber();
351  newSequenceId = cs->messages_.back().getSequenceIDAsNumber();
352 
353  // std::cout << rsock.getLastIncomingIPAddress() << ":" <<
354  // rsock.getLastIncomingPort() << ":newSourceId: " << newSourceId <<
355  // ":" << newSequenceId << __E__;
356 
357  if( // newSequenceId%1000 == 0 || //for debugging missed!
358  (newSourceId != -1 &&
359  sourceLastSequenceID.find(newSourceId) !=
360  sourceLastSequenceID
361  .end() && // ensure not first packet received
362  ((newSequenceId == 0 && sourceLastSequenceID[newSourceId] !=
363  (uint32_t)-1) || // wrap around case
364  newSequenceId !=
365  sourceLastSequenceID[newSourceId] + 1)) // normal sequence case
366  )
367  {
368  // missed some messages!
369  std::stringstream missedSs;
370  missedSs << "Console missed "
371  << (newSequenceId - 1) -
372  (sourceLastSequenceID[newSourceId] + 1) + 1
373  << " packet(s) [" << sourceLastSequenceID[newSourceId]
374  << " -> " << newSequenceId << "] from "
375  << cs->messages_.back().getSource() << ":" << newSourceId
376  << "! (uptime: " << cs->getSupervisorUptime() << " s)"
377  << __E__;
378  __SS__ << missedSs.str();
379  std::cout << ss.str();
380 
381  // seems like some messages at startup ALWAYs come out of order (i.e. 3, 4, 1)
382  // after startup grace period, generate special message to indicate missed packets
383  if(cs->getSupervisorUptime() > 20 /* seconds */ &&
384  sourceLastSequenceID[newSourceId] !=
385  1 && //veto missed error if sequenceID was 1 (this likely means a trace source was restarted)
386  newSequenceId !=
387  1) //veto missed error if sequenceID is 1 (this likely means a trace source was restarted)
388  {
389  cs->messages_.emplace_back(
390  CONSOLE_SPECIAL_WARNING + missedSs.str(),
391  cs->messageCount_++,
392  cs->priorityCustomTriggerList_);
393  //force time to now for auto-generated message
394  cs->messages_.back().setTime(time(0));
395  //Force a custom count because Console Label are ignored! if(cs->messages_.back().hasCustomTriggerMatchAction())
396  if(cs->priorityCustomTriggerList_.size())
397  {
398  cs->customTriggerActionQueue_.push(
399  cs->priorityCustomTriggerList_
400  [0]); //push newest action to back
401  cs->priorityCustomTriggerList_[0]
402  .occurrences++; //increment occurrences
403  cs->customTriggerActionQueue_.back()
404  .triggeredMessageCountIndex =
405  cs->messages_.back().getCount();
406  cs->messages_.back().setCustomTriggerMatch(
407  cs->customTriggerActionQueue_.back());
408  }
409  }
410  }
411 
412  // save the new last sequence ID
413  sourceLastSequenceID[newSourceId] = newSequenceId;
414 
415  while(cs->messages_.size() > 0 &&
416  cs->messages_.size() > cs->maxMessageCount_)
417  {
418  cs->messages_.erase(cs->messages_.begin());
419  }
420 
421  c += strlen(&(buffer.c_str()[c])) + 1;
422  } // end handle message stacking in packet
423  } // end received packet handling
424  else // idle network handling
425  {
426  if(i < 120) // if nothing received for 120 seconds, then something is wrong
427  // with Console configuration
428  ++i;
429 
430  sleep(1); // sleep one second, if timeout
431 
432  // every 60 heartbeatCount (2 seconds each = 1 sleep and 1 timeout) print a
433  // heartbeat message
434  if(i != 200 || // show first message, if not already a message
435  (heartbeatCount < 60 * 5 &&
436  heartbeatCount % 60 == 59)) // every ~2 min for first 5 messages
437  {
438  ++selfGeneratedMessageCount; // increment internal message count
439  __COUT__ << "Console is alive and waiting... (if no messages, next "
440  "heartbeat is in two minutes)"
441  << __E__;
442  }
443  else if(heartbeatCount % (60 * 30) == 59) // approx every hour
444  {
445  ++selfGeneratedMessageCount; // increment internal message count
446  __COUT__ << "Console is alive and waiting a long time... (if no "
447  "messages, next heartbeat is in one hour)"
448  << __E__;
449  }
450 
451  ++heartbeatCount;
452  } // end idle network handling
453 
454  // if nothing received for 2 minutes seconds, then something is wrong with Console
455  // configuration after 5 seconds there is a self-send. Which will at least
456  // confirm configuration. OR if 5 generated messages and never cleared.. then
457  // the forwarding is not working.
458  if(i == 120 || selfGeneratedMessageCount == 5)
459  {
460  __COUTV__(i);
461  __COUTV__(selfGeneratedMessageCount);
462  __COUT__ << "No messages received at Console Supervisor. Exiting Console "
463  "messageFacilityReceiverWorkLoop"
464  << __E__;
465  break; // assume something wrong, and break loop
466  }
467 
468  if(cs->customTriggerActionQueue_.size() && !cs->customTriggerActionThreadExists_)
469  {
470  cs->customTriggerActionThreadExists_ = true;
471 
472  std::thread(
473  [](ConsoleSupervisor* c) {
474  ConsoleSupervisor::customTriggerActionThread(c);
475  },
476  cs)
477  .detach();
478  }
479 
480  } // end infinite loop
481 
482 } // end messageFacilityReceiverWorkLoop()
483 catch(const std::runtime_error& e)
484 {
485  __COUT_ERR__ << "Error caught at Console Supervisor thread: " << e.what() << __E__;
486 }
487 catch(...)
488 {
489  __COUT_ERR__ << "Unknown error caught at Console Supervisor thread." << __E__;
490 } // end messageFacilityReceiverWorkLoop() exception handling
491 
492 //==============================================================================
495 void ConsoleSupervisor::customTriggerActionThread(ConsoleSupervisor* cs)
496 try
497 {
498  __COUT__ << "Starting customTriggerActionThread" << __E__;
499  CustomTriggeredAction_t triggeredAction;
500  while(1) //infinite workloop
501  {
502  { //mutex scope:
503  // lockout the messages array for the remainder of the scope
504  // this guarantees the reading thread can safely access the action queue
505  std::lock_guard<std::mutex> lock(cs->messageMutex_);
506  if(cs->customTriggerActionQueue_.size())
507  {
508  triggeredAction = cs->customTriggerActionQueue_.front();
509  cs->customTriggerActionQueue_.pop(); //pop first/oldest element
510  }
511  } //end mutex scope
512 
513  if(triggeredAction.action.size())
514  {
515  __COUTS__(2) << "Handling action '" << triggeredAction.action
516  << "' on custom count search string: "
517  << StringMacros::vectorToString(triggeredAction.needleSubstrings,
518  {'*'})
519  << __E__;
520  cs->doTriggeredAction(triggeredAction);
521  }
522 
523  triggeredAction.action = ""; //clear action for next in queue
524  triggeredAction.triggeredMessageCountIndex =
525  -1; //clear triggered message ID for next in queue
526  sleep(2); //mostly sleep
527 
528  } //end infinite workloop
529 
530 } // end customTriggerActionThread()
531 catch(const std::runtime_error& e)
532 {
533  __COUT_ERR__ << "Error caught at Console Supervisor Action thread: " << e.what()
534  << __E__;
535 }
536 catch(...)
537 {
538  __COUT_ERR__ << "Unknown error caught at Console Supervisor Action thread." << __E__;
539 } // end customTriggerActionThread() exception handling
540 
541 //==============================================================================
542 void ConsoleSupervisor::doTriggeredAction(const CustomTriggeredAction_t& triggeredAction)
543 {
544  __SUP_COUT_INFO__ << "Launching Triggered Action '" << triggeredAction.action
545  << "' fired on custom count search string: "
546  << StringMacros::vectorToString(triggeredAction.needleSubstrings,
547  {'*'})
548  << __E__;
549 
550  //valid actions:
551  // Halt
552  // Stop
553  // Pause
554  // Soft Error
555  // Hard Error
556  // System Message
557  // Run Script
558 
559  if(CUSTOM_TRIGGER_ACTIONS.find(triggeredAction.action) ==
560  CUSTOM_TRIGGER_ACTIONS.end())
561  {
562  __SUP_SS__ << "Unrecognized triggered action '" << triggeredAction.action
563  << ",' valid actions are "
564  << StringMacros::setToString(CUSTOM_TRIGGER_ACTIONS) << __E__;
565  __SUP_SS_THROW__;
566  }
567 
568  //all FSM commands include a system message
569  if(triggeredAction.action != "Count Only")
570  theRemoteWebUsers_.sendSystemMessage(
571  "*" /* to all users*/,
572  "In the Console Supervisor, a custom count fired the action '" +
573  triggeredAction.action + "' on the search string '" +
574  StringMacros::vectorToString(triggeredAction.needleSubstrings, {'*'}) +
575  "'");
576 
577  if(triggeredAction.action == "Halt")
578  {
579  try
580  {
581  ConsoleSupervisor::sendAsyncExceptionToGateway(
582  "Console-triggered FSM Halt", 0, 0);
583  }
584  catch(...)
585  {
586  theRemoteWebUsers_.sendSystemMessage(
587  "*" /* to all users*/,
588  "FSM Halt from Console Supervisor Triggered Action has failed!");
589  }
590  __SUP_COUTV__("FSM Halt triggered from console");
591  }
592  else if(triggeredAction.action == "Pause")
593  {
594  try
595  {
596  ConsoleSupervisor::sendAsyncExceptionToGateway(
597  "Console-triggered FSM Pause", 1, 0);
598  }
599  catch(...)
600  {
601  theRemoteWebUsers_.sendSystemMessage(
602  "*" /* to all users*/,
603  "FSM Pause from Console Supervisor Triggered Action has failed!");
604  }
605  __SUP_COUTV__("FSM Pause triggered from console");
606  }
607  else if(triggeredAction.action == "Stop")
608  {
609  try
610  {
611  ConsoleSupervisor::sendAsyncExceptionToGateway(
612  "Console-triggered FSM Stop", 0, 1);
613  }
614  catch(const std::exception& e)
615  {
616  theRemoteWebUsers_.sendSystemMessage(
617  "*" /* to all users*/,
618  "FSM Stop from Console Supervisor Triggered Action has failed!");
619  }
620  __SUP_COUTV__("FSM Stop triggered from console");
621  }
622  else if(triggeredAction.action == "Run Script")
623  {
624  std::string triggerScriptPath = "";
625  std::string scriptResult = "";
626  try
627  {
628  triggerScriptPath = __ENV__("OTS_CUSTOM_TRIGGER_SCRIPT");
629  triggerScriptPath = "source " + triggerScriptPath;
630  scriptResult = StringMacros::exec(triggerScriptPath.c_str());
631  __COUT_INFO__ << "The Script " << triggerScriptPath
632  << " Was launched, here is the result " << scriptResult;
633  }
634  catch(...)
635  {
636  __SS__ << "Trigger script path not defined! Please use environment variable "
637  "'OTS_CUSTOM_TRIGGER_SCRIPT' or contact admins."
638  << __E__;
639  __SS_THROW__;
640  }
641  }
642 
643 } // end doTriggeredAction()
644 
645 //==============================================================================
647 void ConsoleSupervisor::addCustomTriggeredAction(const std::string& triggerNeedle,
648  const std::string& triggerAction,
649  uint32_t priority, /* = -1 */
650  uint32_t triggerOnCount,
651  bool doLoop,
652  bool isArmed)
653 {
654  __SUP_COUTV__("Adding custom triggered action");
655  __SUP_COUTV__(triggerNeedle);
656  __SUP_COUTV__(triggerAction);
657  __SUP_COUTV__(priority);
658  __SUP_COUTV__(triggerOnCount);
659  __SUP_COUTV__(doLoop);
660  __SUP_COUTV__(isArmed);
661 
662  bool allAsterisks = true;
663  for(const auto& c : triggerNeedle)
664  if(c != '*')
665  {
666  allAsterisks = false;
667  break;
668  }
669  if(allAsterisks)
670  {
671  __SUP_SS__ << "Illegal empty Search String value for the new Custom Count and "
672  "Action! Please enter a valid Search String (* wildcards are "
673  "allowed, e.g. \"value = * seconds\")."
674  << __E__;
675  __SUP_SS_THROW__;
676  }
677 
678  //check if triggerNeedle already exists
679  uint32_t currentPriority = -1;
680  for(const auto& customTrigger : priorityCustomTriggerList_)
681  {
682  ++currentPriority; //inc first to get to 0
683  if(StringMacros::vectorToString(customTrigger.needleSubstrings, {'*'}) ==
684  triggerNeedle)
685  {
686  __SUP_SS__ << "Failure! Can not add Custom Count Search String that already "
687  "exists. Found '"
688  << triggerNeedle
689  << "' already existing at priority = " << currentPriority << __E__;
690  __SUP_SS_THROW__;
691  }
692  } //end check if already exists
693 
694  if(priority >= priorityCustomTriggerList_.size())
695  priority = priorityCustomTriggerList_.size(); //place at end
696  if(priority == 0 && triggerNeedle != CONSOLE_MISSED_NEEDLE)
697  {
698  __SUP_SS__ << "Illegal priority position of '" << priority
699  << "' requested. Please enter a priority value greater than 0. "
700  "Position 0 is reserved for identifying missing messages at the "
701  "Console Supervisor. Note: the action for missing messages, at "
702  "priority 0, may be customized by the user."
703  << __E__;
704  __SUP_SS_THROW__;
705  }
706  __SUP_COUTV__(priority);
707 
708  //valid actions:
709  // Halt
710  // Stop
711  // Pause
712  // Soft Error
713  // Hard Error
714  // System Message
715  // Run Script
716 
717  if(CUSTOM_TRIGGER_ACTIONS.find(triggerAction) == CUSTOM_TRIGGER_ACTIONS.end())
718  {
719  __SUP_SS__ << "Unrecognized triggered action '" << triggerAction
720  << ",' valid actions are "
721  << StringMacros::setToString(CUSTOM_TRIGGER_ACTIONS) << __E__;
722  __SUP_SS_THROW__;
723  }
724 
725  //insert new custom count at priority position
726  priorityCustomTriggerList_.insert(priorityCustomTriggerList_.begin() + priority,
727  CustomTriggeredAction_t());
728  // priorityCustomTriggerList_.push_back(CustomTriggeredAction_t());
729  // priorityCustomTriggerList_.back()
730 
731  //break up on substring
732  StringMacros::getVectorFromString(
733  triggerNeedle,
734  priorityCustomTriggerList_[priority].needleSubstrings,
735  {'*'} /* delimiter */,
736  {} /* do not ignore whitespace */);
737  priorityCustomTriggerList_[priority].action = triggerAction;
738  priorityCustomTriggerList_[priority].triggerOnCount = triggerOnCount;
739  priorityCustomTriggerList_[priority].doLoop = doLoop;
740  priorityCustomTriggerList_[priority].isArmed = isArmed;
741 
742  __SUP_COUT__ << "Added custom count: '"
743  << StringMacros::vectorToString(
744  priorityCustomTriggerList_[priority].needleSubstrings, "*")
745  << "' at priority: " << priority
746  << " triggered every: " << triggerOnCount << " occurrences";
747  if(doLoop)
748  __SUP_COUT__ << " and will loop.";
749  else
750  __SUP_COUT__ << " and will not loop.";
751  __SUP_COUT__ << __E__;
752 
753 } // end addCustomTriggeredAction()
754 
755 //==============================================================================
759 uint32_t ConsoleSupervisor::modifyCustomTriggeredAction(const std::string& currentNeedle,
760  const std::string& modifyType,
761  const std::string& setNeedle,
762  const std::string& setAction,
763  uint32_t setPriority,
764  uint32_t setTriggerOnCount,
765  bool setDoLoop,
766  bool setIsArmed)
767 {
768  __SUP_COUTV__(currentNeedle);
769  __SUP_COUTV__(modifyType);
770  __SUP_COUTV__(setNeedle);
771  __SUP_COUTV__(setAction);
772  __SUP_COUTV__(setPriority);
773  __SUP_COUTV__(setTriggerOnCount);
774  __SUP_COUTV__(setDoLoop);
775  __SUP_COUTV__(setIsArmed);
776 
777  //find current priority position of currentNeedle
778  uint32_t currentPriority = -1;
779  bool found = false;
780  for(const auto& customTrigger : priorityCustomTriggerList_)
781  {
782  ++currentPriority; //inc first to get to 0, -1 indicates not found
783  if(StringMacros::vectorToString(customTrigger.needleSubstrings, {'*'}) ==
784  currentNeedle)
785  {
786  found = true;
787  break; //found
788  }
789  }
790 
791  __SUP_COUTV__(currentPriority);
792  if(!found)
793  {
794  __SUP_SS__ << "Attempt to modify Custom Count Search String failed. Could not "
795  "find specified Search String '"
796  << currentNeedle << "' in prioritized list." << __E__;
797  __SUP_SS_THROW__;
798  }
799 
800  if(modifyType == "Deletion")
801  {
802  if(currentPriority == 0)
803  {
804  __SUP_SS__ << "Illegal deletion requested of priority position 0. Position 0 "
805  "is reserved for identifying missing messages at the Console "
806  "Supervisor. Note: the action of priority 0 may be customized "
807  "by the user, but it can not be deleted."
808  << __E__;
809  __SUP_SS_THROW__;
810  }
811 
812  __SUP_COUT__ << "Deleting custom count: "
813  << StringMacros::vectorToString(
814  priorityCustomTriggerList_[currentPriority].needleSubstrings,
815  {'*'})
816  << " w/action: "
817  << priorityCustomTriggerList_[currentPriority].action
818  << " and priority: " << currentPriority << __E__;
819  priorityCustomTriggerList_.erase(priorityCustomTriggerList_.begin() +
820  currentPriority);
821  return -1;
822  }
823 
824  if(modifyType == "Priority" || modifyType == "All")
825  {
826  if(setPriority >= priorityCustomTriggerList_.size())
827  setPriority = priorityCustomTriggerList_.size(); //place at end
828  if(setPriority == 0 && setNeedle != CONSOLE_MISSED_NEEDLE)
829  {
830  __SUP_SS__ << "Illegal priority position of '" << setPriority
831  << "' requested. Position 0 is reserved for identifying missing "
832  "messages at the Console Supervisor. Note: the action of "
833  "priority 0 may be customized by the user."
834  << __E__;
835  __SUP_SS_THROW__;
836  }
837  }
838  else //keep existing
839  setPriority = currentPriority;
840 
841  if(modifyType == "Action" || modifyType == "All")
842  {
843  //valid actions:
844  // Halt
845  // Stop
846  // Pause
847  // Soft Error
848  // Hard Error
849  // System Message
850 
851  if(CUSTOM_TRIGGER_ACTIONS.find(setAction) == CUSTOM_TRIGGER_ACTIONS.end())
852  {
853  __SUP_SS__ << "Unrecognized custom count action '" << setAction
854  << ",' valid actions are "
855  << StringMacros::setToString(CUSTOM_TRIGGER_ACTIONS) << __E__;
856  __SUP_SS_THROW__;
857  }
858  //modify existing action
859  priorityCustomTriggerList_[currentPriority].action = setAction;
860  }
861 
862  if(modifyType == "Search String" || modifyType == "All")
863  {
864  //modify existing needle
865  priorityCustomTriggerList_[currentPriority].needleSubstrings.clear();
866  StringMacros::getVectorFromString(
867  setNeedle,
868  priorityCustomTriggerList_[currentPriority].needleSubstrings,
869  {'*'} /* delimiter */,
870  {} /* do not ignore whitespace */);
871  }
872 
873  if(modifyType == "Trigger on Count" || modifyType == "All")
874  {
875  //modify existing action
876  priorityCustomTriggerList_[currentPriority].triggerOnCount = setTriggerOnCount;
877  }
878  if(modifyType == "Do Loop" || modifyType == "All")
879  {
880  //modify existing action
881  priorityCustomTriggerList_[currentPriority].doLoop = setDoLoop;
882  }
883  if(modifyType == "Arm Trigger" || modifyType == "All")
884  {
885  //modify existing action
886  priorityCustomTriggerList_[currentPriority].isArmed = setIsArmed;
887  }
888 
889  if(currentPriority != setPriority) //then need to copy
890  {
891  //insert new custom count at priority position
892  priorityCustomTriggerList_.insert(
893  priorityCustomTriggerList_.begin() + setPriority,
894  priorityCustomTriggerList_[currentPriority]);
895 
896  //delete from old position
897  if(currentPriority >= setPriority) //then increment after insert
898  ++currentPriority;
899 
900  priorityCustomTriggerList_.erase(priorityCustomTriggerList_.begin() +
901  currentPriority);
902 
903  if(currentPriority < setPriority) //then decrement after delete
904  --setPriority;
905  }
906 
907  __SUP_COUT__ << "Modified '" << modifyType << "' custom count: "
908  << StringMacros::vectorToString(
909  priorityCustomTriggerList_[setPriority].needleSubstrings, {'*'})
910  << " now w/action: " << priorityCustomTriggerList_[setPriority].action
911  << " and at priority: " << setPriority << __E__;
912 
913  return setPriority;
914 } // end modifyCustomTriggeredAction()
915 
916 //==============================================================================
917 void ConsoleSupervisor::loadCustomCountList()
918 {
919  // TODO: Migrate to a more robust read/write method
920  __SUP_COUT__ << "loadCustomCountList() from "
921  << USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME << __E__;
922 
923  FILE* fp = fopen((USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME).c_str(), "r");
924  if(!fp)
925  {
926  __SUP_COUT__ << "Ignoring missing Custom Count list file at path: "
927  << (USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME) << __E__;
928  return;
929  }
930  priorityCustomTriggerList_.clear();
931 
932  char line[1000]; //do not allow larger than 1000 chars!
933  uint32_t i = 0;
934  std::string needle;
935  std::string action;
936  size_t priority = 0;
937  size_t triggerOnCount = 0;
938  bool doLoop = false;
939  bool isArmed = false;
940  while(fgets(line, 1000, fp))
941  {
942  ++i;
943 
944  //ignore new line
945  if(strlen(line))
946  line[strlen(line) - 1] = '\0';
947 
948  __SUP_COUTV__(i);
949  __SUP_COUTV__(line);
950 
951  if(i == 1) //needle
952  needle = line;
953  else if(i == 2)
954  action = line;
955  else if(i == 3)
956  priority = std::stoi(line);
957  else if(i == 4)
958  triggerOnCount = std::stoi(line);
959  else if(i == 5)
960  doLoop = std::stoi(line) > 0 ? true : false;
961  else if(i == 6) // last line
962  {
963  isArmed = std::stoi(line) > 0 ? true : false;
964  __SUP_COUTTV__(needle);
965  __SUP_COUTTV__(priority);
966  __SUP_COUTTV__(action);
967  __SUP_COUTTV__(triggerOnCount);
968  __SUP_COUTTV__(doLoop);
969  __SUP_COUTTV__(isArmed);
970  if(i == 1 &&
971  needle !=
972  CONSOLE_MISSED_NEEDLE) //then force missed Console message as priority 0
973  addCustomTriggeredAction(
974  CONSOLE_MISSED_NEEDLE, "System Message", 0, 1, false, true);
975  addCustomTriggeredAction(
976  needle, action, priority, triggerOnCount, doLoop, isArmed);
977  i = 0; //reset for next entry
978  }
979  }
980  fclose(fp);
981 
982 } // end loadCustomCountList()
983 
984 //==============================================================================
985 void ConsoleSupervisor::saveCustomCountList()
986 {
987  __SUP_COUT__ << "saveCustomCountList()" << __E__;
988 
989  FILE* fp = fopen((USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME).c_str(), "w");
990  if(!fp)
991  {
992  __SUP_SS__ << "Failed to create Custom Count list file at path: "
993  << (USER_CONSOLE_PREF_PATH + CUSTOM_COUNT_LIST_FILENAME) << __E__;
994  __SUP_SS_THROW__;
995  }
996  unsigned int priority = 0;
997  for(auto& customCount : priorityCustomTriggerList_)
998  {
999  fprintf(fp,
1000  (StringMacros::vectorToString(customCount.needleSubstrings, {'*'}) + "\n")
1001  .c_str());
1002  fprintf(fp, "%s\n", customCount.action.c_str());
1003  fprintf(fp, "%d\n", priority);
1004  fprintf(fp, "%zu\n", customCount.triggerOnCount);
1005  fprintf(fp, "%s\n", customCount.doLoop ? "1" : "0");
1006  fprintf(fp, "%s\n", customCount.isArmed ? "1" : "0");
1007  ++priority;
1008  }
1009  fclose(fp);
1010 } // end saveCustomCountList()
1011 
1012 //==============================================================================
1013 void ConsoleSupervisor::defaultPage(xgi::Input* /*in*/, xgi::Output* out)
1014 {
1015  // __SUP_COUT__ << "ApplicationDescriptor LID="
1016  // << getApplicationDescriptor()->getLocalId() << __E__;
1017  *out << "<!DOCTYPE HTML><html lang='en'><frameset col='100%' row='100%'><frame "
1018  "src='/WebPath/html/Console.html?urn="
1019  << getApplicationDescriptor()->getLocalId() << "'></frameset></html>";
1020 } // end defaultPage()
1021 
1022 //==============================================================================
1025 void ConsoleSupervisor::forceSupervisorPropertyValues()
1026 {
1027  CorePropertySupervisorBase::setSupervisorProperty(
1028  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AutomatedRequestTypes,
1029  "GetConsoleMsgs");
1030 } // end forceSupervisorPropertyValues()
1031 
1032 //==============================================================================
1036 void ConsoleSupervisor::request(const std::string& requestType,
1037  cgicc::Cgicc& cgiIn,
1038  HttpXmlDocument& xmlOut,
1039  const WebUsers::RequestUserInfo& userInfo)
1040 {
1041  //__SUP_COUT__ << "requestType " << requestType << __E__;
1042 
1043  // Commands:
1044  // GetConsoleMsgs
1045  // PrependHistoricMessages
1046  // SaveUserPreferences
1047  // LoadUserPreferences
1048  // GetTraceLevels
1049  // SetTraceLevels
1050  // GetTriggerStatus
1051  // SetTriggerEnable
1052  // ResetTRACE
1053  // EnableTRACE
1054  // GetTraceSnapshot
1055  // GetCustomCountsAndActions
1056  // AddCustomCountsAndAction
1057  // ModifyCustomCountsAndAction
1058 
1059  // Note: to report to logbook admin status use
1060  // xmlOut.addTextElementToData(XML_ADMIN_STATUS,refreshTempStr_);
1061 
1062  if(requestType == "GetConsoleMsgs")
1063  {
1064  // lindex of -1 means first time and user just gets update lcount and lindex
1065  std::string lastUpdateCountStr = CgiDataUtilities::postData(cgiIn, "lcount");
1066 
1067  if(lastUpdateCountStr == "")
1068  {
1069  __SUP_COUT_ERR__ << "Invalid Parameters! lastUpdateCount="
1070  << lastUpdateCountStr << __E__;
1071  xmlOut.addTextElementToData("Error",
1072  "Error - Invalid parameters for GetConsoleMsgs.");
1073  return;
1074  }
1075 
1076  size_t lastUpdateCount = std::stoull(lastUpdateCountStr);
1077 
1078  // __SUP_COUT__ << "lastUpdateCount=" << lastUpdateCount << __E__;
1079 
1080  insertMessageRefresh(&xmlOut, lastUpdateCount);
1081  }
1082  else if(requestType == "PrependHistoricMessages")
1083  {
1084  size_t earliestOnhandMessageCount =
1085  CgiDataUtilities::postDataAsInt(cgiIn, "earlyCount");
1086  __SUP_COUTV__(earliestOnhandMessageCount);
1087  prependHistoricMessages(&xmlOut, earliestOnhandMessageCount);
1088  }
1089  else if(requestType == "SaveUserPreferences")
1090  {
1091  int colorIndex = CgiDataUtilities::postDataAsInt(cgiIn, "colorIndex");
1092  int showSideBar = CgiDataUtilities::postDataAsInt(cgiIn, "showSideBar");
1093  int noWrap = CgiDataUtilities::postDataAsInt(cgiIn, "noWrap");
1094  int messageOnly = CgiDataUtilities::postDataAsInt(cgiIn, "messageOnly");
1095  int hideLineNumers = CgiDataUtilities::postDataAsInt(cgiIn, "hideLineNumers");
1096 
1097  // __SUP_COUT__ << "requestType " << requestType << __E__;
1098  // __SUP_COUT__ << "colorIndex: " << colorIndex << __E__;
1099  // __SUP_COUT__ << "showSideBar: " << showSideBar << __E__;
1100  // __SUP_COUT__ << "noWrap: " << noWrap << __E__;
1101  // __SUP_COUT__ << "messageOnly: " << messageOnly << __E__;
1102  // __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << __E__;
1103 
1104  if(userInfo.username_ == "") // should never happen?
1105  {
1106  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_
1107  << __E__;
1108  xmlOut.addTextElementToData("Error",
1109  "Error - InvauserInfo.username_user found.");
1110  return;
1111  }
1112 
1113  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." +
1114  (std::string)USERS_PREFERENCES_FILETYPE;
1115 
1116  // __SUP_COUT__ << "Save preferences: " << fn << __E__;
1117  FILE* fp = fopen(fn.c_str(), "w");
1118  if(!fp)
1119  {
1120  __SS__;
1121  __THROW__(ss.str() + "Could not open file: " + fn);
1122  }
1123  fprintf(fp, "colorIndex %d\n", colorIndex);
1124  fprintf(fp, "showSideBar %d\n", showSideBar);
1125  fprintf(fp, "noWrap %d\n", noWrap);
1126  fprintf(fp, "messageOnly %d\n", messageOnly);
1127  fprintf(fp, "hideLineNumers %d\n", hideLineNumers);
1128  fclose(fp);
1129  }
1130  else if(requestType == "LoadUserPreferences")
1131  {
1132  // __SUP_COUT__ << "requestType " << requestType << __E__;
1133 
1134  unsigned int colorIndex, showSideBar, noWrap, messageOnly, hideLineNumers;
1135 
1136  if(userInfo.username_ == "") // should never happen?
1137  {
1138  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_
1139  << __E__;
1140  xmlOut.addTextElementToData("Error", "Error - Invalid user found.");
1141  return;
1142  }
1143 
1144  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." +
1145  (std::string)USERS_PREFERENCES_FILETYPE;
1146 
1147  // __SUP_COUT__ << "Load preferences: " << fn << __E__;
1148 
1149  FILE* fp = fopen(fn.c_str(), "r");
1150  if(!fp)
1151  {
1152  // return defaults
1153  __SUP_COUT__ << "Returning defaults." << __E__;
1154  xmlOut.addTextElementToData("colorIndex", "0");
1155  xmlOut.addTextElementToData("showSideBar", "0");
1156  xmlOut.addTextElementToData("noWrap", "1");
1157  xmlOut.addTextElementToData("messageOnly", "0");
1158  xmlOut.addTextElementToData("hideLineNumers", "1");
1159  return;
1160  }
1161  fscanf(fp, "%*s %u", &colorIndex);
1162  fscanf(fp, "%*s %u", &showSideBar);
1163  fscanf(fp, "%*s %u", &noWrap);
1164  fscanf(fp, "%*s %u", &messageOnly);
1165  fscanf(fp, "%*s %u", &hideLineNumers);
1166  fclose(fp);
1167  // __SUP_COUT__ << "colorIndex: " << colorIndex << __E__;
1168  // __SUP_COUT__ << "showSideBar: " << showSideBar << __E__;
1169  // __SUP_COUT__ << "noWrap: " << noWrap << __E__;
1170  // __SUP_COUT__ << "messageOnly: " << messageOnly << __E__;
1171  // __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << __E__;
1172 
1173  char tmpStr[20];
1174  sprintf(tmpStr, "%u", colorIndex);
1175  xmlOut.addTextElementToData("colorIndex", tmpStr);
1176  sprintf(tmpStr, "%u", showSideBar);
1177  xmlOut.addTextElementToData("showSideBar", tmpStr);
1178  sprintf(tmpStr, "%u", noWrap);
1179  xmlOut.addTextElementToData("noWrap", tmpStr);
1180  sprintf(tmpStr, "%u", messageOnly);
1181  xmlOut.addTextElementToData("messageOnly", tmpStr);
1182  sprintf(tmpStr, "%u", hideLineNumers);
1183  xmlOut.addTextElementToData("hideLineNumers", tmpStr);
1184  }
1185  else if(requestType == "GetTraceLevels")
1186  {
1187  __SUP_COUT__ << "requestType " << requestType << __E__;
1188 
1189  SOAPParameters txParameters; // params for xoap to send
1190  txParameters.addParameter("Request", "GetTraceLevels");
1191 
1192  SOAPParameters rxParameters; // params for xoap to recv
1193  rxParameters.addParameter("Command");
1194  rxParameters.addParameter("Error");
1195  rxParameters.addParameter("TRACEHostnameList");
1196  rxParameters.addParameter("TRACEList");
1197 
1198  traceMapToXDAQHostname_.clear(); // reset
1199 
1200  std::string traceList = "";
1201  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1202  for(const auto& appInfo : allTraceApps)
1203  {
1204  __SUP_COUT__ << "Supervisor hostname = " << appInfo.first << "/"
1205  << appInfo.second.getId()
1206  << " name = " << appInfo.second.getName()
1207  << " class = " << appInfo.second.getClass()
1208  << " hostname = " << appInfo.second.getHostname() << __E__;
1209  try
1210  {
1211  xoap::MessageReference retMsg =
1212  SOAPMessenger::sendWithSOAPReply(appInfo.second.getDescriptor(),
1213  "TRACESupervisorRequest",
1214  txParameters);
1215  SOAPUtilities::receive(retMsg, rxParameters);
1216  __SUP_COUT__ << "Received TRACE response: "
1217  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1218  << SOAPUtilities::translate(retMsg) << __E__;
1219 
1220  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1221  {
1222  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1223  "hostname = "
1224  << appInfo.first << "/" << appInfo.second.getId()
1225  << " name = " << appInfo.second.getName()
1226  << " class = " << appInfo.second.getClass()
1227  << " hostname = " << appInfo.second.getHostname() << __E__;
1228  __SUP_SS_THROW__;
1229  }
1230  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1231  {
1232  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1233  << __E__;
1234  __SUP_SS_THROW__;
1235  }
1236  }
1237  catch(const xdaq::exception::Exception& e)
1238  {
1239  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1240  << appInfo.second.getId()
1241  << " name = " << appInfo.second.getName() << ". \n\n"
1242  << e.what() << __E__;
1243  //do not throw exception, because unable to set levels when some Supervisors are down
1244  //__SUP_SS_THROW__;
1245  __SUP_COUT_ERR__ << ss.str();
1246  continue; //skip bad Supervisor
1247  }
1248 
1249  std::vector<std::string> traceHostnameArr;
1250  __COUTTV__(rxParameters.getValue("TRACEHostnameList"));
1251  StringMacros::getVectorFromString(
1252  rxParameters.getValue("TRACEHostnameList"), traceHostnameArr, {';'});
1253  for(const auto& traceHostname : traceHostnameArr)
1254  {
1255  if(traceHostname == "")
1256  continue; //skip blanks
1257  traceMapToXDAQHostname_[traceHostname] = appInfo.first;
1258  }
1259 
1260  // traceList += ";" + appInfo.first; //insert xdaq context version of
1261  // name
1262  // //FIXME and create mapp from user's typed in xdaq
1263  // context name to TRACE hostname resolution
1264 
1265  __COUTTV__(rxParameters.getValue("TRACEList"));
1266  traceList += rxParameters.getValue("TRACEList");
1267 
1268  } // end app get TRACE loop
1269  __SUP_COUT__ << "TRACE hostname map received: \n"
1270  << StringMacros::mapToString(traceMapToXDAQHostname_) << __E__;
1271  __SUP_COUT__ << "TRACE List received: \n" << traceList << __E__;
1272  xmlOut.addTextElementToData("traceList", traceList);
1273  } // end GetTraceLevels
1274  else if(requestType == "SetTraceLevels")
1275  {
1276  __SUP_COUT__ << "requestType " << requestType << __E__;
1277 
1278  std::string individualValues =
1279  CgiDataUtilities::postData(cgiIn, "individualValues");
1280  std::string hostLabelMap = CgiDataUtilities::postData(cgiIn, "hostLabelMap");
1281  std::string setMode = CgiDataUtilities::postData(cgiIn, "setMode");
1282  std::string setValueMSB = CgiDataUtilities::postData(cgiIn, "setValueMSB");
1283  std::string setValueLSB = CgiDataUtilities::postData(cgiIn, "setValueLSB");
1284 
1285  __SUP_COUTV__(individualValues);
1286  __SUP_COUTV__(setMode);
1287  // set modes: SLOW, FAST, TRIGGER
1288  __SUP_COUTV__(setValueMSB);
1289  __SUP_COUTV__(setValueLSB);
1290 
1291  std::map<std::string /*host*/, std::string /*labelArr*/> hostToLabelMap;
1292 
1293  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1294 
1295  SOAPParameters rxParameters; // params for xoap to recv
1296  rxParameters.addParameter("Command");
1297  rxParameters.addParameter("Error");
1298  rxParameters.addParameter("TRACEList");
1299 
1300  std::string modifiedTraceList = "";
1301  std::string xdaqHostname;
1302  StringMacros::getMapFromString(hostLabelMap, hostToLabelMap, {';'}, {':'});
1303  for(auto& hostLabelsPair : hostToLabelMap)
1304  {
1305  // identify artdaq hosts to go through ARTDAQ supervisor
1306  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1307  __SUP_COUTV__(hostLabelsPair.first);
1308  __SUP_COUTV__(hostLabelsPair.second);
1309 
1310  // use map to convert to xdaq host
1311  try
1312  {
1313  xdaqHostname = traceMapToXDAQHostname_.at(hostLabelsPair.first);
1314  }
1315  catch(...)
1316  {
1317  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1318  << hostLabelsPair.first << "' to xdaq Context hostname."
1319  << __E__;
1320  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1321  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1322  << __E__;
1323  __SUP_SS_THROW__;
1324  }
1325 
1326  __SUP_COUTV__(xdaqHostname);
1327 
1328  auto& appInfo = allTraceApps.at(xdaqHostname);
1329  __SUP_COUT__ << "Supervisor hostname = " << hostLabelsPair.first << "/"
1330  << xdaqHostname << ":" << appInfo.getId()
1331  << " name = " << appInfo.getName()
1332  << " class = " << appInfo.getClass()
1333  << " hostname = " << appInfo.getHostname() << __E__;
1334  try
1335  {
1336  SOAPParameters txParameters; // params for xoap to send
1337  txParameters.addParameter("Request", "SetTraceLevels");
1338  txParameters.addParameter("IndividualValues", individualValues);
1339  txParameters.addParameter("Host", hostLabelsPair.first);
1340  txParameters.addParameter("SetMode", setMode);
1341  txParameters.addParameter("Labels", hostLabelsPair.second);
1342  txParameters.addParameter("SetValueMSB", setValueMSB);
1343  txParameters.addParameter("SetValueLSB", setValueLSB);
1344 
1345  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1346  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1347  SOAPUtilities::receive(retMsg, rxParameters);
1348  __SUP_COUT__ << "Received TRACE response: "
1349  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1350  << SOAPUtilities::translate(retMsg) << __E__;
1351 
1352  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1353  {
1354  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1355  "hostname = "
1356  << hostLabelsPair.first << "/" << appInfo.getId()
1357  << " name = " << appInfo.getName()
1358  << " class = " << appInfo.getClass()
1359  << " hostname = " << appInfo.getHostname() << __E__;
1360  __SUP_SS_THROW__;
1361  }
1362  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1363  {
1364  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1365  << __E__;
1366  __SUP_SS_THROW__;
1367  }
1368  }
1369  catch(const xdaq::exception::Exception& e)
1370  {
1371  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1372  << appInfo.getId() << " name = " << appInfo.getName()
1373  << ". \n\n"
1374  << e.what() << __E__;
1375  __SUP_SS_THROW__;
1376  }
1377 
1378  modifiedTraceList +=
1379  ";" + hostLabelsPair.first; // insert xdaq context version of name
1380  // FIXME and create mapp from user's typed in xdaq
1381  // context name to TRACE hostname resolution
1382 
1383  modifiedTraceList += rxParameters.getValue("TRACEList");
1384 
1385  } // end host set TRACE loop
1386 
1387  __SUP_COUT__ << "mod'd TRACE List received: \n" << modifiedTraceList << __E__;
1388  xmlOut.addTextElementToData("modTraceList", modifiedTraceList);
1389  } // end SetTraceLevels
1390  else if(requestType == "GetTriggerStatus")
1391  {
1392  __SUP_COUT__ << "requestType " << requestType << __E__;
1393  SOAPParameters txParameters; // params for xoap to send
1394  txParameters.addParameter("Request", "GetTriggerStatus");
1395 
1396  SOAPParameters rxParameters; // params for xoap to recv
1397  rxParameters.addParameter("Command");
1398  rxParameters.addParameter("Error");
1399  rxParameters.addParameter("TRACETriggerStatus");
1400 
1401  std::string traceTriggerStatus = "";
1402  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1403  for(const auto& appInfo : allTraceApps)
1404  {
1405  __SUP_COUT__ << "Supervisor hostname = " << appInfo.first << "/"
1406  << appInfo.second.getId()
1407  << " name = " << appInfo.second.getName()
1408  << " class = " << appInfo.second.getClass()
1409  << " hostname = " << appInfo.second.getHostname() << __E__;
1410  try
1411  {
1412  xoap::MessageReference retMsg =
1413  SOAPMessenger::sendWithSOAPReply(appInfo.second.getDescriptor(),
1414  "TRACESupervisorRequest",
1415  txParameters);
1416  SOAPUtilities::receive(retMsg, rxParameters);
1417  __SUP_COUT__ << "Received TRACE response: "
1418  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1419  << SOAPUtilities::translate(retMsg) << __E__;
1420 
1421  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1422  {
1423  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1424  "hostname = "
1425  << appInfo.first << "/" << appInfo.second.getId()
1426  << " name = " << appInfo.second.getName()
1427  << " class = " << appInfo.second.getClass()
1428  << " hostname = " << appInfo.second.getHostname() << __E__;
1429  __SUP_SS_THROW__;
1430  }
1431  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1432  {
1433  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1434  << __E__;
1435  __SUP_SS_THROW__;
1436  }
1437  }
1438  catch(const xdaq::exception::Exception& e)
1439  {
1440  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1441  << appInfo.second.getId()
1442  << " name = " << appInfo.second.getName() << ". \n\n"
1443  << e.what() << __E__;
1444  __SUP_SS_THROW__;
1445  }
1446 
1447  traceTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1448 
1449  } // end app get TRACE loop
1450  __SUP_COUT__ << "TRACE Trigger Status received: \n"
1451  << traceTriggerStatus << __E__;
1452  xmlOut.addTextElementToData("traceTriggerStatus", traceTriggerStatus);
1453  } // end GetTriggerStatus
1454  else if(requestType == "SetTriggerEnable")
1455  {
1456  __SUP_COUT__ << "requestType " << requestType << __E__;
1457 
1458  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1459 
1460  __SUP_COUTV__(hostList);
1461 
1462  std::vector<std::string /*host*/> hosts;
1463 
1464  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1465 
1466  SOAPParameters rxParameters; // params for xoap to recv
1467  rxParameters.addParameter("Command");
1468  rxParameters.addParameter("Error");
1469  rxParameters.addParameter("TRACETriggerStatus");
1470 
1471  std::string modifiedTriggerStatus = "";
1472  std::string xdaqHostname;
1473  StringMacros::getVectorFromString(hostList, hosts, {';'});
1474  for(auto& host : hosts)
1475  {
1476  // identify artdaq hosts to go through ARTDAQ supervisor
1477  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1478  __SUP_COUTV__(host);
1479  if(host.size() < 3)
1480  continue; // skip bad hostnames
1481 
1482  // use map to convert to xdaq host
1483  try
1484  {
1485  xdaqHostname = traceMapToXDAQHostname_.at(host);
1486  }
1487  catch(...)
1488  {
1489  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1490  << host << "' to xdaq Context hostname." << __E__;
1491  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1492  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1493  << __E__;
1494  __SUP_SS_THROW__;
1495  }
1496 
1497  __SUP_COUTV__(xdaqHostname);
1498 
1499  auto& appInfo = allTraceApps.at(xdaqHostname);
1500  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1501  << appInfo.getId() << " name = " << appInfo.getName()
1502  << " class = " << appInfo.getClass()
1503  << " hostname = " << appInfo.getHostname() << __E__;
1504  try
1505  {
1506  SOAPParameters txParameters; // params for xoap to send
1507  txParameters.addParameter("Request", "SetTriggerEnable");
1508  txParameters.addParameter("Host", host);
1509 
1510  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1511  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1512  SOAPUtilities::receive(retMsg, rxParameters);
1513  __SUP_COUT__ << "Received TRACE response: "
1514  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1515  << SOAPUtilities::translate(retMsg) << __E__;
1516 
1517  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1518  {
1519  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1520  "hostname = "
1521  << host << "/" << appInfo.getId()
1522  << " name = " << appInfo.getName()
1523  << " class = " << appInfo.getClass()
1524  << " hostname = " << appInfo.getHostname() << __E__;
1525  __SUP_SS_THROW__;
1526  }
1527  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1528  {
1529  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1530  << __E__;
1531  __SUP_SS_THROW__;
1532  }
1533  }
1534  catch(const xdaq::exception::Exception& e)
1535  {
1536  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1537  << appInfo.getId() << " name = " << appInfo.getName()
1538  << ". \n\n"
1539  << e.what() << __E__;
1540  __SUP_SS_THROW__;
1541  }
1542 
1543  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1544  } // end host set TRACE loop
1545 
1546  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1547  << modifiedTriggerStatus << __E__;
1548  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1549  } // end SetTriggerEnable
1550  else if(requestType == "ResetTRACE")
1551  {
1552  __SUP_COUT__ << "requestType " << requestType << __E__;
1553 
1554  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1555 
1556  __SUP_COUTV__(hostList);
1557 
1558  std::vector<std::string /*host*/> hosts;
1559 
1560  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1561 
1562  SOAPParameters rxParameters; // params for xoap to recv
1563  rxParameters.addParameter("Command");
1564  rxParameters.addParameter("Error");
1565  rxParameters.addParameter("TRACETriggerStatus");
1566 
1567  std::string modifiedTriggerStatus = "";
1568  std::string xdaqHostname;
1569  StringMacros::getVectorFromString(hostList, hosts, {';'});
1570  for(auto& host : hosts)
1571  {
1572  // identify artdaq hosts to go through ARTDAQ supervisor
1573  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1574  __SUP_COUTV__(host);
1575  if(host.size() < 3)
1576  continue; // skip bad hostnames
1577 
1578  // use map to convert to xdaq host
1579  try
1580  {
1581  xdaqHostname = traceMapToXDAQHostname_.at(host);
1582  }
1583  catch(...)
1584  {
1585  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1586  << host << "' to xdaq Context hostname." << __E__;
1587  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1588  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1589  << __E__;
1590  __SUP_SS_THROW__;
1591  }
1592 
1593  __SUP_COUTV__(xdaqHostname);
1594 
1595  auto& appInfo = allTraceApps.at(xdaqHostname);
1596  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1597  << appInfo.getId() << " name = " << appInfo.getName()
1598  << " class = " << appInfo.getClass()
1599  << " hostname = " << appInfo.getHostname() << __E__;
1600  try
1601  {
1602  SOAPParameters txParameters; // params for xoap to send
1603  txParameters.addParameter("Request", "ResetTRACE");
1604  txParameters.addParameter("Host", host);
1605 
1606  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1607  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1608  SOAPUtilities::receive(retMsg, rxParameters);
1609  __SUP_COUT__ << "Received TRACE response: "
1610  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1611  << SOAPUtilities::translate(retMsg) << __E__;
1612 
1613  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1614  {
1615  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1616  "hostname = "
1617  << host << "/" << appInfo.getId()
1618  << " name = " << appInfo.getName()
1619  << " class = " << appInfo.getClass()
1620  << " hostname = " << appInfo.getHostname() << __E__;
1621  __SUP_SS_THROW__;
1622  }
1623  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1624  {
1625  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1626  << __E__;
1627  __SUP_SS_THROW__;
1628  }
1629  }
1630  catch(const xdaq::exception::Exception& e)
1631  {
1632  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1633  << appInfo.getId() << " name = " << appInfo.getName()
1634  << ". \n\n"
1635  << e.what() << __E__;
1636  __SUP_SS_THROW__;
1637  }
1638 
1639  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1640  } // end host set TRACE loop
1641 
1642  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1643  << modifiedTriggerStatus << __E__;
1644  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1645  } // end ResetTRACE
1646  else if(requestType == "EnableTRACE")
1647  {
1648  __SUP_COUT__ << "requestType " << requestType << __E__;
1649 
1650  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1651  std::string enable = CgiDataUtilities::postData(cgiIn, "enable");
1652 
1653  __SUP_COUTV__(hostList);
1654  __SUP_COUTV__(enable);
1655 
1656  std::vector<std::string /*host*/> hosts;
1657 
1658  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1659 
1660  SOAPParameters rxParameters; // params for xoap to recv
1661  rxParameters.addParameter("Command");
1662  rxParameters.addParameter("Error");
1663  rxParameters.addParameter("TRACETriggerStatus");
1664 
1665  std::string modifiedTriggerStatus = "";
1666  std::string xdaqHostname;
1667  StringMacros::getVectorFromString(hostList, hosts, {';'});
1668  for(auto& host : hosts)
1669  {
1670  // identify artdaq hosts to go through ARTDAQ supervisor
1671  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1672  __SUP_COUTV__(host);
1673  if(host.size() < 3)
1674  continue; // skip bad hostnames
1675 
1676  // use map to convert to xdaq host
1677  try
1678  {
1679  xdaqHostname = traceMapToXDAQHostname_.at(host);
1680  }
1681  catch(...)
1682  {
1683  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1684  << host << "' to xdaq Context hostname." << __E__;
1685  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1686  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1687  << __E__;
1688  __SUP_SS_THROW__;
1689  }
1690 
1691  __SUP_COUTV__(xdaqHostname);
1692 
1693  auto& appInfo = allTraceApps.at(xdaqHostname);
1694  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1695  << appInfo.getId() << " name = " << appInfo.getName()
1696  << " class = " << appInfo.getClass()
1697  << " hostname = " << appInfo.getHostname() << __E__;
1698  try
1699  {
1700  SOAPParameters txParameters; // params for xoap to send
1701  txParameters.addParameter("Request", "EnableTRACE");
1702  txParameters.addParameter("Host", host);
1703  txParameters.addParameter("SetEnable", enable);
1704 
1705  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1706  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1707  SOAPUtilities::receive(retMsg, rxParameters);
1708  __SUP_COUT__ << "Received TRACE response: "
1709  << SOAPUtilities::translate(retMsg).getCommand() << " ==> "
1710  << SOAPUtilities::translate(retMsg) << __E__;
1711 
1712  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1713  {
1714  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1715  "hostname = "
1716  << host << "/" << appInfo.getId()
1717  << " name = " << appInfo.getName()
1718  << " class = " << appInfo.getClass()
1719  << " hostname = " << appInfo.getHostname() << __E__;
1720  __SUP_SS_THROW__;
1721  }
1722  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1723  {
1724  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1725  << __E__;
1726  __SUP_SS_THROW__;
1727  }
1728  }
1729  catch(const xdaq::exception::Exception& e)
1730  {
1731  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1732  << appInfo.getId() << " name = " << appInfo.getName()
1733  << ". \n\n"
1734  << e.what() << __E__;
1735  __SUP_SS_THROW__;
1736  }
1737 
1738  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1739  } // end host set TRACE loop
1740 
1741  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1742  << modifiedTriggerStatus << __E__;
1743  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1744  } // end EnableTRACE
1745  else if(requestType == "GetTraceSnapshot")
1746  {
1747  __SUP_COUT__ << "requestType " << requestType << __E__;
1748 
1749  std::string hostList = CgiDataUtilities::postData(cgiIn, "hostList");
1750  std::string filterFor = CgiDataUtilities::postData(cgiIn, "filterFor");
1751  std::string filterOut = CgiDataUtilities::postData(cgiIn, "filterOut");
1752 
1753  __SUP_COUTV__(hostList);
1754  __SUP_COUTV__(filterFor);
1755  __SUP_COUTV__(filterOut);
1756 
1757  std::vector<std::string /*host*/> hosts;
1758 
1759  auto& allTraceApps = allSupervisorInfo_.getAllTraceControllerSupervisorInfo();
1760 
1761  SOAPParameters rxParameters; // params for xoap to recv
1762  rxParameters.addParameter("Command");
1763  rxParameters.addParameter("Error");
1764  rxParameters.addParameter("TRACETriggerStatus");
1765  rxParameters.addParameter("TRACESnapshot");
1766 
1767  std::string modifiedTriggerStatus = "";
1768  std::string xdaqHostname;
1769  StringMacros::getVectorFromString(hostList, hosts, {';'});
1770  for(auto& host : hosts)
1771  {
1772  // identify artdaq hosts to go through ARTDAQ supervisor
1773  // by adding "artdaq.." to hostname artdaq..correlator2.fnal.gov
1774  __SUP_COUTV__(host);
1775  if(host.size() < 3)
1776  continue; // skip bad hostnames
1777 
1778  // use map to convert to xdaq host
1779  try
1780  {
1781  xdaqHostname = traceMapToXDAQHostname_.at(host);
1782  }
1783  catch(...)
1784  {
1785  __SUP_SS__ << "Could not find the translation from TRACE hostname '"
1786  << host << "' to xdaq Context hostname." << __E__;
1787  ss << "Here is the existing map (size=" << traceMapToXDAQHostname_.size()
1788  << "): " << StringMacros::mapToString(traceMapToXDAQHostname_)
1789  << __E__;
1790  __SUP_SS_THROW__;
1791  }
1792 
1793  __SUP_COUTV__(xdaqHostname);
1794 
1795  auto& appInfo = allTraceApps.at(xdaqHostname);
1796  __SUP_COUT__ << "Supervisor hostname = " << host << "/" << xdaqHostname << ":"
1797  << appInfo.getId() << " name = " << appInfo.getName()
1798  << " class = " << appInfo.getClass()
1799  << " hostname = " << appInfo.getHostname() << __E__;
1800  try
1801  {
1802  SOAPParameters txParameters; // params for xoap to send
1803  txParameters.addParameter("Request", "GetSnapshot");
1804  txParameters.addParameter("Host", host);
1805  txParameters.addParameter("FilterForCSV", filterFor);
1806  txParameters.addParameter("FilterOutCSV", filterOut);
1807 
1808  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
1809  appInfo.getDescriptor(), "TRACESupervisorRequest", txParameters);
1810  SOAPUtilities::receive(retMsg, rxParameters);
1811  __SUP_COUT__ << "Received TRACE response: "
1812  << SOAPUtilities::translate(retMsg).getCommand() << __E__;
1813  //<< " ==> Bytes " << SOAPUtilities::translate(retMsg) << __E__;
1814 
1815  if(SOAPUtilities::translate(retMsg).getCommand() == "Fault")
1816  {
1817  __SUP_SS__ << "Unrecognized command at destination TRACE Supervisor "
1818  "hostname = "
1819  << host << "/" << appInfo.getId()
1820  << " name = " << appInfo.getName()
1821  << " class = " << appInfo.getClass()
1822  << " hostname = " << appInfo.getHostname() << __E__;
1823  __SUP_SS_THROW__;
1824  }
1825  else if(SOAPUtilities::translate(retMsg).getCommand() == "TRACEFault")
1826  {
1827  __SUP_SS__ << "Error received: " << rxParameters.getValue("Error")
1828  << __E__;
1829  __SUP_SS_THROW__;
1830  }
1831  }
1832  catch(const xdaq::exception::Exception& e)
1833  {
1834  __SUP_SS__ << "Error transmitting request to TRACE Supervisor LID = "
1835  << appInfo.getId() << " name = " << appInfo.getName()
1836  << ". \n\n"
1837  << e.what() << __E__;
1838  __SUP_SS_THROW__;
1839  }
1840 
1841  modifiedTriggerStatus += rxParameters.getValue("TRACETriggerStatus");
1842  xmlOut.addTextElementToData("host", host);
1843  std::string snapshot = rxParameters.getValue("TRACESnapshot");
1844  // if(snapshot.size() > 100000)
1845  // {
1846  // __SUP_COUT__ << "Truncating snapshot" << __E__;
1847  // snapshot.resize(100000);
1848  // }
1849  // xmlOut.addTextElementToData("hostSnapshot", snapshot);
1850 
1851  {
1852  std::string filename =
1853  USER_CONSOLE_SNAPSHOT_PATH + "snapshot_" + host + ".txt";
1854  __SUP_COUTV__(filename);
1855  FILE* fp = fopen(filename.c_str(), "w");
1856  if(!fp)
1857  {
1858  __SUP_SS__ << "Failed to create snapshot file: " << filename << __E__;
1859  __SUP_SS_THROW__;
1860  }
1861  fprintf(fp,
1862  "TRACE Snapshot taken at %s\n",
1863  StringMacros::getTimestampString().c_str());
1864 
1865  if(snapshot.size() > 5 && snapshot[2] != 'i')
1866  {
1867  // add header lines
1868  fprintf(
1869  fp,
1870  " idx us_tod delta pid tid cpu "
1871  " trcname lvl r msg \n");
1872  fprintf(fp,
1873  "----- ---------------- ----------- ------ ------ --- "
1874  "-------------------------------------- --- - "
1875  "--------------------------\n");
1876  }
1877  fprintf(fp, "%s", snapshot.c_str());
1878  fclose(fp);
1879  }
1880  } // end host set TRACE loop
1881 
1882  __SUP_COUT__ << "mod'd TRACE Trigger Status received: \n"
1883  << modifiedTriggerStatus << __E__;
1884  xmlOut.addTextElementToData("modTriggerStatus", modifiedTriggerStatus);
1885  } // end getTraceSnapshot
1886  else if(requestType == "GetCustomCountsAndActions" ||
1887  requestType == "AddCustomCountsAndAction" ||
1888  requestType == "ModifyCustomCountsAndAction")
1889  {
1890  __SUP_COUT__ << "requestType " << requestType
1891  << " size=" << priorityCustomTriggerList_.size() << __E__;
1892 
1893  //mutex scope:
1894  // lockout the messages array for the remainder of the scope
1895  // this guarantees can safely access the action queue
1896  std::lock_guard<std::mutex> lock(messageMutex_);
1897 
1898  if(requestType == "AddCustomCountsAndAction" ||
1899  requestType == "ModifyCustomCountsAndAction")
1900  {
1901  std::string needle = StringMacros::decodeURIComponent(
1902  CgiDataUtilities::postData(cgiIn, "needle"));
1903  uint32_t priority = CgiDataUtilities::postDataAsInt(cgiIn, "priority");
1904  std::string action = StringMacros::decodeURIComponent(
1905  CgiDataUtilities::postData(cgiIn, "action"));
1906  uint32_t triggerOnCount =
1907  CgiDataUtilities::postDataAsInt(cgiIn, "triggerOnCount");
1908  bool doLoop =
1909  CgiDataUtilities::postDataAsInt(cgiIn, "doLoop") > 0 ? true : false;
1910  bool isArmed =
1911  CgiDataUtilities::postDataAsInt(cgiIn, "isArmed") > 0 ? true : false;
1912 
1913  __SUP_COUTV__(needle);
1914  __SUP_COUTV__(priority);
1915  __SUP_COUTV__(action);
1916  __SUP_COUTV__(triggerOnCount);
1917  __SUP_COUTV__(doLoop);
1918  __SUP_COUTV__(isArmed);
1919 
1920  if(requestType == "ModifyCustomCountsAndAction")
1921  {
1922  std::string buttonDo = StringMacros::decodeURIComponent(
1923  CgiDataUtilities::postData(cgiIn, "buttonDo"));
1924  std::string currentNeedle =
1925  CgiDataUtilities::postData(cgiIn, "currentNeedle");
1926 
1927  //treat needle as CSV list and do in reverse order to maintain priority of group
1928  std::vector<std::string> csvNeedles =
1929  StringMacros::getVectorFromString(currentNeedle, {','});
1930  for(size_t i = csvNeedles.size() - 1; i < csvNeedles.size(); --i)
1931  {
1932  if(csvNeedles[i].size() == 0)
1933  continue; //skip empty entries
1934  //change the priority to the last placed priority entry to keep group order
1935  priority = modifyCustomTriggeredAction(
1936  StringMacros::decodeURIComponent(csvNeedles[i]),
1937  buttonDo,
1938  needle,
1939  action,
1940  priority,
1941  triggerOnCount,
1942  doLoop,
1943  isArmed);
1944  } //end csv needle list handling
1945  }
1946  else
1947  addCustomTriggeredAction(
1948  needle, action, priority, triggerOnCount, doLoop, isArmed);
1949 
1950  saveCustomCountList();
1951  } // end AddCustomCountsAndAction
1952 
1953  //always calculate untriggered count
1954  size_t untriggeredCount =
1955  messageCount_; //copy and then decrement "unique" incrementing ID for messages
1956 
1957  for(const auto& customCount : priorityCustomTriggerList_)
1958  {
1959  xercesc::DOMElement* customCountParent =
1960  xmlOut.addTextElementToData("customCount", "");
1961 
1962  if(customCount.occurrences < untriggeredCount)
1963  untriggeredCount -= customCount.occurrences;
1964  else
1965  {
1966  __SUP_SS__ << "Impossible custom count; notify admins! "
1967  << customCount.occurrences << " > " << untriggeredCount
1968  << " for "
1969  << StringMacros::vectorToString(customCount.needleSubstrings,
1970  {'*'})
1971  << __E__;
1972  __SUP_SS_THROW__;
1973  }
1974  xmlOut.addTextElementToParent(
1975  "needle",
1976  StringMacros::vectorToString(customCount.needleSubstrings, {'*'}),
1977  customCountParent);
1978  xmlOut.addTextElementToParent(
1979  "count", std::to_string(customCount.occurrences), customCountParent);
1980  xmlOut.addTextElementToParent(
1981  "action", customCount.action, customCountParent);
1982  xmlOut.addTextElementToParent("triggerOnCount",
1983  std::to_string(customCount.triggerOnCount),
1984  customCountParent);
1985  xmlOut.addTextElementToParent(
1986  "doLoop", std::to_string(customCount.doLoop), customCountParent);
1987  xmlOut.addTextElementToParent(
1988  "isArmed", std::to_string(customCount.isArmed), customCountParent);
1989  } //end adding custom counts to response xml loop
1990 
1991  //add untriggered always last
1992  xercesc::DOMElement* customCountParent =
1993  xmlOut.addTextElementToData("customCount", "");
1994  xmlOut.addTextElementToParent("needle", "< Untriggered >", customCountParent);
1995  xmlOut.addTextElementToParent(
1996  "count", std::to_string(untriggeredCount), customCountParent);
1997  xmlOut.addTextElementToParent("action", "Count Only", customCountParent);
1998  xmlOut.addTextElementToParent("triggerOnCount", "1", customCountParent);
1999  xmlOut.addTextElementToParent("doLoop", "1", customCountParent);
2000 
2001  } // end GetCustomCountsAndActions or AddCustomCountsAndAction
2002  else
2003  {
2004  __SUP_SS__ << "requestType Request, " << requestType
2005  << ", not recognized by the Console Supervisor (was it intended for "
2006  "another Supervisor?)."
2007  << __E__;
2008  __SUP_SS_THROW__;
2009  }
2010 } // end request()
2011 
2012 //==============================================================================
2015 std::string ConsoleSupervisor::getStatusProgressDetail(void)
2016 {
2017  //Console Supervisor status detatil format is:
2018  // uptime, Err count, Warn count, Last Error msg, Last Warn msg
2019 
2020  //return uptime detail
2021  std::stringstream ss;
2022  ss << "Uptime: "
2023  << StringMacros::encodeURIComponent(StringMacros::getTimeDurationString(
2024  CorePropertySupervisorBase::getSupervisorUptime()));
2025 
2026  //return Err count, Warn count, Last Error msg, Last Warn msg, Last Info msg, Info count
2027 
2028  // size_t errorCount_ = 0, warnCount_ = 0;
2029  // std::string lastErrorMessage_, lastWarnMessage_;
2030  // time_t lastErrorMessageTime_ = 0, lastWarnMessageTime_ = 0;
2031 
2032  ss << ", Error #: " << errorCount_;
2033  ss << ", Warn #: " << warnCount_;
2034  ss << ", Last Error ("
2035  << (lastErrorMessageTime_ ? StringMacros::getTimestampString(lastErrorMessageTime_)
2036  : "0")
2037  << "): "
2038  << (lastErrorMessageTime_ ? StringMacros::encodeURIComponent(lastErrorMessage_)
2039  : "");
2040  ss << ", Last Warn ("
2041  << (lastWarnMessageTime_ ? StringMacros::getTimestampString(lastWarnMessageTime_)
2042  : "0")
2043  << "): "
2044  << (lastWarnMessageTime_ ? StringMacros::encodeURIComponent(lastWarnMessage_)
2045  : "");
2046  ss << ", Last Info ("
2047  << (lastInfoMessageTime_ ? StringMacros::getTimestampString(lastInfoMessageTime_)
2048  : "0")
2049  << "): "
2050  << (lastInfoMessageTime_ ? StringMacros::encodeURIComponent(lastInfoMessage_)
2051  : "");
2052  ss << ", Info #: " << infoCount_;
2053  ss << ", First Error ("
2054  << (firstErrorMessageTime_
2055  ? StringMacros::getTimestampString(firstErrorMessageTime_)
2056  : "0")
2057  << "): "
2058  << (firstErrorMessageTime_ ? StringMacros::encodeURIComponent(firstErrorMessage_)
2059  : "");
2060  ss << ", First Warn ("
2061  << (firstWarnMessageTime_ ? StringMacros::getTimestampString(firstWarnMessageTime_)
2062  : "0")
2063  << "): "
2064  << (firstWarnMessageTime_ ? StringMacros::encodeURIComponent(firstWarnMessage_)
2065  : "");
2066  ss << ", First Info ("
2067  << (firstInfoMessageTime_ ? StringMacros::getTimestampString(firstInfoMessageTime_)
2068  : "0")
2069  << "): "
2070  << (firstInfoMessageTime_ ? StringMacros::encodeURIComponent(firstInfoMessage_)
2071  : "");
2072 
2073  return ss.str();
2074 } // end getStatusProgressDetail()
2075 
2076 //==============================================================================
2099 void ConsoleSupervisor::insertMessageRefresh(HttpXmlDocument* xmlOut,
2100  const size_t lastUpdateCount)
2101 {
2102  //__SUP_COUT__ << __E__;
2103 
2104  if(messages_.size() == 0)
2105  return;
2106 
2107  // validate lastUpdateCount
2108  if(lastUpdateCount > messages_.back().getCount() && lastUpdateCount != (size_t)-1)
2109  {
2110  __SS__ << "Invalid lastUpdateCount: " << lastUpdateCount
2111  << " messagesArray size = " << messages_.back().getCount() << __E__;
2112  __SS_THROW__;
2113  }
2114 
2115  // lockout the messages array for the remainder of the scope
2116  // this guarantees the reading thread can safely access the messages
2117  std::lock_guard<std::mutex> lock(messageMutex_);
2118 
2119  xmlOut->addTextElementToData("last_update_count",
2120  std::to_string(messages_.back().getCount()));
2121 
2122  refreshParent_ = xmlOut->addTextElementToData("messages", "");
2123 
2124  bool requestOutOfSync = false;
2125  std::string requestOutOfSyncMsg;
2126 
2127  size_t refreshReadPointer = 0;
2128  if(lastUpdateCount != (size_t)-1)
2129  {
2130  while(refreshReadPointer < messages_.size() &&
2131  messages_[refreshReadPointer].getCount() <= lastUpdateCount)
2132  {
2133  ++refreshReadPointer;
2134  }
2135  }
2136 
2137  if(refreshReadPointer >= messages_.size())
2138  return;
2139 
2140  // limit number of catch-up messages
2141  if(messages_.size() - refreshReadPointer > maxClientMessageRequest_)
2142  {
2143  // __SUP_COUT__ << "Only sending latest " << maxClientMessageRequest_ << "
2144  // messages!";
2145 
2146  // auto oldrrp = refreshReadPointer;
2147  refreshReadPointer = messages_.size() - maxClientMessageRequest_;
2148 
2149  // __SS__ << "Skipping " << (refreshReadPointer - oldrrp)
2150  // << " messages because the web console has fallen behind!" << __E__;
2151  // __COUT__ << ss.str();
2152  // ConsoleMessageStruct msg(CONSOLE_SPECIAL_WARNING + ss.str(), lastUpdateCount);
2153  // auto it = messages_.begin();
2154  // std::advance(it, refreshReadPointer + 1);
2155  // messages_.insert(it, msg);
2156  }
2157 
2158  //return first_update_count, so that older messages could be retrieved later if desired by user
2159  xmlOut->addTextElementToData(
2160  "earliest_update_count",
2161  std::to_string(messages_[refreshReadPointer].getCount()));
2162 
2163  // output oldest to new
2164  for(; refreshReadPointer < messages_.size(); ++refreshReadPointer)
2165  {
2166  auto msg = messages_[refreshReadPointer];
2167  if(msg.getCount() < lastUpdateCount)
2168  {
2169  if(!requestOutOfSync) // record out of sync message once only
2170  {
2171  requestOutOfSync = true;
2172  __SS__ << "Request is out of sync! Message count should be more recent! "
2173  << msg.getCount() << " < " << lastUpdateCount << __E__;
2174  requestOutOfSyncMsg = ss.str();
2175  }
2176  // assume these messages are new (due to a system restart)
2177  // continue;
2178  }
2179 
2180  addMessageToResponse(xmlOut, msg);
2181 
2182  } //end main message add loop
2183 
2184  if(requestOutOfSync) // if request was out of sync, show message
2185  __SUP_COUT__ << requestOutOfSyncMsg;
2186 } // end insertMessageRefresh()
2187 
2188 //==============================================================================
2209 void ConsoleSupervisor::prependHistoricMessages(HttpXmlDocument* xmlOut,
2210  const size_t earliestOnhandMessageCount)
2211 {
2212  //__SUP_COUT__ << __E__;
2213 
2214  if(messages_.size() == 0)
2215  return;
2216 
2217  // validate earliestOnhandMessageCount
2218  if(earliestOnhandMessageCount >= messages_.back().getCount())
2219  {
2220  __SS__
2221  << "Invalid claim from user request of earliest onhand message sequence ID = "
2222  << earliestOnhandMessageCount
2223  << ". Latest existing sequence ID = " << messages_.back().getCount()
2224  << ". Was the Console Supervisor restarted?" << __E__;
2225  __SS_THROW__;
2226  }
2227 
2228  // lockout the messages array for the remainder of the scope
2229  // this guarantees the reading thread can safely access the messages
2230  std::lock_guard<std::mutex> lock(messageMutex_);
2231 
2232  refreshParent_ = xmlOut->addTextElementToData("messages", "");
2233 
2234  size_t refreshReadPointer = 0;
2235  size_t readCountStart = earliestOnhandMessageCount - maxClientMessageRequest_;
2236  if(readCountStart >= messages_.back().getCount()) //then wrapped around, so set to 0
2237  readCountStart = 0;
2238 
2239  //find starting read pointer
2240  while(refreshReadPointer < messages_.size() &&
2241  messages_[refreshReadPointer].getCount() < readCountStart)
2242  {
2243  ++refreshReadPointer;
2244  }
2245 
2246  if(refreshReadPointer >= messages_.size())
2247  return;
2248 
2249  xmlOut->addTextElementToData("earliest_update_count", //return new early onhand count
2250  std::to_string(readCountStart));
2251 
2252  //messages returned will be from readCountStart to earliestOnhandMessageCount-1
2253  // output oldest to new
2254  for(; refreshReadPointer < messages_.size(); ++refreshReadPointer)
2255  {
2256  auto msg = messages_[refreshReadPointer];
2257  if(messages_[refreshReadPointer].getCount() >= earliestOnhandMessageCount)
2258  break; //found last message
2259 
2260  addMessageToResponse(xmlOut, msg);
2261 
2262  } //end main message add loop
2263 
2264 } // end prependHistoricMessages()
2265 
2266 //==============================================================================
2267 void ConsoleSupervisor::addMessageToResponse(HttpXmlDocument* xmlOut,
2268  ConsoleSupervisor::ConsoleMessageStruct& msg)
2269 {
2270  // for all fields, give value
2271  for(auto& field : msg.fields)
2272  {
2273  if(field.first == ConsoleMessageStruct::FieldType::SOURCE)
2274  continue; // skip, not userful
2275  if(field.first == ConsoleMessageStruct::FieldType::SOURCEID)
2276  continue; // skip, not userful
2277  if(field.first == ConsoleMessageStruct::FieldType::SEQID)
2278  continue; // skip, not userful
2279  if(field.first == ConsoleMessageStruct::FieldType::TIMESTAMP) //use Time instead
2280  continue; // skip, not userful
2281  if(field.first ==
2282  ConsoleMessageStruct::FieldType::LEVEL) //use modified getLevel instead
2283  continue; // skip, not userful
2284 
2285  xmlOut->addTextElementToParent(
2286  "message_" + ConsoleMessageStruct::fieldNames.at(field.first),
2287  field.second,
2288  refreshParent_);
2289  } //end msg field loop
2290 
2291  // give modified level also
2292  xmlOut->addTextElementToParent("message_Level", msg.getLevel(), refreshParent_);
2293 
2294  // give timestamp also
2295  xmlOut->addTextElementToParent("message_Time", msg.getTime(), refreshParent_);
2296 
2297  // give global count index also
2298  xmlOut->addTextElementToParent(
2299  "message_Count", std::to_string(msg.getCount()), refreshParent_);
2300 
2301  //give Custom count label also (i.e., which search string this message matches, or blank "" for no match)
2302  xmlOut->addTextElementToParent(
2303  "message_Custom",
2304  StringMacros::vectorToString(msg.getCustomTriggerMatch().needleSubstrings, {'*'}),
2305  refreshParent_);
2306 
2307 } // end addMessageToResponse()
ots::CgiDataUtilities::postData
static std::string postData(cgicc::Cgicc &cgi, const std::string &needle)
ots::ConsoleSupervisor::forceSupervisorPropertyValues
virtual void forceSupervisorPropertyValues(void) override
Definition: ConsoleSupervisor.cc:1025
ots::ConsoleSupervisor::request
virtual void request(const std::string &requestType, cgicc::Cgicc &cgiIn, HttpXmlDocument &xmlOut, const WebUsers::RequestUserInfo &userInfo) override
Definition: ConsoleSupervisor.cc:1036
ots::ConsoleSupervisor::CUSTOM_TRIGGER_ACTIONS
static const std::set< std::string > CUSTOM_TRIGGER_ACTIONS
Count always happens, and System Message always happens for FSM commands.
Definition: ConsoleSupervisor.h:61
ots::ConsoleSupervisor::getStatusProgressDetail
virtual std::string getStatusProgressDetail(void) override
Definition: ConsoleSupervisor.cc:2015
ots::CoreSupervisorBase::sendAsyncExceptionToGateway
void sendAsyncExceptionToGateway(const std::string &errMsg, bool isPauseException, bool isStopException)
Parameters< std::string, std::string >::addParameter
void addParameter(const std::string name, const std::string value)
ots::XmlDocument::addTextElementToParent
xercesc::DOMElement * addTextElementToParent(const std::string &childName, const std::string &childText, xercesc::DOMElement *parent)
ots::INIT_MF
void INIT_MF(const char *name)
ots::ConsoleSupervisor::ConsoleMessageStruct::FieldType::SEQID
@ SEQID
sequence ID is incrementing number independent from each source
ots::StringMacros::getTimestampString
static std::string getTimestampString(const std::string &linuxTimeInSeconds)
ots::StringMacros::getVectorFromString
static void getVectorFromString(const std::string &inputString, std::vector< std::string > &listToReturn, const std::set< char > &delimiter={',', '|', '&'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'}, std::vector< char > *listOfDelimiters=0, bool decodeURIComponents=false)
ots::StringMacros::exec
static std::string exec(const char *cmd)
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
ots::StringMacros::vectorToString
static std::string vectorToString(const std::vector< T > &setToReturn, const std::string &delimeter=", ")
ots::StringMacros::mapToString
static std::string mapToString(const std::map< std::string, T > &mapToReturn, const std::string &primaryDelimeter=", ", const std::string &secondaryDelimeter=": ")
ots::StringMacros::getMapFromString
static void getMapFromString(const std::string &inputString, std::map< S, T > &mapToReturn, const std::set< char > &pairPairDelimiter={',', '|', '&'}, const std::set< char > &nameValueDelimiter={'=', ':'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'})
ots::StringMacros::getTimeDurationString
static std::string getTimeDurationString(const time_t durationInSeconds=time(0))
ots::StringMacros::decodeURIComponent
static std::string decodeURIComponent(const std::string &data)