otsdaq  3.03.00
Iterator.cc
1 #include "otsdaq/GatewaySupervisor/Iterator.h"
2 #include "otsdaq/CoreSupervisors/CoreSupervisorBase.h"
3 #include "otsdaq/GatewaySupervisor/GatewaySupervisor.h"
4 #include "otsdaq/Macros/CoutMacros.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/WebUsersUtilities/WebUsers.h"
7 
8 #include <iostream>
9 #include <thread> //for std::thread
10 
11 #undef __MF_SUBJECT__
12 #define __MF_SUBJECT__ "Iterator"
13 
14 using namespace ots;
15 
16 #define ITERATOR_PLAN_HISTORY_FILENAME \
17  ((getenv("SERVICE_DATA_PATH") == NULL) \
18  ? (std::string(__ENV__("USER_DATA")) + "/ServiceData") \
19  : (std::string(__ENV__("SERVICE_DATA_PATH")))) + \
20  "/IteratorPlanHistory.hist"
21 
22 const std::string Iterator::RESERVED_GEN_PLAN_NAME = "---GENERATED_PLAN---";
23 
24 //==============================================================================
25 Iterator::Iterator(GatewaySupervisor* supervisor)
26  : workloopRunning_(false)
27  , activePlanIsRunning_(false)
28  , iteratorBusy_(false)
29  , commandPlay_(false)
30  , commandPause_(false)
31  , commandHalt_(false)
32  , activePlanName_("")
33  , activeCommandIndex_(-1)
34  , activeCommandStartTime_(0)
35  , theSupervisor_(supervisor)
36 {
37  __COUT__ << "Iterator constructed." << __E__;
38 
39  //restore lastStartedPlanName_ and lastFinishedPlanName_ from file
40  __COUT__ << "Filename for iterator history: " << ITERATOR_PLAN_HISTORY_FILENAME
41  << __E__;
42  FILE* fp = fopen((ITERATOR_PLAN_HISTORY_FILENAME).c_str(), "r");
43  if(fp) // check for all core table names in file, and force their presence
44  {
45  char line[100];
46  int i = 0;
47  while(fgets(line, 100, fp))
48  {
49  if(strlen(line) < 3)
50  continue;
51  line[strlen(line) - 1] = '\0'; // remove endline
52  __COUTV__(line);
53  if(i == 0)
54  lastStartedPlanName_ = line;
55  else if(i == 1)
56  {
57  lastFinishedPlanName_ = line;
58  break; //only 2 lines in file (for now)
59  }
60  ++i;
61  }
62  fclose(fp);
63  __COUTV__(lastStartedPlanName_);
64  __COUTV__(lastFinishedPlanName_);
65  } //end get iterator plan history
66 } //end constructor()
67 
68 //==============================================================================
69 Iterator::~Iterator(void) {}
70 
71 //==============================================================================
72 void Iterator::IteratorWorkLoop(Iterator* iterator)
73 try
74 {
75  __COUT__ << "Iterator work loop starting..." << __E__;
76 
77  // mutex init scope
78  {
79  // lockout the messages array for the remainder of the scope
80  // this guarantees the reading thread can safely access the messages
81  if(iterator->theSupervisor_->VERBOSE_MUTEX)
82  __COUT__ << "Waiting for iterator access" << __E__;
83  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
84  if(iterator->theSupervisor_->VERBOSE_MUTEX)
85  __COUT__ << "Have iterator access" << __E__;
86 
87  iterator->errorMessage_ = ""; // clear error message
88  }
89 
90  ConfigurationManagerRW theConfigurationManager(
91  WebUsers::DEFAULT_ITERATOR_USERNAME); // this is a restricted username
92  // theConfigurationManager.init();
93  theConfigurationManager.getAllTableInfo(true /* refresh */, // to prep all info
94  0 /* accumulatedWarnings */,
95  "" /* errorFilterName */,
96  false /* getGroupKeys */,
97  false /* getGroupInfo */,
98  true /* initializeActiveGroups */);
99 
100  __COUT__ << "Iterator work loop starting..." << __E__;
101  IteratorWorkLoopStruct theIteratorStruct(iterator, &theConfigurationManager);
102  __COUT__ << "Iterator work loop starting..." << __E__;
103 
104  const IterateTable* itConfig;
105 
106  std::vector<IterateTable::Command> commands;
107 
108  try
109  {
110  while(1)
111  {
112  // Process:
113  // - always "listen" for commands
114  // - play: if no plan running, activePlanIsRunning_ = true,
115  // and start or continue plan based on name/commandIndex
116  // - pause: if plan playing, pause it, activePlanIsRunning_ = false
117  // and do not clear commandIndex or name, iteratorBusy_ = true
118  // - halt: if plan playing or not, activePlanIsRunning_ = false
119  // and clear commandIndex or name, iteratorBusy_ = false
120  // - when running
121  // - go through each command
122  // - start the command, commandBusy = true
123  // - check for complete, then commandBusy = false
124 
125  // start command handling
126  // define mutex scope
127  {
128  // lockout the messages array for the remainder of the scope
129  // this guarantees the reading thread can safely access the messages
130  if(iterator->theSupervisor_->VERBOSE_MUTEX)
131  __COUT__ << "Waiting for iterator access" << __E__;
132  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
133  if(iterator->theSupervisor_->VERBOSE_MUTEX)
134  __COUT__ << "Have iterator access" << __E__;
135 
136  if(iterator->commandPlay_)
137  {
138  iterator->commandPlay_ = false; // clear
139 
140  if(!iterator->activePlanIsRunning_)
141  {
142  // valid PLAY command!
143 
144  iterator->activePlanIsRunning_ = true;
145  iterator->iteratorBusy_ = true;
146 
147  if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
148  {
149  __COUT__ << "New plan name encountered old="
150  << theIteratorStruct.activePlan_
151  << " vs new=" << iterator->activePlanName_ << __E__;
152  theIteratorStruct.commandIndex_ = -1; // reset for new plan
153  }
154 
155  theIteratorStruct.activePlan_ = iterator->activePlanName_;
156  iterator->lastStartedPlanName_ = iterator->activePlanName_;
157  FILE* fp = fopen((ITERATOR_PLAN_HISTORY_FILENAME).c_str(), "w");
158  if(fp)
159  {
160  fprintf(fp, "%s\n", iterator->lastStartedPlanName_.c_str());
161  fprintf(fp, "%s\n", iterator->lastFinishedPlanName_.c_str());
162  fclose(fp);
163  }
164  else
165  __COUT_WARN__ << "Could not open Iterator history file: "
166  << ITERATOR_PLAN_HISTORY_FILENAME << __E__;
167 
168  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
169  {
170  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_
171  << ".'" << __E__;
172  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_
173  << ".'" << __E__;
174  }
175  else
176  {
177  theIteratorStruct.doResumeAction_ = true;
178  __COUT__ << "Continuing plan '"
179  << theIteratorStruct.activePlan_
180  << "' at command index "
181  << theIteratorStruct.commandIndex_ << ". " << __E__;
182  __COUT__ << "Continuing plan '"
183  << theIteratorStruct.activePlan_
184  << "' at command index "
185  << theIteratorStruct.commandIndex_ << ". " << __E__;
186  }
187  }
188  }
189  else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
190  {
191  theIteratorStruct.doPauseAction_ = true;
192  iterator->commandPause_ = false; // clear
193  }
194  else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
195  {
196  theIteratorStruct.doHaltAction_ = true;
197  iterator->commandHalt_ = false; // clear
198  }
199 
200  theIteratorStruct.running_ = iterator->activePlanIsRunning_;
201 
202  if(iterator
203  ->activeCommandIndex_ != // update active command status if changed
204  theIteratorStruct.commandIndex_)
205  {
206  iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
207  if(theIteratorStruct.commandIndex_ <
208  theIteratorStruct.commands_.size())
209  iterator->activeCommandType_ =
210  theIteratorStruct.commands_[theIteratorStruct.commandIndex_]
211  .type_;
212  else
213  iterator->activeCommandType_ = "";
214  iterator->activeCommandStartTime_ = time(0); // reset on any change
215 
216  if(theIteratorStruct.commandIndex_ <
217  theIteratorStruct.commandIterations_.size())
218  iterator->activeCommandIteration_ =
219  theIteratorStruct
220  .commandIterations_[theIteratorStruct.commandIndex_];
221  else
222  iterator->activeCommandIteration_ = -1;
223 
224  iterator->depthIterationStack_.clear();
225  for(const auto& depthIteration : theIteratorStruct.stepIndexStack_)
226  iterator->depthIterationStack_.push_back(depthIteration);
227  // if(theIteratorStruct.stepIndexStack_.size())
228  // iterator->activeLoopIteration_ =
229  // theIteratorStruct.stepIndexStack_.back(); else
230  // iterator->activeLoopIteration_ = -1;
231 
232  iterator->activeNumberOfCommands_ =
233  theIteratorStruct.commands_.size();
234  }
235 
236  } // end command handling and iterator mutex
237 
240  // do halt or pause action outside of iterator mutex
241 
242  if(theIteratorStruct.doPauseAction_)
243  {
244  // valid PAUSE-iterator command!
245 
246  // safely pause plan!
247  // i.e. check that command is complete
248 
249  __COUT__ << "Waiting to pause..." << __E__;
250  while(!iterator->checkCommand(&theIteratorStruct))
251  __COUT__ << "Waiting to pause..." << __E__;
252 
253  __COUT__ << "Completing pause..." << __E__;
254 
255  theIteratorStruct.doPauseAction_ = false; // clear
256 
257  // lockout the messages array for the remainder of the scope
258  // this guarantees the reading thread can safely access the messages
259  if(iterator->theSupervisor_->VERBOSE_MUTEX)
260  __COUT__ << "Waiting for iterator access" << __E__;
261  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
262  if(iterator->theSupervisor_->VERBOSE_MUTEX)
263  __COUT__ << "Have iterator access" << __E__;
264 
265  iterator->activePlanIsRunning_ = false;
266 
267  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_
268  << "' at command index " << theIteratorStruct.commandIndex_
269  << ". " << __E__;
270  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_
271  << "' at command index " << theIteratorStruct.commandIndex_
272  << ". " << __E__;
273 
274  continue; // resume workloop
275  }
276  else if(theIteratorStruct.doHaltAction_)
277  {
278  // valid HALT-iterator command!
279 
280  // safely end plan!
281  // i.e. check that command is complete
282 
283  __COUT__ << "Waiting to halt..." << __E__;
284  while(!iterator->checkCommand(&theIteratorStruct))
285  __COUT__ << "Waiting to halt..." << __E__;
286 
287  __COUT__ << "Completing halt..." << __E__;
288 
289  theIteratorStruct.doHaltAction_ = false; // clear
290 
291  iterator->haltIterator(iterator, &theIteratorStruct);
292 
293  // //last ditch effort to make sure FSM is halted
294  // iterator->haltIterator(
295  // iterator->theSupervisor_,
296  // theIteratorStruct.fsmName_);
297  //
298  // //lockout the messages array for the remainder of the scope
299  // //this guarantees the reading thread can safely access the
300  // messages
301  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting
302  // for iterator access" << __E__; std::lock_guard<std::mutex>
303  // lock(iterator->accessMutex_);
304  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have
305  // iterator access" << __E__;
306  //
307  // iterator->activePlanIsRunning_ = false;
308  // iterator->iteratorBusy_ = false;
309  //
310  // __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
311  // at command index " << theIteratorStruct.commandIndex_ <<
312  //". " << __E__;
313  // __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
314  // at command index " << theIteratorStruct.commandIndex_ <<
315  //". " << __E__;
316  //
317  // theIteratorStruct.activePlan_ = ""; //clear
318  // theIteratorStruct.commandIndex_ = -1; //clear
319 
320  continue; // resume workloop
321  }
322 
325  // handle running
326  // __COUT__ << "thinking.." << theIteratorStruct.running_ << " " <<
327  // theIteratorStruct.activePlan_ << " cmd=" <<
328  // theIteratorStruct.commandIndex_ << __E__;
329  if(theIteratorStruct.running_ &&
330  theIteratorStruct.activePlan_ !=
331  "") // important, because after errors, still "running" until halt
332  {
333  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
334  {
335  // initialize the running plan
336 
337  __COUT__ << "Get commands" << __E__;
338 
339  theIteratorStruct.commandIndex_ = 0;
340 
341  theIteratorStruct.cfgMgr_
342  ->init(); // completely reset to re-align with any changes
343 
344  if(theIteratorStruct.activePlan_ == Iterator::RESERVED_GEN_PLAN_NAME)
345  {
346  __COUT__ << "Using generated plan..." << __E__;
347  theIteratorStruct.onlyConfigIfNotConfigured_ =
348  iterator->genKeepConfiguration_;
349  theIteratorStruct.commands_ =
350  generateIterationPlan(iterator->genFsmName_,
351  iterator->genConfigAlias_,
352  iterator->genPlanDurationSeconds_,
353  iterator->genPlanNumberOfRuns_);
354  }
355  else
356  {
357  __COUT__ << "Getting iterator table..." << __E__;
358  itConfig =
359  theIteratorStruct.cfgMgr_->__GET_CONFIG__(IterateTable);
360  theIteratorStruct.onlyConfigIfNotConfigured_ = false;
361  theIteratorStruct.commands_ = itConfig->getPlanCommands(
362  theIteratorStruct.cfgMgr_, theIteratorStruct.activePlan_);
363  }
364 
365  // reset commandIteration counts
366  theIteratorStruct.commandIterations_.clear();
367  for(auto& command : theIteratorStruct.commands_)
368  {
369  theIteratorStruct.commandIterations_.push_back(0);
370  __COUT__ << "command " << command.type_ << __E__;
371  __COUT__ << "table "
372  << IterateTable::commandToTableMap_.at(command.type_)
373  << __E__;
374  __COUT__ << "param count = " << command.params_.size() << __E__;
375 
376  for(auto& param : command.params_)
377  {
378  __COUT__ << "\t param " << param.first << " : "
379  << param.second << __E__;
380  }
381  }
382 
383  theIteratorStruct.originalTrackChanges_ =
384  ConfigurationInterface::isVersionTrackingEnabled();
385  theIteratorStruct.originalConfigGroup_ =
386  theIteratorStruct.cfgMgr_->getActiveGroupName();
387  theIteratorStruct.originalConfigKey_ =
388  theIteratorStruct.cfgMgr_->getActiveGroupKey();
389 
390  __COUT__ << "originalTrackChanges "
391  << theIteratorStruct.originalTrackChanges_ << __E__;
392  __COUT__ << "originalConfigGroup "
393  << theIteratorStruct.originalConfigGroup_ << __E__;
394  __COUT__ << "originalConfigKey "
395  << theIteratorStruct.originalConfigKey_ << __E__;
396 
397  } // end initial section
398 
399  if(!theIteratorStruct.commandBusy_)
400  {
401  if(theIteratorStruct.commandIndex_ <
402  theIteratorStruct.commands_.size())
403  {
404  // execute command
405  theIteratorStruct.commandBusy_ = true;
406 
407  __COUT__ << "Iterator starting command "
408  << theIteratorStruct.commandIndex_ + 1 << ": "
409  << theIteratorStruct
410  .commands_[theIteratorStruct.commandIndex_]
411  .type_
412  << __E__;
413  __COUT__ << "Iterator starting command "
414  << theIteratorStruct.commandIndex_ + 1 << ": "
415  << theIteratorStruct
416  .commands_[theIteratorStruct.commandIndex_]
417  .type_
418  << __E__;
419 
420  iterator->startCommand(&theIteratorStruct);
421  }
422  else if(theIteratorStruct.commandIndex_ ==
423  theIteratorStruct.commands_.size()) // Done!
424  {
425  __COUT__ << "Finished Iteration Plan '"
426  << theIteratorStruct.activePlan_ << __E__;
427  __COUT__ << "Finished Iteration Plan '"
428  << theIteratorStruct.activePlan_ << __E__;
429 
430  __COUT__ << "Reverting track changes." << __E__;
431  ConfigurationInterface::setVersionTrackingEnabled(
432  theIteratorStruct.originalTrackChanges_);
433 
434  //l
435  __COUT__ << "Activating original group..." << __E__;
436  try
437  {
438  theIteratorStruct.cfgMgr_->activateTableGroup(
439  theIteratorStruct.originalConfigGroup_,
440  theIteratorStruct.originalConfigKey_);
441  }
442  catch(...)
443  {
444  __COUT_WARN__ << "Original group could not be activated."
445  << __E__;
446  }
447 
448  // leave FSM halted
449  __COUT__ << "Completing Iteration Plan and cleaning up..."
450  << __E__;
451 
452  iterator->haltIterator(
453  iterator, &theIteratorStruct, true /* doNotHaltFSM */);
454  }
455  }
456  else if(theIteratorStruct.commandBusy_)
457  {
458  // check for command completion
459  if(iterator->checkCommand(&theIteratorStruct))
460  {
461  theIteratorStruct.commandBusy_ = false; // command complete
462 
463  ++theIteratorStruct.commandIndex_;
464 
465  __COUT__ << "Ready for next command. Done with "
466  << theIteratorStruct.commandIndex_ << " of "
467  << theIteratorStruct.commands_.size() << __E__;
468  __COUT__ << "Iterator ready for next command. Done with "
469  << theIteratorStruct.commandIndex_ << " of "
470  << theIteratorStruct.commands_.size() << __E__;
471  }
472 
473  // Note: check command gets one shot to resume
474  if(theIteratorStruct.doResumeAction_) // end resume action
475  theIteratorStruct.doResumeAction_ = false;
476  }
477 
478  } // end running
479  else
480  sleep(1); // when inactive sleep a lot
481 
484 
485  // __COUT__ << "end loop.." << theIteratorStruct.running_ << " " <<
486  // theIteratorStruct.activePlan_ << " cmd=" <<
487  // theIteratorStruct.commandIndex_ << __E__;
488  } /* code */
489  } //end try/catch
490  catch(const std::runtime_error& e) //insert info about the state of the iterator
491  {
492  if(theIteratorStruct.activePlan_ != "")
493  {
494  __SS__ << "The active Iterator plan name is '"
495  << theIteratorStruct.activePlan_
496  << "'... Here was the error: " << e.what() << __E__;
497  __SS_THROW__;
498  }
499  else
500  throw;
501  }
502 
503  iterator->workloopRunning_ = false; // if we ever exit
504 } //end IteratorWorkLoop()
505 catch(const std::runtime_error& e)
506 {
507  __SS__ << "Encountered error in Iterator thread. Here is the error:\n"
508  << e.what() << __E__;
509  __COUT_ERR__ << ss.str();
510 
511  // lockout the messages array for the remainder of the scope
512  // this guarantees the reading thread can safely access the messages
513  if(iterator->theSupervisor_->VERBOSE_MUTEX)
514  __COUT__ << "Waiting for iterator access" << __E__;
515  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
516  if(iterator->theSupervisor_->VERBOSE_MUTEX)
517  __COUT__ << "Have iterator access" << __E__;
518 
519  iterator->workloopRunning_ = false; // if we ever exit
520  iterator->errorMessage_ = ss.str();
521 }
522 catch(...)
523 {
524  __SS__ << "Encountered unknown error in Iterator thread." << __E__;
525  try
526  {
527  throw;
528  } //one more try to printout extra info
529  catch(const std::exception& e)
530  {
531  ss << "Exception message: " << e.what();
532  }
533  catch(...)
534  {
535  }
536  __COUT_ERR__ << ss.str();
537 
538  // lockout the messages array for the remainder of the scope
539  // this guarantees the reading thread can safely access the messages
540  if(iterator->theSupervisor_->VERBOSE_MUTEX)
541  __COUT__ << "Waiting for iterator access" << __E__;
542  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
543  if(iterator->theSupervisor_->VERBOSE_MUTEX)
544  __COUT__ << "Have iterator access" << __E__;
545 
546  iterator->workloopRunning_ = false; // if we ever exit
547  iterator->errorMessage_ = ss.str();
548 } // end IteratorWorkLoop() exception handling
549 
550 //==============================================================================
551 void Iterator::startCommand(IteratorWorkLoopStruct* iteratorStruct)
552 try
553 {
554  {
555  int i = 0;
556  for(const auto& depthIteration : iteratorStruct->stepIndexStack_)
557  {
558  __COUT__ << i++ << ":" << depthIteration << __E__;
559  }
560  }
561 
562  // should be mutually exclusive with GatewaySupervisor main thread state machine
563  // accesses lockout the messages array for the remainder of the scope this
564  // guarantees the reading thread can safely access the messages
565  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
566  __COUT__ << "Waiting for FSM access" << __E__;
567  std::lock_guard<std::mutex> lock(
568  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
569  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
570  __COUT__ << "Have FSM access" << __E__;
571 
572  // for out of range, throw exception - should never happen
573  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
574  {
575  __SS__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
576  << " in size = " << iteratorStruct->commands_.size() << __E__;
577  __SS_THROW__;
578  }
579 
580  // increment iteration count for command
581  ++iteratorStruct->commandIterations_[iteratorStruct->commandIndex_];
582 
583  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
584  if(type == IterateTable::COMMAND_BEGIN_LABEL)
585  {
586  return startCommandBeginLabel(iteratorStruct);
587  }
588  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
589  {
590  return startCommandChooseFSM(
591  iteratorStruct,
592  iteratorStruct->commands_[iteratorStruct->commandIndex_]
593  .params_[IterateTable::commandChooseFSMParams_.NameOfFSM_]);
594  }
595  else if(type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP)
596  {
597  return startCommandConfigureActive(iteratorStruct);
598  }
599  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS)
600  {
601  return startCommandConfigureAlias(
602  iteratorStruct,
603  iteratorStruct->commands_[iteratorStruct->commandIndex_]
604  .params_[IterateTable::commandConfigureAliasParams_.SystemAlias_]);
605  }
606  else if(type == IterateTable::COMMAND_CONFIGURE_GROUP)
607  {
608  return startCommandConfigureGroup(iteratorStruct);
609  }
610  else if(type == IterateTable::COMMAND_ACTIVATE_ALIAS)
611  {
612  ConfigurationManagerRW* cfgMgr = iteratorStruct->cfgMgr_;
613  std::pair<std::string, TableGroupKey> newActiveGroup =
614  cfgMgr->getTableGroupFromAlias(
615  iteratorStruct->commands_[iteratorStruct->commandIndex_]
616  .params_[IterateTable::commandActivateAliasParams_.SystemAlias_]);
617  cfgMgr->loadTableGroup(
618  newActiveGroup.first, newActiveGroup.second, true /*activate*/);
619  }
620  else if(type == IterateTable::COMMAND_ACTIVATE_GROUP)
621  {
622  ConfigurationManagerRW* cfgMgr = iteratorStruct->cfgMgr_;
623  cfgMgr->loadTableGroup(
624  iteratorStruct->commands_[iteratorStruct->commandIndex_]
625  .params_[IterateTable::commandActivateGroupParams_.GroupName_],
626  TableGroupKey(
627  iteratorStruct->commands_[iteratorStruct->commandIndex_]
628  .params_[IterateTable::commandActivateGroupParams_.GroupKey_]),
629  true /*activate*/);
630  }
631  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
632  {
633  return startCommandMacro(iteratorStruct, true /*isFEMacro*/);
634  }
635  else if(type == IterateTable::COMMAND_EXECUTE_MACRO)
636  {
637  return startCommandMacro(iteratorStruct, false /*isFEMacro*/);
638  }
639  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
640  {
641  return startCommandModifyActive(iteratorStruct);
642  }
643  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
644  {
645  return startCommandRepeatLabel(iteratorStruct);
646  }
647  else if(type == IterateTable::COMMAND_RUN)
648  {
649  return startCommandRun(iteratorStruct);
650  }
651  else if(type == IterateTable::COMMAND_WAIT)
652  {
653  return startCommandWait(iteratorStruct);
654  }
655  else if(type == IterateTable::COMMAND_START)
656  {
657  return startCommandFSMTransition(iteratorStruct,
658  RunControlStateMachine::START_TRANSITION_NAME);
659  }
660  else if(type == IterateTable::COMMAND_STOP)
661  {
662  return startCommandFSMTransition(iteratorStruct,
663  RunControlStateMachine::STOP_TRANSITION_NAME);
664  }
665  else if(type == IterateTable::COMMAND_PAUSE)
666  {
667  return startCommandFSMTransition(iteratorStruct,
668  RunControlStateMachine::PAUSE_TRANSITION_NAME);
669  }
670  else if(type == IterateTable::COMMAND_RESUME)
671  {
672  return startCommandFSMTransition(iteratorStruct,
673  RunControlStateMachine::RESUME_TRANSITION_NAME);
674  }
675  else if(type == IterateTable::COMMAND_HALT)
676  {
677  return startCommandFSMTransition(iteratorStruct,
678  RunControlStateMachine::HALT_TRANSITION_NAME);
679  }
680  else
681  {
682  __SS__ << "Failed attempt to start unrecognized command type = " << type << __E__;
683  __COUT_ERR__ << ss.str();
684  __SS_THROW__;
685  }
686 }
687 catch(...)
688 {
689  __COUT__ << "Error caught. Reverting track changes." << __E__;
690  ConfigurationInterface::setVersionTrackingEnabled(
691  iteratorStruct->originalTrackChanges_);
692 
693  __COUT__ << "Activating original group..." << __E__;
694  try
695  {
696  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
697  iteratorStruct->originalConfigKey_);
698  }
699  catch(...)
700  {
701  __COUT_WARN__ << "Original group could not be activated." << __E__;
702  }
703  throw;
704 } // end startCommand()
705 
706 //==============================================================================
710 bool Iterator::checkCommand(IteratorWorkLoopStruct* iteratorStruct)
711 try
712 {
713  // for out of range, return done
714  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
715  {
716  __COUT__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
717  << " in size = " << iteratorStruct->commands_.size() << __E__;
718  return true;
719  }
720 
721  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
722  if(type == IterateTable::COMMAND_BEGIN_LABEL)
723  {
724  // do nothing
725  return true;
726  }
727  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
728  {
729  // do nothing
730  return true;
731  }
732  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS ||
733  type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP ||
734  type == IterateTable::COMMAND_CONFIGURE_GROUP)
735  {
736  return checkCommandConfigure(iteratorStruct);
737  }
738  else if(type == IterateTable::COMMAND_ACTIVATE_ALIAS ||
739  type == IterateTable::COMMAND_ACTIVATE_GROUP)
740  {
741  // do nothing
742  return true;
743  }
744  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
745  {
746  return checkCommandMacro(iteratorStruct, true /*isFEMacro*/);
747  }
748  else if(type == IterateTable::COMMAND_EXECUTE_MACRO)
749  {
750  return checkCommandMacro(iteratorStruct, false /*isFEMacro*/);
751  }
752  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
753  {
754  // do nothing
755  return true;
756  }
757  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
758  {
759  // do nothing
760  return true;
761  }
762  else if(type == IterateTable::COMMAND_RUN)
763  {
764  return checkCommandRun(iteratorStruct);
765  }
766  else if(type == IterateTable::COMMAND_WAIT)
767  {
768  return checkCommandWait(iteratorStruct);
769  }
770  else if(type == IterateTable::COMMAND_START)
771  {
772  return checkCommandFSMTransition(iteratorStruct, "Running");
773  }
774  else if(type == IterateTable::COMMAND_STOP)
775  {
776  return checkCommandFSMTransition(iteratorStruct, "Configured");
777  }
778  else if(type == IterateTable::COMMAND_PAUSE)
779  {
780  return checkCommandFSMTransition(iteratorStruct, "Paused");
781  }
782  else if(type == IterateTable::COMMAND_RESUME)
783  {
784  return checkCommandFSMTransition(iteratorStruct, "Running");
785  }
786  else if(type == IterateTable::COMMAND_HALT)
787  {
788  return checkCommandFSMTransition(iteratorStruct,
789  RunControlStateMachine::HALTED_STATE_NAME);
790  }
791  else
792  {
793  __SS__ << "Attempt to check unrecognized command type = " << type << __E__;
794  __COUT_ERR__ << ss.str();
795  __SS_THROW__;
796  }
797 }
798 catch(...)
799 {
800  __COUT__ << "Error caught. Reverting track changes." << __E__;
801  ConfigurationInterface::setVersionTrackingEnabled(
802  iteratorStruct->originalTrackChanges_);
803 
804  __COUT__ << "Activating original group..." << __E__;
805  try
806  {
807  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
808  iteratorStruct->originalConfigKey_);
809  }
810  catch(...)
811  {
812  __COUT_WARN__ << "Original group could not be activated." << __E__;
813  }
814 
815  throw;
816 } // end checkCommand()
817 
818 //==============================================================================
819 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct* iteratorStruct,
820  const std::string& fsmName)
821 {
822  __COUT__ << "fsmName " << fsmName << __E__;
823 
824  iteratorStruct->fsmName_ = fsmName;
825  iteratorStruct->theIterator_->lastFsmName_ = fsmName;
826 
827  // Translate fsmName
828  // to gives us run alias (fsmRunAlias_) and next run number (fsmNextRunNumber_)
829 
830  // CAREFUL?? Threads
831 
833 
834  iteratorStruct->fsmRunAlias_ = "Run"; // default to "Run"
835 
836  // get stateMachineAliasFilter if possible
837  ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorTableNode(
838  iteratorStruct->theIterator_->theSupervisor_->getContextUID(),
839  iteratorStruct->theIterator_->theSupervisor_->getSupervisorUID());
840 
841  if(!configLinkNode.isDisconnected())
842  {
843  try // for backwards compatibility
844  {
845  ConfigurationTree fsmLinkNode =
846  configLinkNode.getNode("LinkToStateMachineTable");
847  if(!fsmLinkNode.isDisconnected())
848  iteratorStruct->fsmRunAlias_ =
849  fsmLinkNode.getNode(fsmName + "/RunDisplayAlias")
850  .getValue<std::string>();
851  else
852  __COUT_INFO__ << "FSM Link disconnected." << __E__;
853  }
854  catch(std::runtime_error& e)
855  {
856  //__COUT_INFO__ << e.what() << __E__;
857  __COUT_INFO__
858  << "No state machine Run alias. Ignoring and assuming alias of '"
859  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
860  }
861  catch(...)
862  {
863  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
864 
865  __COUT_INFO__
866  << "No state machine Run alias. Ignoring and assuming alias of '"
867  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
868  }
869  }
870  else
871  __COUT_INFO__ << "FSM Link disconnected." << __E__;
872 
873  __COUT__ << "fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
874 
876 
877  iteratorStruct->fsmNextRunNumber_ =
878  iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
879  iteratorStruct->fsmName_);
880 
881  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
882  .getCurrentStateName() == "Running" ||
883  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
884  .getCurrentStateName() == "Paused")
885  --iteratorStruct->fsmNextRunNumber_; // current run number is one back
886 
887  __COUT__ << "fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
888 } // end startCommandChooseFSM()
889 
890 //==============================================================================
892 bool Iterator::haltIterator(Iterator* iterator,
893  IteratorWorkLoopStruct* iteratorStruct /* = 0 */,
894  bool doNotHaltFSM /* = false */)
895 
896 {
897  GatewaySupervisor* theSupervisor = iterator->theSupervisor_;
898  const std::string& fsmName = iterator->lastFsmName_;
899 
900  std::vector<std::string> fsmCommandParameters;
901  std::string errorStr = "";
902  std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
903 
904  __COUTV__(currentState);
905 
906  bool haltAttempted = true;
907  if(doNotHaltFSM)
908  {
909  __COUT_INFO__ << "Iterator is leaving FSM in current state: " << currentState
910  << ". If this is undesireable, add a Halt command, for example, to "
911  "the end of your Iteration plan."
912  << __E__;
913  haltAttempted = false;
914  }
915  else if(currentState == RunControlStateMachine::INITIAL_STATE_NAME ||
916  currentState == RunControlStateMachine::HALTED_STATE_NAME)
917  {
918  __COUT__ << "Do nothing. Already halted." << __E__;
919  haltAttempted = false;
920  }
921  else if(currentState == "Running")
922  errorStr = theSupervisor->attemptStateMachineTransition(
923  0,
924  0,
925  RunControlStateMachine::ABORT_TRANSITION_NAME,
926  fsmName,
927  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
928  WebUsers::DEFAULT_ITERATOR_USERNAME,
929  fsmCommandParameters);
930  else
931  errorStr = theSupervisor->attemptStateMachineTransition(
932  0,
933  0,
934  RunControlStateMachine::HALT_TRANSITION_NAME,
935  fsmName,
936  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
937  WebUsers::DEFAULT_ITERATOR_USERNAME,
938  fsmCommandParameters);
939 
940  if(haltAttempted)
941  {
942  if(errorStr != "")
943  {
944  __SS__ << "Iterator failed to halt because of the following error: "
945  << errorStr;
946  __SS_THROW__;
947  }
948 
949  // else successfully launched
950  __COUT__ << "FSM in transition = "
951  << theSupervisor->theStateMachine_.isInTransition() << __E__;
952  __COUT__ << "halting state machine launched." << __E__;
953  }
954 
955  // finish up cleanup of the iterator
956  __COUT__ << "Conducting Iterator cleanup." << __E__;
957 
958  if(iteratorStruct)
959  {
960  __COUT__ << "Reverting track changes." << __E__;
961  ConfigurationInterface::setVersionTrackingEnabled(
962  iteratorStruct->originalTrackChanges_);
963 
964  if(!doNotHaltFSM)
965  {
966  __COUT__ << "Activating original group..." << __E__;
967  try
968  {
969  iteratorStruct->cfgMgr_->activateTableGroup(
970  iteratorStruct->originalConfigGroup_,
971  iteratorStruct->originalConfigKey_);
972  }
973  catch(...)
974  {
975  __COUT_WARN__ << "Original group could not be activated." << __E__;
976  }
977  }
978  }
979 
980  // lockout the messages array for the remainder of the scope
981  // this guarantees the reading thread can safely access the messages
982  if(iterator->theSupervisor_->VERBOSE_MUTEX)
983  __COUT__ << "Waiting for iterator access" << __E__;
984  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
985  if(iterator->theSupervisor_->VERBOSE_MUTEX)
986  __COUT__ << "Have iterator access" << __E__;
987 
988  iterator->activePlanIsRunning_ = false;
989  iterator->iteratorBusy_ = false;
990 
991  // clear
992  iterator->activePlanName_ = "";
993  iterator->activeCommandIndex_ = -1;
994 
995  if(iteratorStruct)
996  {
997  __COUT__ << "Iterator cleanup complete of plan '" << iteratorStruct->activePlan_
998  << "' at command index " << iteratorStruct->commandIndex_ << ". "
999  << __E__;
1000 
1001  iterator->lastFinishedPlanName_ = iteratorStruct->activePlan_;
1002  FILE* fp = fopen((ITERATOR_PLAN_HISTORY_FILENAME).c_str(), "w");
1003  if(fp)
1004  {
1005  fprintf(fp, "%s\n", iterator->lastStartedPlanName_.c_str());
1006  fprintf(fp, "%s\n", iterator->lastFinishedPlanName_.c_str());
1007  fclose(fp);
1008  }
1009  else
1010  __COUT_WARN__ << "Could not open Iterator history file: "
1011  << ITERATOR_PLAN_HISTORY_FILENAME << __E__;
1012 
1013  iteratorStruct->activePlan_ = ""; // clear
1014  iteratorStruct->commandIndex_ = -1; // clear
1015  }
1016 
1017  return haltAttempted;
1018 } // end haltIterator()
1019 
1020 //==============================================================================
1021 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct* iteratorStruct)
1022 {
1023  __COUT__ << "Entering label '"
1024  << iteratorStruct->commands_[iteratorStruct->commandIndex_]
1025  .params_[IterateTable::commandBeginLabelParams_.Label_]
1026  << "'..." << std::endl;
1027 
1028  // add new step index to stack
1029  iteratorStruct->stepIndexStack_.push_back(0);
1030 } // end startCommandBeginLabel()
1031 
1032 //==============================================================================
1033 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct* iteratorStruct)
1034 {
1035  // search for first matching label backward and set command to there
1036 
1037  int numOfRepetitions;
1038  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1039  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_]
1040  .c_str(),
1041  "%d",
1042  &numOfRepetitions);
1043  __COUT__ << "numOfRepetitions remaining = " << numOfRepetitions << __E__;
1044 
1045  char repStr[200];
1046 
1047  if(numOfRepetitions <= 0)
1048  {
1049  // write original number of repetitions value back
1050  sprintf(repStr, "%d", iteratorStruct->stepIndexStack_.back());
1051  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1052  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
1053  repStr; // re-store as string
1054 
1055  // remove step index from stack
1056  iteratorStruct->stepIndexStack_.pop_back();
1057 
1058  return; // no more repetitions
1059  }
1060 
1061  --numOfRepetitions;
1062 
1063  // increment step index in stack
1064  ++(iteratorStruct->stepIndexStack_.back());
1065 
1066  unsigned int i;
1067  for(i = iteratorStruct->commandIndex_; i > 0;
1068  --i) // assume 0 is always the fallback option
1069  if(iteratorStruct->commands_[i].type_ == IterateTable::COMMAND_BEGIN_LABEL &&
1070  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1071  .params_[IterateTable::commandRepeatLabelParams_.Label_] ==
1072  iteratorStruct->commands_[i]
1073  .params_[IterateTable::commandBeginLabelParams_.Label_])
1074  break;
1075 
1076  sprintf(repStr, "%d", numOfRepetitions);
1077  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1078  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
1079  repStr; // re-store as string
1080 
1081  iteratorStruct->commandIndex_ = i;
1082  __COUT__ << "Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
1083 } // end startCommandRepeatLabel()
1084 
1085 //==============================================================================
1086 void Iterator::startCommandRun(IteratorWorkLoopStruct* iteratorStruct)
1087 {
1088  __COUT__ << "startCommandRun " << __E__;
1089 
1090  iteratorStruct->runIsDone_ = false;
1091  iteratorStruct->fsmCommandParameters_.clear();
1092 
1093  std::string errorStr = "";
1094  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1095  ->theStateMachine_.getCurrentStateName();
1096 
1097  // execute first transition (may need two)
1098 
1099  __COUTTV__(iteratorStruct->theIterator_->genLogEntry_);
1100 
1101  if(currentState == RunControlStateMachine::CONFIGURED_STATE_NAME)
1102  errorStr =
1103  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1104  0,
1105  0,
1106  RunControlStateMachine::START_TRANSITION_NAME,
1107  iteratorStruct->fsmName_,
1108  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1109  WebUsers::DEFAULT_ITERATOR_USERNAME,
1110  iteratorStruct->fsmCommandParameters_,
1111  iteratorStruct->activePlan_ == Iterator::RESERVED_GEN_PLAN_NAME
1112  ? iteratorStruct->theIterator_->genLogEntry_
1113  : "");
1114  else
1115  errorStr = "Can only Run from the Configured state. The current state is " +
1116  currentState;
1117 
1118  if(errorStr != "")
1119  {
1120  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1121  __SS_THROW__;
1122  }
1123 
1124  // save original duration
1125  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1126  .params_[IterateTable::commandRunParams_.DurationInSeconds_]
1127  .c_str(),
1128  "%ld",
1129  &iteratorStruct->originalDurationInSeconds_);
1130  __COUTV__(iteratorStruct->originalDurationInSeconds_);
1131 
1132  // else successfully launched
1133  __COUT__
1134  << "FSM in transition = "
1135  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1136  << __E__;
1137  __COUT__ << "startCommandRun success." << __E__;
1138 } // end startCommandRun()
1139 
1140 //==============================================================================
1141 void Iterator::startCommandWait(IteratorWorkLoopStruct* iteratorStruct)
1142 {
1143  __COUT__ << "startCommandWait " << __E__;
1144 
1145  iteratorStruct->waitIsDone_ = false;
1146 
1147  // Get original duration
1148  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1149  .params_[IterateTable::commandWaitParams_.DurationInSeconds_]
1150  .c_str(),
1151  "%ld",
1152  &iteratorStruct->originalDurationInSeconds_);
1153  __COUTV__(iteratorStruct->originalDurationInSeconds_);
1154 
1155  __COUT__ << "startCommandWait success." << __E__;
1156 }
1157 
1158 //==============================================================================
1159 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct* iteratorStruct)
1160 {
1161  __COUT__ << "startCommandConfigureActive " << __E__;
1162 
1163  // steps:
1164  // get active config group
1165  // transition to configure with parameters describing group
1166 
1167  std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
1168  TableGroupKey key = iteratorStruct->cfgMgr_->getActiveGroupKey();
1169 
1170  __COUT__ << "group " << group << __E__;
1171  __COUT__ << "key " << key << __E__;
1172 
1173  // create special alias for this group using : separators
1174 
1175  std::stringstream systemAlias;
1176  systemAlias << "GROUP:" << group << ":" << key;
1177  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
1178 } // end startCommandConfigureActive()
1179 
1180 //==============================================================================
1181 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct* iteratorStruct)
1182 {
1183  __COUT__ << "startCommandConfigureGroup " << __E__;
1184 
1185  // steps:
1186  // transition to configure with parameters describing group
1187 
1188  std::string group =
1189  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1190  .params_[IterateTable::commandConfigureGroupParams_.GroupName_];
1191  TableGroupKey key =
1192  TableGroupKey(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1193  .params_[IterateTable::commandConfigureGroupParams_.GroupKey_]);
1194 
1195  __COUT__ << "group " << group << __E__;
1196  __COUT__ << "key " << key << __E__;
1197 
1198  // create special alias for this group using : separators
1199 
1200  std::stringstream systemAlias;
1201  systemAlias << "GROUP:" << group << ":" << key;
1202  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
1203 } // end startCommandConfigureGroup()
1204 
1205 //==============================================================================
1206 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct* iteratorStruct,
1207  const std::string& systemAlias)
1208 {
1209  __COUT__ << "systemAlias " << systemAlias << __E__;
1210 
1211  iteratorStruct->fsmCommandParameters_.clear();
1212  iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
1213 
1214  std::string errorStr = "";
1215  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1216  ->theStateMachine_.getCurrentStateName();
1217 
1218  // execute first transition (may need two in conjunction with checkCommandConfigure())
1219 
1220  if(currentState == RunControlStateMachine::INITIAL_STATE_NAME)
1221  errorStr =
1222  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1223  0,
1224  0,
1225  RunControlStateMachine::INIT_TRANSITION_NAME,
1226  iteratorStruct->fsmName_,
1227  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1228  WebUsers::DEFAULT_ITERATOR_USERNAME,
1229  iteratorStruct->fsmCommandParameters_);
1230  else if(currentState == RunControlStateMachine::HALTED_STATE_NAME)
1231  errorStr =
1232  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1233  0,
1234  0,
1235  RunControlStateMachine::CONFIGURE_TRANSITION_NAME,
1236  iteratorStruct->fsmName_,
1237  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1238  WebUsers::DEFAULT_ITERATOR_USERNAME,
1239  iteratorStruct->fsmCommandParameters_);
1240  else if(currentState == RunControlStateMachine::CONFIGURED_STATE_NAME ||
1241  currentState == RunControlStateMachine::FAILED_STATE_NAME)
1242  {
1243  if(iteratorStruct->onlyConfigIfNotConfigured_ &&
1244  currentState != RunControlStateMachine::FAILED_STATE_NAME)
1245  __COUT__ << "Already configured, so do nothing!" << __E__;
1246  else
1247  errorStr = iteratorStruct->theIterator_->theSupervisor_
1248  ->attemptStateMachineTransition(
1249  0,
1250  0,
1251  RunControlStateMachine::HALT_TRANSITION_NAME,
1252  iteratorStruct->fsmName_,
1253  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1254  WebUsers::DEFAULT_ITERATOR_USERNAME,
1255  iteratorStruct->fsmCommandParameters_);
1256  }
1257  else
1258  errorStr =
1259  "Can only Configure from the Initial or Halted state. The current state is " +
1260  currentState;
1261 
1262  if(errorStr != "")
1263  {
1264  __SS__ << "Iterator failed to configure with system alias '"
1265  << (iteratorStruct->fsmCommandParameters_.size()
1266  ? iteratorStruct->fsmCommandParameters_[0]
1267  : "UNKNOWN")
1268  << "' because of the following error: " << errorStr;
1269  __SS_THROW__;
1270  }
1271 
1272  // else successfully launched
1273  __COUT__
1274  << "FSM in transition = "
1275  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1276  << __E__;
1277  __COUT__ << "startCommandConfigureAlias success." << __E__;
1278 } // end startCommandConfigureAlias()
1279 
1280 //==============================================================================
1281 void Iterator::startCommandFSMTransition(IteratorWorkLoopStruct* iteratorStruct,
1282  const std::string& transitionCommand)
1283 {
1284  __COUTV__(transitionCommand);
1285 
1286  iteratorStruct->fsmCommandParameters_.clear();
1287 
1288  std::string errorStr = "";
1289  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1290  ->theStateMachine_.getCurrentStateName();
1291 
1292  // execute first transition (may need two in conjunction with checkCommandConfigure())
1293 
1294  __COUTV__(currentState);
1295 
1296  errorStr =
1297  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1298  0,
1299  0,
1300  transitionCommand,
1301  iteratorStruct->fsmName_,
1302  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1303  WebUsers::DEFAULT_ITERATOR_USERNAME,
1304  iteratorStruct->fsmCommandParameters_,
1305  (transitionCommand == RunControlStateMachine::START_TRANSITION_NAME &&
1306  iteratorStruct->activePlan_ == Iterator::RESERVED_GEN_PLAN_NAME)
1307  ? iteratorStruct->theIterator_->genLogEntry_
1308  : "");
1309 
1310  if(errorStr != "")
1311  {
1312  __SS__ << "Iterator failed to " << transitionCommand << " with ";
1313  if(iteratorStruct->fsmCommandParameters_.size() == 0)
1314  ss << "no parameters ";
1315  else
1316  ss << "parameters '"
1317  << StringMacros::vectorToString(iteratorStruct->fsmCommandParameters_)
1318  << "' ";
1319  ss << "' because of the following error: " << errorStr;
1320  __SS_THROW__;
1321  }
1322 
1323  // else successfully launched
1324  __COUT__
1325  << "FSM in transition = "
1326  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1327  << __E__;
1328  __COUT__ << "startCommandFSMTransition success." << __E__;
1329 } // end startCommandFSMTransition()
1330 
1331 //==============================================================================
1332 void Iterator::startCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1333  bool isFrontEndMacro)
1334 {
1335  // Steps:
1336  // 4 parameters CommandExecuteFEMacroParams:
1337  // //targets
1338  // const std::string FEMacroName_ = "FEMacroName";
1339  // //macro parameters (table/groupID)
1340 
1341  const std::string& macroName =
1342  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1343  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1344  const std::string& enableSavingOutput =
1345  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1346  .params_[IterateTable::commandExecuteMacroParams_.EnableSavingOutput_];
1347  const std::string& outputFilePath =
1348  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1349  .params_[IterateTable::commandExecuteMacroParams_.OutputFilePath_];
1350  const std::string& outputFileRadix =
1351  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1352  .params_[IterateTable::commandExecuteMacroParams_.OutputFileRadix_];
1353  const std::string& inputArgs =
1354  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1355  .params_[IterateTable::commandExecuteMacroParams_.MacroArgumentString_];
1356 
1357  __COUTV__(macroName);
1358  __COUTV__(enableSavingOutput);
1359  __COUTV__(outputFilePath);
1360  __COUTV__(outputFileRadix);
1361  __COUTV__(inputArgs);
1362 
1363  // send request to MacroMaker a la FEVInterface::runFrontEndMacro
1364  // but need to pass iteration information, so that the call is launched by just one
1365  // send to each front end. Front-ends must immediately respond that is started
1366  // FEVInterfacesManager.. must start a thread for running the macro iterations
1367  // Then check for complete.
1368 
1369  iteratorStruct->targetsDone_.clear(); // reset
1370 
1371  __COUTV__(iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size());
1372  for(const auto& target :
1373  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_)
1374  {
1375  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1376 
1377  // for each target, init to not done
1378  iteratorStruct->targetsDone_.push_back(false);
1379 
1380  xoap::MessageReference message =
1381  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1382 
1383  SOAPParameters parameters;
1384  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalStart"
1385  : "macroMultiDimensionalStart";
1386  parameters.addParameter("type", type);
1387  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1388  parameters.addParameter("targetInterfaceID", target.UID_);
1389  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1390  parameters.addParameter("enableSavingOutput", enableSavingOutput);
1391  parameters.addParameter("outputFilePath", outputFilePath);
1392  parameters.addParameter("outputFileRadix", outputFileRadix);
1393  parameters.addParameter("inputArgs", inputArgs);
1394  SOAPUtilities::addParameters(message, parameters);
1395 
1396  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1397  << __E__;
1398 
1399  xoap::MessageReference replyMessage =
1400  iteratorStruct->theIterator_->theSupervisor_
1401  ->SOAPMessenger::sendWithSOAPReply(
1402  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1403  .getAllMacroMakerTypeSupervisorInfo()
1404  .begin()
1405  ->second.getDescriptor(),
1406  message);
1407 
1408  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1409  << __E__;
1410 
1411  SOAPParameters rxParameters;
1412  rxParameters.addParameter("Error");
1413  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1414 
1415  std::string error = rxParameters.getValue("Error");
1416 
1417  if(response != type + "Done" || error != "")
1418  {
1419  // error occurred!
1420  __SS__ << "Error transmitting request to target interface '" << target.UID_
1421  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1422  << response << "' with error: " << error << __E__;
1423  __SS_THROW__;
1424  }
1425  } // end target loop
1426 
1427 } // end startCommandMacro()
1428 
1429 //==============================================================================
1430 bool Iterator::checkCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1431  bool isFrontEndMacro)
1432 {
1433  sleep(1);
1434 
1435  // Steps:
1436  // 4 parameters CommandExecuteFEMacroParams:
1437  // //targets
1438  // const std::string FEMacroName_ = "FEMacroName";
1439  // //macro parameters (table/groupID)
1440 
1441  const std::string& macroName =
1442  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1443  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1444 
1445  __COUTV__(macroName);
1446 
1447  // send request to MacroMaker to check completion of macro
1448  // as targets are identified complete, remove targets_ from vector
1449 
1450  for(unsigned int i = 0;
1451  i < iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size();
1452  ++i)
1453  {
1454  ots::IterateTable::CommandTarget& target =
1455  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_[i];
1456 
1457  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1458 
1459  xoap::MessageReference message =
1460  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1461 
1462  SOAPParameters parameters;
1463  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalCheck"
1464  : "macroMultiDimensionalCheck";
1465  parameters.addParameter("type", type);
1466  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1467  parameters.addParameter("targetInterfaceID", target.UID_);
1468  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1469  SOAPUtilities::addParameters(message, parameters);
1470 
1471  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1472  << __E__;
1473 
1474  xoap::MessageReference replyMessage =
1475  iteratorStruct->theIterator_->theSupervisor_
1476  ->SOAPMessenger::sendWithSOAPReply(
1477  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1478  .getAllMacroMakerTypeSupervisorInfo()
1479  .begin()
1480  ->second.getDescriptor(),
1481  message);
1482 
1483  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1484  << __E__;
1485 
1486  SOAPParameters rxParameters;
1487  rxParameters.addParameter("Error");
1488  rxParameters.addParameter("Done");
1489  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1490 
1491  std::string error = rxParameters.getValue("Error");
1492  bool done = rxParameters.getValue("Done") == "1";
1493 
1494  if(response != type + "Done" || error != "")
1495  {
1496  // error occurred!
1497  __SS__ << "Error transmitting request to target interface '" << target.UID_
1498  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1499  << response << "' with error: " << error << __E__;
1500  __SS_THROW__;
1501  }
1502 
1503  if(!done) // still more to do so give up checking
1504  return false;
1505 
1506  // mark target done
1507  iteratorStruct->targetsDone_[i] = true;
1508 
1509  // iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.erase(
1510  // targetIt--); //go back after delete
1511 
1512  } // end target loop
1513 
1514  // if here all targets are done
1515  return true;
1516 } // end checkCommandMacro()
1517 
1518 //==============================================================================
1519 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct* iteratorStruct)
1520 {
1521  // Steps:
1522  // 4 parameters commandModifyActiveParams_:
1523  // const std::string DoTrackGroupChanges_ TrueFalse
1524  // //targets
1525  // const std::string RelativePathToField_ = "RelativePathToField";
1526  // const std::string FieldStartValue_ = "FieldStartValue";
1527  // const std::string FieldIterationStepSize_ = "FieldIterationStepSize";
1528  //
1529  // if tracking changes,
1530  // create a new group
1531  // for every enabled FE
1532  // set field = start value + stepSize * currentStepIndex_
1533  // activate group
1534  // else
1535  // load scratch group
1536  // for every enabled FE
1537  // set field = start value + stepSize * stepIndex
1538  // activate group
1539 
1540  bool doTrackGroupChanges = false;
1541  if("True" ==
1542  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1543  .params_[IterateTable::commandModifyActiveParams_.DoTrackGroupChanges_])
1544  doTrackGroupChanges = true;
1545 
1546  const std::string& startValueStr =
1547  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1548  .params_[IterateTable::commandModifyActiveParams_.FieldStartValue_];
1549  const std::string& stepSizeStr =
1550  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1551  .params_[IterateTable::commandModifyActiveParams_.FieldIterationStepSize_];
1552 
1553  const unsigned int stepIndex = iteratorStruct->stepIndexStack_.back();
1554 
1555  __COUT__ << "doTrackGroupChanges " << (doTrackGroupChanges ? "yes" : "no")
1556  << std::endl;
1557  __COUTV__(startValueStr);
1558  __COUTV__(stepSizeStr);
1559  __COUTV__(stepIndex);
1560 
1561  ConfigurationInterface::setVersionTrackingEnabled(doTrackGroupChanges);
1562 
1563  // two approaches: double or long handling
1564  // OR TODO -- if step is 0 and startValue is NaN.. then handle as string
1565 
1566  if(((stepSizeStr.size() && stepSizeStr[0] == '0') || !stepSizeStr.size()) &&
1567  !StringMacros::isNumber(
1568  startValueStr)) //if not a number and no step size, then interpret as string
1569  {
1570  __COUT__ << "Treating start value as string: " << startValueStr << __E__;
1571  helpCommandModifyActive(iteratorStruct, startValueStr, doTrackGroupChanges);
1572  }
1573  else if(startValueStr.size() && (startValueStr[startValueStr.size() - 1] == 'f' ||
1574  startValueStr.find('.') != std::string::npos))
1575  {
1576  // handle as double
1577  double startValue = strtod(startValueStr.c_str(), 0);
1578  double stepSize = strtod(stepSizeStr.c_str(), 0);
1579 
1580  __COUT__ << "startValue " << startValue << std::endl;
1581  __COUT__ << "stepSize " << stepSize << std::endl;
1582  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1583 
1584  helpCommandModifyActive(
1585  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1586  }
1587  else // handle as long
1588  {
1589  long int startValue;
1590  long int stepSize;
1591 
1592  StringMacros::getNumber(startValueStr, startValue);
1593  StringMacros::getNumber(startValueStr, stepSize);
1594 
1595  __COUT__ << "startValue " << startValue << std::endl;
1596  __COUT__ << "stepSize " << stepSize << std::endl;
1597  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1598 
1599  helpCommandModifyActive(
1600  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1601  }
1602 
1603 } // end startCommandModifyActive()
1604 
1605 //==============================================================================
1614 bool Iterator::checkCommandRun(IteratorWorkLoopStruct* iteratorStruct)
1615 {
1616  sleep(1); // sleep to give FSM time to transition
1617 
1618  // all RunControlStateMachine access commands should be mutually exclusive with
1619  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
1620  // with GatewaySupervisor main thread state machine accesses lockout the messages
1621  // array for the remainder of the scope this guarantees the reading thread can safely
1622  // access the messages
1623  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1624  __COUT__ << "Waiting for FSM access" << __E__;
1625  std::lock_guard<std::mutex> lock(
1626  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1627  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1628  __COUT__ << "Have FSM access" << __E__;
1629 
1630  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1631  return false;
1632 
1633  iteratorStruct->fsmCommandParameters_.clear();
1634 
1635  std::string errorStr = "";
1636  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1637  ->theStateMachine_.getCurrentStateName();
1638 
1640  // check for imposed actions and forced exits
1641  if(iteratorStruct->doPauseAction_)
1642  {
1643  // transition to pause state
1644  __COUT__ << "Transitioning FSM to Paused..." << __E__;
1645 
1646  if(currentState == "Paused")
1647  {
1648  // done with early pause exit!
1649  __COUT__ << "Transition to Paused complete." << __E__;
1650  return true;
1651  }
1652  else if(currentState == "Running") // launch transition to pause
1653  errorStr = iteratorStruct->theIterator_->theSupervisor_
1654  ->attemptStateMachineTransition(
1655  0,
1656  0,
1657  "Pause",
1658  iteratorStruct->fsmName_,
1659  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1660  WebUsers::DEFAULT_ITERATOR_USERNAME,
1661  iteratorStruct->fsmCommandParameters_);
1662  else if(currentState == "Configured")
1663  {
1664  // no need to pause state machine, no run going on
1665  __COUT__ << "In Configured state. No need to transition to Paused." << __E__;
1666  return true;
1667  }
1668  else
1669  errorStr = "Expected to be in Paused. Unexpectedly, the current state is " +
1670  currentState +
1671  ". Last State Machine error message was as follows: " +
1672  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1673  .getErrorMessage();
1674 
1675  if(errorStr != "")
1676  {
1677  __SS__ << "Iterator failed to pause because of the following error: "
1678  << errorStr;
1679  __SS_THROW__;
1680  }
1681  return false;
1682  }
1683  else if(iteratorStruct->doHaltAction_)
1684  {
1685  // transition to halted state
1686  __COUT__ << "Transitioning FSM to Halted..." << __E__;
1687 
1688  if(currentState == RunControlStateMachine::HALTED_STATE_NAME)
1689  {
1690  // done with early halt exit!
1691  __COUT__ << "Transition to Halted complete." << __E__;
1692  return true;
1693  }
1694  else if(currentState == RunControlStateMachine::
1695  RUNNING_STATE_NAME || // launch transition to halt
1696  currentState == RunControlStateMachine::PAUSED_STATE_NAME)
1697  errorStr = iteratorStruct->theIterator_->theSupervisor_
1698  ->attemptStateMachineTransition(
1699  0,
1700  0,
1701  RunControlStateMachine::ABORT_TRANSITION_NAME,
1702  iteratorStruct->fsmName_,
1703  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1704  WebUsers::DEFAULT_ITERATOR_USERNAME,
1705  iteratorStruct->fsmCommandParameters_);
1706  else if(currentState == RunControlStateMachine::
1707  CONFIGURED_STATE_NAME) // launch transition to halt
1708  errorStr = iteratorStruct->theIterator_->theSupervisor_
1709  ->attemptStateMachineTransition(
1710  0,
1711  0,
1712  RunControlStateMachine::HALT_TRANSITION_NAME,
1713  iteratorStruct->fsmName_,
1714  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1715  WebUsers::DEFAULT_ITERATOR_USERNAME,
1716  iteratorStruct->fsmCommandParameters_);
1717  else
1718  errorStr = "Expected to be in Halted. Unexpectedly, the current state is " +
1719  currentState +
1720  ". Last State Machine error message was as follows: " +
1721  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1722  .getErrorMessage();
1723 
1724  if(errorStr != "")
1725  {
1726  __SS__ << "Iterator failed to halt because of the following error: "
1727  << errorStr;
1728  __SS_THROW__;
1729  }
1730  return false;
1731  }
1732  else if(iteratorStruct->doResumeAction_)
1733  {
1734  // Note: check command gets one shot to resume
1735 
1736  // transition to running state
1737  __COUT__ << "Transitioning FSM to Running..." << __E__;
1738 
1739  if(currentState == "Paused") // launch transition to running
1740  errorStr = iteratorStruct->theIterator_->theSupervisor_
1741  ->attemptStateMachineTransition(
1742  0,
1743  0,
1744  RunControlStateMachine::RESUME_TRANSITION_NAME,
1745  iteratorStruct->fsmName_,
1746  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1747  WebUsers::DEFAULT_ITERATOR_USERNAME,
1748  iteratorStruct->fsmCommandParameters_);
1749 
1750  if(errorStr != "")
1751  {
1752  __SS__ << "Iterator failed to run because of the following error: "
1753  << errorStr;
1754  __SS_THROW__;
1755  }
1756  return false;
1757  }
1758 
1760  // normal running
1761 
1762  if(currentState != "Running")
1763  {
1764  if(iteratorStruct->runIsDone_ && currentState == "Configured")
1765  {
1766  // indication of done
1767  __COUT__ << "Reached end of run " << iteratorStruct->fsmNextRunNumber_
1768  << __E__;
1769  return true;
1770  }
1771 
1772  errorStr = "Expected to be in Running. Unexpectedly, the current state is " +
1773  currentState + ". Last State Machine error message was as follows: " +
1774  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1775  .getErrorMessage();
1776  }
1777  else // else in Running state! Check for end of run
1778  {
1779  bool waitOnRunningThreads = false;
1780  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_]
1781  .params_[IterateTable::commandRunParams_.WaitOnRunningThreads_])
1782  waitOnRunningThreads = true;
1783 
1784  time_t remainingDurationInSeconds;
1785  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1786  .params_[IterateTable::commandRunParams_.DurationInSeconds_]
1787  .c_str(),
1788  "%ld",
1789  &remainingDurationInSeconds);
1790 
1791  __COUT__ << "waitOnRunningThreads " << waitOnRunningThreads << __E__;
1792  __COUT__ << "remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
1793 
1795  // priority 1 is waiting on running threads
1796  if(waitOnRunningThreads)
1797  {
1798  // get status of all running FE workloops
1799  GatewaySupervisor* theSupervisor =
1800  iteratorStruct->theIterator_->theSupervisor_;
1801 
1802  bool allFrontEndsAreDone = true;
1803  for(auto& it : theSupervisor->allSupervisorInfo_.getAllFETypeSupervisorInfo())
1804  {
1805  try
1806  {
1807  std::string status = theSupervisor->send(it.second.getDescriptor(),
1808  "WorkLoopStatusRequest");
1809 
1810  __COUT__ << "FESupervisor instance " << it.first
1811  << " has status = " << status << std::endl;
1812 
1813  if(status != CoreSupervisorBase::WORK_LOOP_DONE)
1814  {
1815  allFrontEndsAreDone = false;
1816  break;
1817  }
1818  }
1819  catch(xdaq::exception::Exception& e)
1820  {
1821  __SS__ << "Could not retrieve status from FESupervisor instance "
1822  << it.first << "." << std::endl;
1823  __COUT_WARN__ << ss.str();
1824  errorStr = ss.str();
1825 
1826  if(errorStr != "")
1827  {
1828  __SS__
1829  << "Iterator failed to run because of the following error: "
1830  << errorStr;
1831  __SS_THROW__;
1832  }
1833  }
1834  }
1835 
1836  if(allFrontEndsAreDone)
1837  {
1838  // need to end run!
1839  __COUT__ << "FE workloops all complete! Stopping run..." << __E__;
1840 
1841  errorStr = iteratorStruct->theIterator_->theSupervisor_
1842  ->attemptStateMachineTransition(
1843  0,
1844  0,
1845  "Stop",
1846  iteratorStruct->fsmName_,
1847  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1848  WebUsers::DEFAULT_ITERATOR_USERNAME,
1849  iteratorStruct->fsmCommandParameters_);
1850 
1851  if(errorStr != "")
1852  {
1853  __SS__
1854  << "Iterator failed to stop run because of the following error: "
1855  << errorStr;
1856  __SS_THROW__;
1857  }
1858 
1859  // write indication of run done into duration
1860  iteratorStruct->runIsDone_ = true;
1861 
1862  return false;
1863  }
1864  } //end waitOnRunningThreads
1865 
1867  // priority 2 is duration, if <= 0 it is ignored
1868  if(remainingDurationInSeconds > 1)
1869  {
1870  //reset command start time (for displays) once running is stable
1871  if(remainingDurationInSeconds == iteratorStruct->originalDurationInSeconds_)
1872  {
1873  iteratorStruct->theIterator_->activeCommandStartTime_ =
1874  time(0); // reset on any change
1875  __COUT__ << "Starting run duration of " << remainingDurationInSeconds
1876  << " [s] at time = "
1877  << iteratorStruct->theIterator_->activeCommandStartTime_ << " "
1878  << StringMacros::getTimestampString(
1879  iteratorStruct->theIterator_->activeCommandStartTime_)
1880  << __E__;
1881  }
1882 
1883  --remainingDurationInSeconds;
1884 
1885  // write back to string
1886  char str[200];
1887  sprintf(str, "%ld", remainingDurationInSeconds);
1888  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1889  .params_[IterateTable::commandRunParams_.DurationInSeconds_] =
1890  str; // re-store as string
1891  }
1892  else if(remainingDurationInSeconds == 1)
1893  {
1894  // need to end run!
1895  __COUT__ << "Time duration reached! Stopping run..." << __E__;
1896 
1897  errorStr = iteratorStruct->theIterator_->theSupervisor_
1898  ->attemptStateMachineTransition(
1899  0,
1900  0,
1901  "Stop",
1902  iteratorStruct->fsmName_,
1903  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1904  WebUsers::DEFAULT_ITERATOR_USERNAME,
1905  iteratorStruct->fsmCommandParameters_);
1906 
1907  if(errorStr != "")
1908  {
1909  __SS__ << "Iterator failed to stop run because of the following error: "
1910  << errorStr;
1911  __SS_THROW__;
1912  }
1913 
1914  // write indication of run done
1915  iteratorStruct->runIsDone_ = true;
1916 
1917  // write original duration back to string
1918  char str[200];
1919  sprintf(str, "%ld", iteratorStruct->originalDurationInSeconds_);
1920  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1921  .params_[IterateTable::commandRunParams_.DurationInSeconds_] =
1922  str; // re-store as string
1923  return false;
1924  }
1925  }
1926 
1927  if(errorStr != "")
1928  {
1929  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1930  __SS_THROW__;
1931  }
1932  return false;
1933 } // end checkCommandRun()
1934 
1935 //==============================================================================
1936 bool Iterator::checkCommandWait(IteratorWorkLoopStruct* iteratorStruct)
1937 {
1938  sleep(1); // sleep for a second
1939 
1940  // Get the remaining time
1941  long remainingDurationInSeconds;
1942  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1943  .params_[IterateTable::commandWaitParams_.DurationInSeconds_]
1944  .c_str(),
1945  "%ld",
1946  &remainingDurationInSeconds);
1947 
1948  __COUT__ << "Wait remaining time: " << remainingDurationInSeconds << " seconds."
1949  << __E__;
1950 
1951  // If this is the first check, reset the command start time for the display
1952  if(remainingDurationInSeconds == iteratorStruct->originalDurationInSeconds_)
1953  {
1954  iteratorStruct->theIterator_->activeCommandStartTime_ = time(0);
1955  __COUT__ << "Starting wait duration of " << remainingDurationInSeconds
1956  << " [s] at time = "
1957  << iteratorStruct->theIterator_->activeCommandStartTime_ << " "
1958  << StringMacros::getTimestampString(
1959  iteratorStruct->theIterator_->activeCommandStartTime_)
1960  << __E__;
1961  }
1962 
1963  // Handle pause and halt actions if requested
1964  if(iteratorStruct->doPauseAction_ || iteratorStruct->doHaltAction_)
1965  {
1966  // Restore the original duration before exiting
1967  char str[200];
1968  sprintf(str, "%ld", iteratorStruct->originalDurationInSeconds_);
1969  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1970  .params_[IterateTable::commandWaitParams_.DurationInSeconds_] = str;
1971 
1972  return true; // Command is done when pause or halt is requested
1973  }
1974 
1975  // Check if we're done waiting
1976  if(remainingDurationInSeconds <= 1)
1977  {
1978  __COUT__ << "Wait duration complete!" << __E__;
1979 
1980  // Restore the original duration
1981  char str[200];
1982  sprintf(str, "%ld", iteratorStruct->originalDurationInSeconds_);
1983  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1984  .params_[IterateTable::commandWaitParams_.DurationInSeconds_] = str;
1985 
1986  iteratorStruct->waitIsDone_ = true;
1987  return true; // Command is done
1988  }
1989  else
1990  {
1991  // Decrement the remaining time
1992  --remainingDurationInSeconds;
1993 
1994  // Write back to string
1995  char str[200];
1996  sprintf(str, "%ld", remainingDurationInSeconds);
1997  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1998  .params_[IterateTable::commandWaitParams_.DurationInSeconds_] = str;
1999  }
2000 
2001  return false; // Command still in progress
2002 }
2003 
2004 //==============================================================================
2006 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct* iteratorStruct)
2007 {
2008  sleep(1); // sleep to give FSM time to transition
2009 
2010  // all RunControlStateMachine access commands should be mutually exclusive with
2011  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
2012  // with GatewaySupervisor main thread state machine accesses lockout the messages
2013  // array for the remainder of the scope this guarantees the reading thread can safely
2014  // access the messages
2015  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2016  __COUT__ << "Waiting for FSM access" << __E__;
2017  std::lock_guard<std::mutex> lock(
2018  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
2019  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2020  __COUT__ << "Have FSM access" << __E__;
2021 
2022  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
2023  return false;
2024 
2025  std::string errorStr = "";
2026  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
2027  ->theStateMachine_.getCurrentStateName();
2028 
2029  if(currentState == RunControlStateMachine::HALTED_STATE_NAME)
2030  errorStr =
2031  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
2032  0,
2033  0,
2034  "Configure",
2035  iteratorStruct->fsmName_,
2036  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
2037  WebUsers::DEFAULT_ITERATOR_USERNAME,
2038  iteratorStruct->fsmCommandParameters_);
2039  else if(currentState != RunControlStateMachine::CONFIGURED_STATE_NAME)
2040  errorStr = "Expected to be in '" + RunControlStateMachine::CONFIGURED_STATE_NAME +
2041  "' state. Unexpectedly, the current state is " + currentState + "." +
2042  ". Last State Machine error message was as follows: " +
2043  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
2044  .getErrorMessage();
2045  else // else successfully done (in Configured state!)
2046  {
2047  __COUT__ << "checkCommandConfigureAlias complete." << __E__;
2048 
2049  // also activate the Iterator config manager to mimic active config
2050  std::pair<std::string, TableGroupKey> newActiveGroup =
2051  iteratorStruct->cfgMgr_->getTableGroupFromAlias(
2052  iteratorStruct->fsmCommandParameters_[0]);
2053  iteratorStruct->cfgMgr_->loadTableGroup(
2054  newActiveGroup.first, newActiveGroup.second, true /*activate*/);
2055 
2056  __COUT__ << "originalTrackChanges " << iteratorStruct->originalTrackChanges_
2057  << __E__;
2058  __COUT__ << "originalConfigGroup " << iteratorStruct->originalConfigGroup_
2059  << __E__;
2060  __COUT__ << "originalConfigKey " << iteratorStruct->originalConfigKey_ << __E__;
2061 
2062  __COUT__ << "currentTrackChanges "
2063  << ConfigurationInterface::isVersionTrackingEnabled() << __E__;
2064  __COUT__ << "originalConfigGroup "
2065  << iteratorStruct->cfgMgr_->getActiveGroupName() << __E__;
2066  __COUT__ << "originalConfigKey " << iteratorStruct->cfgMgr_->getActiveGroupKey()
2067  << __E__;
2068 
2069  return true;
2070  }
2071 
2072  if(errorStr != "")
2073  {
2074  __SS__ << "Iterator failed to configure with system alias '"
2075  << (iteratorStruct->fsmCommandParameters_.size()
2076  ? iteratorStruct->fsmCommandParameters_[0]
2077  : "UNKNOWN")
2078  << "' because of the following error: " << errorStr;
2079  __SS_THROW__;
2080  }
2081  return false;
2082 } // end checkCommandConfigure()
2083 
2084 //==============================================================================
2086 bool Iterator::checkCommandFSMTransition(IteratorWorkLoopStruct* iteratorStruct,
2087  const std::string& finalState)
2088 {
2089  __COUTV__(finalState);
2090 
2091  sleep(1); // sleep to give FSM time to transition
2092 
2093  // all RunControlStateMachine access commands should be mutually exclusive with
2094  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
2095  // with GatewaySupervisor main thread state machine accesses lockout the messages
2096  // array for the remainder of the scope this guarantees the reading thread can safely
2097  // access the messages
2098  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2099  __COUT__ << "Waiting for FSM access" << __E__;
2100  std::lock_guard<std::mutex> lock(
2101  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
2102  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2103  __COUT__ << "Have FSM access" << __E__;
2104 
2105  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
2106  return false;
2107 
2108  std::string errorStr = "";
2109  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
2110  ->theStateMachine_.getCurrentStateName();
2111 
2112  if(currentState != finalState)
2113  errorStr = "Expected to be in " + finalState +
2114  ". Unexpectedly, the current state is " + currentState + "." +
2115  ". Last State Machine error message was as follows: " +
2116  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
2117  .getErrorMessage();
2118  else // else successfully done (in Configured state!)
2119  {
2120  __COUT__ << "checkCommandFSMTransition complete." << __E__;
2121 
2122  return true;
2123  }
2124 
2125  if(errorStr != "")
2126  {
2127  __SS__ << "Iterator failed to reach final state '" << finalState
2128  << "' because of the following error: " << errorStr;
2129  __SS_THROW__;
2130  }
2131  return false;
2132 } // end checkCommandConfigure()
2133 
2134 //==============================================================================
2136 std::vector<IterateTable::Command> Iterator::generateIterationPlan(
2137  const std::string& fsmName,
2138  const std::string& configAlias,
2139  uint64_t durationSeconds /* = -1 */,
2140  unsigned int numberOfRuns /* = 1 */)
2141 {
2142  std::vector<IterateTable::Command> commands;
2143 
2144  //example commands:
2145  // Choose FSM name
2146  // Configure if needed to selected system alias
2147  // Start run
2148  // count seconds
2149  // Stop run numberOfRuns
2150  // loop
2151 
2152  if(durationSeconds == (uint64_t)-1)
2153  numberOfRuns = 1;
2154  __COUTV__(durationSeconds);
2155  __COUTV__(numberOfRuns);
2156 
2157  // -------- Choose FSM name
2158  {
2159  commands.push_back(IterateTable::Command());
2160  commands.back().type_ = IterateTable::COMMAND_CHOOSE_FSM;
2161 
2162  commands.back().params_.emplace(
2163  std::pair<std::string /*param name*/, std::string /*param value*/>(
2164  IterateTable::commandChooseFSMParams_.NameOfFSM_, fsmName));
2165  }
2166 
2167  // -------- Configure if needed to selected system alias
2168  {
2169  commands.push_back(IterateTable::Command());
2170  commands.back().type_ = IterateTable::COMMAND_CONFIGURE_ALIAS;
2171 
2172  commands.back().params_.emplace(
2173  std::pair<std::string /*param name*/, std::string /*param value*/>(
2174  IterateTable::commandConfigureAliasParams_.SystemAlias_, configAlias));
2175  }
2176 
2177  if(durationSeconds == (uint64_t)-1)
2178  {
2179  __COUT__ << "Open ended run..." << __E__;
2180 
2181  // -------- Start FSM transtion
2182  {
2183  commands.push_back(IterateTable::Command());
2184  commands.back().type_ = IterateTable::COMMAND_START;
2185  }
2186  }
2187  else
2188  {
2189  __COUT__ << "Finite duration run(s)..." << __E__;
2190 
2191  if(numberOfRuns > 1)
2192  {
2193  __COUT__ << "Setting up iterator loop for " << numberOfRuns << " runs."
2194  << __E__;
2195 
2196  commands.push_back(IterateTable::Command());
2197  commands.back().type_ = IterateTable::COMMAND_BEGIN_LABEL;
2198 
2199  commands.back().params_.emplace(
2200  std::pair<std::string /*param name*/, std::string /*param value*/>(
2201  IterateTable::commandBeginLabelParams_.Label_, "GENERATED_LABEL"));
2202  }
2203 
2204  // -------- Start managed run
2205  {
2206  commands.push_back(IterateTable::Command());
2207  commands.back().type_ = IterateTable::COMMAND_RUN;
2208 
2209  commands.back().params_.emplace(
2210  std::pair<std::string /*param name*/, std::string /*param value*/>(
2211  IterateTable::commandRunParams_.DurationInSeconds_,
2212  std::to_string(durationSeconds)));
2213  }
2214 
2215  if(numberOfRuns > 1)
2216  {
2217  commands.push_back(IterateTable::Command());
2218  commands.back().type_ = IterateTable::COMMAND_REPEAT_LABEL;
2219 
2220  commands.back().params_.emplace(
2221  std::pair<std::string /*param name*/, std::string /*param value*/>(
2222  IterateTable::commandRepeatLabelParams_.Label_, "GENERATED_LABEL"));
2223  commands.back().params_.emplace(
2224  std::pair<std::string /*param name*/, std::string /*param value*/>(
2225  IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_,
2226  std::to_string(numberOfRuns -
2227  1) //number of repeats (i.e., 1 repeat, gives 2 runs)
2228  ));
2229  }
2230  }
2231 
2232  __COUTV__(commands.size());
2233 
2234  return commands;
2235 } //end generateIterationPlan()
2236 
2237 //==============================================================================
2238 bool Iterator::handleCommandRequest(HttpXmlDocument& xmldoc,
2239  const std::string& command,
2240  const std::string& parameter)
2241 try
2242 {
2243  //__COUTV__(command);
2244  if(command == "iteratePlay")
2245  {
2246  playIterationPlan(xmldoc, parameter);
2247  return true;
2248  }
2249  else if(command == "iteratePlayGenerated")
2250  {
2251  playGeneratedIterationPlan(xmldoc, parameter);
2252  return true;
2253  }
2254  else if(command == "iteratePause")
2255  {
2256  pauseIterationPlan(xmldoc);
2257  return true;
2258  }
2259  else if(command == "iterateHalt")
2260  {
2261  haltIterationPlan(xmldoc);
2262  return true;
2263  }
2264  else if(command == "getIterationPlanStatus")
2265  {
2266  if(activePlanName_ == "" &&
2267  parameter !=
2268  "") //take parameter to set active plan name from GUI manipulations
2269  {
2270  activePlanName_ = parameter;
2271  __COUTV__(activePlanName_);
2272  }
2273  getIterationPlanStatus(xmldoc);
2274  return true;
2275  }
2276  else // return true if iterator has control of state machine
2277  {
2278  // lockout the messages array for the remainder of the scope
2279  // this guarantees the reading thread can safely access the messages
2280  if(theSupervisor_->VERBOSE_MUTEX)
2281  __COUT__ << "Waiting for iterator access" << __E__;
2282  std::lock_guard<std::mutex> lock(accessMutex_);
2283  if(theSupervisor_->VERBOSE_MUTEX)
2284  __COUT__ << "Have iterator access" << __E__;
2285 
2286  if(iteratorBusy_)
2287  {
2288  __SS__ << "Error - Can not accept request because the Iterator "
2289  << "is currently "
2290  << "in control of State Machine progress. ";
2291  __COUT_ERR__ << "\n" << ss.str();
2292  __COUT_ERR__ << "\n" << ss.str();
2293 
2294  xmldoc.addTextElementToData("state_tranisition_attempted",
2295  "0"); // indicate to GUI transition NOT attempted
2296  xmldoc.addTextElementToData(
2297  "state_tranisition_attempted_err",
2298  ss.str()); // indicate to GUI transition NOT attempted
2299  theSupervisor_->theStateMachine_.setErrorMessage(ss.str());
2300  return true; // to block other commands
2301  }
2302  }
2303  return false;
2304 } //end handleCommandRequest()
2305 catch(...)
2306 {
2307  __SS__ << "Error caught by Iterator command handling!" << __E__;
2308  try
2309  {
2310  throw;
2311  }
2312  catch(const std::runtime_error& e)
2313  {
2314  ss << "\nHere is the error: " << e.what() << __E__;
2315  }
2316  catch(...)
2317  {
2318  ss << "Uknown error caught." << __E__;
2319  }
2320 
2321  __COUT_ERR__ << "\n" << ss.str();
2322 
2323  xmldoc.addTextElementToData("state_tranisition_attempted",
2324  "0"); // indicate to GUI transition NOT attempted
2325  xmldoc.addTextElementToData("state_tranisition_attempted_err",
2326  ss.str()); // indicate to GUI transition NOT attempted
2327  theSupervisor_->theStateMachine_.setErrorMessage(ss.str());
2328  return true;
2329 } // end handleCommandRequest() error handling
2330 
2331 //==============================================================================
2332 void Iterator::playIterationPlan(HttpXmlDocument& xmldoc, const std::string& planName)
2333 {
2334  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
2335 
2336  if(planName == Iterator::RESERVED_GEN_PLAN_NAME)
2337  {
2338  __SS__ << "Illegal use of reserved iteration plan name '"
2339  << Iterator::RESERVED_GEN_PLAN_NAME << "!' Please select a different name."
2340  << __E__;
2341  __SS_THROW__;
2342  }
2343 
2344  playIterationPlanPrivate(xmldoc, planName);
2345 
2346 } //end playIterationPlan()
2347 
2348 //==============================================================================
2349 void Iterator::playGeneratedIterationPlan(HttpXmlDocument& xmldoc,
2350  const std::string& parametersCSV)
2351 {
2352  __COUTV__(parametersCSV);
2353  std::vector<std::string> parameters =
2354  StringMacros::getVectorFromString(parametersCSV, {','});
2355 
2356  if(parameters.size() != 6)
2357  {
2358  __SS__ << "Malformed CSV parameters to playGeneratedIterationPlan(), must be 6 "
2359  "arguments and there were "
2360  << parameters.size() << ": " << parametersCSV << __E__;
2361  __SS_THROW__;
2362  }
2363  // parameters[0] /*fsmName*/,
2364  // parameters[1] /*configAlias*/,
2365  // parameters[2] /*durationSeconds*/,
2366  // parameters[3] /*numberOfRuns*/,
2367  // parameters[4] /*keepConfiguration*/,
2368  // parameters[5] /*logEntry*/ double encoded
2369  parameters[5] = StringMacros::decodeURIComponent(parameters[5]);
2370 
2371  uint64_t durationSeconds;
2372  sscanf(parameters[2].c_str(), "%lu", &durationSeconds);
2373  unsigned int numberOfRuns;
2374  sscanf(parameters[3].c_str(), "%u", &numberOfRuns);
2375  unsigned int keepConfiguration;
2376  sscanf(parameters[4].c_str(), "%u", &keepConfiguration);
2377  playGeneratedIterationPlan(xmldoc,
2378  parameters[0] /*fsmName*/,
2379  parameters[1] /*configAlias*/,
2380  durationSeconds,
2381  numberOfRuns,
2382  keepConfiguration,
2383  parameters[5] /*logEntry*/
2384  );
2385 
2386 } //end playGeneratedIterationPlan()
2387 
2388 //==============================================================================
2389 void Iterator::playGeneratedIterationPlan(HttpXmlDocument& xmldoc,
2390  const std::string& fsmName,
2391  const std::string& configAlias,
2392  uint64_t durationSeconds /* = -1 */,
2393  unsigned int numberOfRuns /* = 1 */,
2394  bool keepConfiguration /* = false */,
2395  const std::string& logEntry)
2396 {
2397  std::string planName = Iterator::RESERVED_GEN_PLAN_NAME;
2398  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
2399 
2400  genFsmName_ = fsmName;
2401  genConfigAlias_ = configAlias;
2402  genPlanDurationSeconds_ = durationSeconds;
2403  genPlanNumberOfRuns_ = numberOfRuns;
2404  genKeepConfiguration_ = keepConfiguration;
2405  genLogEntry_ = logEntry;
2406 
2407  __COUTV__(genFsmName_);
2408  __COUTV__(genConfigAlias_);
2409  __COUTV__(genPlanDurationSeconds_);
2410  __COUTV__(genPlanNumberOfRuns_);
2411  __COUTV__(genKeepConfiguration_);
2412  __COUTV__(genLogEntry_);
2413 
2414  playIterationPlanPrivate(xmldoc, planName);
2415 
2416 } //end playGeneratedIterationPlan()
2417 
2418 //==============================================================================
2420 void Iterator::playIterationPlanPrivate(HttpXmlDocument& xmldoc,
2421  const std::string& planName)
2422 {
2423  // setup "play" command
2424 
2425  // lockout the messages array for the remainder of the scope
2426  // this guarantees the reading thread can safely access the messages
2427  if(theSupervisor_->VERBOSE_MUTEX)
2428  __COUT__ << "Waiting for iterator access" << __E__;
2429  std::lock_guard<std::mutex> lock(accessMutex_);
2430  if(theSupervisor_->VERBOSE_MUTEX)
2431  __COUT__ << "Have iterator access" << __E__;
2432 
2433  if(!activePlanIsRunning_ && !commandPlay_)
2434  {
2435  if(!workloopRunning_)
2436  {
2437  // start thread with member variables initialized
2438 
2439  workloopRunning_ = true;
2440 
2441  // must start thread first
2442  std::thread([](Iterator* iterator) { Iterator::IteratorWorkLoop(iterator); },
2443  this)
2444  .detach();
2445  }
2446 
2447  activePlanName_ = planName;
2448  commandPlay_ = true;
2449  }
2450  else
2451  {
2452  __SS__ << "Invalid play command attempted. Can only play when the Iterator is "
2453  "inactive or paused."
2454  << " If you would like to restart an iteration plan, first try halting "
2455  "the Iterator."
2456  << __E__;
2457  __COUT__ << ss.str();
2458 
2459  xmldoc.addTextElementToData("error_message", ss.str());
2460 
2461  __COUT__ << "Invalid play command attempted. " << activePlanIsRunning_ << " "
2462  << commandPlay_ << " " << activePlanName_ << __E__;
2463  }
2464 } //end playIterationPlan()
2465 
2466 //==============================================================================
2467 void Iterator::pauseIterationPlan(HttpXmlDocument& xmldoc)
2468 {
2469  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
2470  << __E__;
2471  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
2472  << __E__;
2473 
2474  // setup "pause" command
2475 
2476  // lockout the messages array for the remainder of the scope
2477  // this guarantees the reading thread can safely access the messages
2478  if(theSupervisor_->VERBOSE_MUTEX)
2479  __COUT__ << "Waiting for iterator access" << __E__;
2480  std::lock_guard<std::mutex> lock(accessMutex_);
2481  if(theSupervisor_->VERBOSE_MUTEX)
2482  __COUT__ << "Have iterator access" << __E__;
2483 
2484  if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
2485  {
2486  commandPause_ = true;
2487  }
2488  else
2489  {
2490  __SS__ << "Invalid pause command attempted. Can only pause when running."
2491  << __E__;
2492  __COUT__ << ss.str();
2493 
2494  xmldoc.addTextElementToData("error_message", ss.str());
2495 
2496  __COUT__ << "Invalid pause command attempted. " << workloopRunning_ << " "
2497  << activePlanIsRunning_ << " " << commandPause_ << " " << activePlanName_
2498  << __E__;
2499  }
2500 } //end pauseIterationPlan()
2501 
2502 //==============================================================================
2503 void Iterator::haltIterationPlan(HttpXmlDocument& /*xmldoc*/)
2504 {
2505  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
2506  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
2507 
2508  // setup "halt" command
2509 
2510  if(workloopRunning_)
2511  {
2512  // lockout the messages array for the remainder of the scope
2513  // this guarantees the reading thread can safely access the messages
2514  if(theSupervisor_->VERBOSE_MUTEX)
2515  __COUT__ << "Waiting for iterator access" << __E__;
2516  std::lock_guard<std::mutex> lock(accessMutex_);
2517  if(theSupervisor_->VERBOSE_MUTEX)
2518  __COUT__ << "Have iterator access" << __E__;
2519 
2520  __COUT__ << "activePlanIsRunning_: " << activePlanIsRunning_ << __E__;
2521  __COUT__ << "Passing halt command to iterator thread." << __E__;
2522  commandHalt_ = true;
2523 
2524  // clear
2525  activePlanName_ = "";
2526  activeCommandIndex_ = -1;
2527  }
2528  else // no thread, so halt (and reset Error') without command to thread
2529  {
2530  __COUT__ << "No thread, so conducting halt." << __E__;
2531  Iterator::haltIterator(this);
2532  }
2533 } //end haltIterationPlan()
2534 
2535 //==============================================================================
2537 void Iterator::getIterationPlanStatus(HttpXmlDocument& xmldoc)
2538 {
2539  xmldoc.addTextElementToData(
2540  "current_state",
2541  theSupervisor_->theStateMachine_.isInTransition()
2542  ? theSupervisor_->theStateMachine_.getCurrentTransitionName(
2543  theSupervisor_->stateMachineLastCommandInput_)
2544  : theSupervisor_->theStateMachine_.getCurrentStateName());
2545 
2546  // xmldoc.addTextElementToData("in_transition",
2547  // theSupervisor_->theStateMachine_.isInTransition() ? "1" : "0");
2548  if(theSupervisor_->theStateMachine_.isInTransition())
2549  xmldoc.addTextElementToData(
2550  "transition_progress",
2551  theSupervisor_->theProgressBar_.readPercentageString());
2552  else
2553  xmldoc.addTextElementToData("transition_progress", "100");
2554 
2555  xmldoc.addNumberElementToData("time_in_state",
2556  theSupervisor_->theStateMachine_.getTimeInState());
2557 
2558  // lockout the messages array for the remainder of the scope
2559  // this guarantees the reading thread can safely access the messages
2560  if(theSupervisor_->VERBOSE_MUTEX)
2561  __COUT__ << "Waiting for iterator access" << __E__;
2562  std::lock_guard<std::mutex> lock(accessMutex_);
2563  if(theSupervisor_->VERBOSE_MUTEX)
2564  __COUT__ << "Have iterator access" << __E__;
2565 
2566  xmldoc.addTextElementToData("active_plan", activePlanName_);
2567  xmldoc.addTextElementToData("last_started_plan", lastStartedPlanName_);
2568  xmldoc.addTextElementToData("last_finished_plan", lastFinishedPlanName_);
2569 
2570  xmldoc.addNumberElementToData("current_command_index", activeCommandIndex_);
2571  xmldoc.addNumberElementToData("current_number_of_commands", activeNumberOfCommands_);
2572  xmldoc.addTextElementToData("current_command_type", activeCommandType_);
2573  xmldoc.addNumberElementToData("current_command_duration",
2574  time(0) - activeCommandStartTime_);
2575  xmldoc.addNumberElementToData("current_command_iteration", activeCommandIteration_);
2576  for(const auto& depthIteration : depthIterationStack_)
2577  xmldoc.addNumberElementToData("depth_iteration", depthIteration);
2578 
2579  if(activePlanName_ == Iterator::RESERVED_GEN_PLAN_NAME)
2580  {
2581  xmldoc.addNumberElementToData("generated_number_of_runs", genPlanNumberOfRuns_);
2582  xmldoc.addNumberElementToData("generated_duration_of_runs",
2583  genPlanDurationSeconds_);
2584  }
2585 
2586  if(activePlanIsRunning_ && iteratorBusy_)
2587  {
2588  if(workloopRunning_)
2589  xmldoc.addTextElementToData("active_plan_status", "Running");
2590  else
2591  xmldoc.addTextElementToData("active_plan_status", "Error");
2592  }
2593  else if(!activePlanIsRunning_ && iteratorBusy_)
2594  xmldoc.addTextElementToData("active_plan_status", "Paused");
2595  else
2596  xmldoc.addTextElementToData("active_plan_status", "Inactive");
2597 
2598  xmldoc.addTextElementToData("error_message", errorMessage_);
2599 } //end getIterationPlanStatus()
ots::ConfigurationManagerRW::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false)
Definition: ConfigurationManagerRW.cc:887
ots::ConfigurationManager::getTableGroupFromAlias
std::pair< std::string, TableGroupKey > getTableGroupFromAlias(std::string systemAlias, ProgressBar *progressBar=0)
Getters.
Definition: ConfigurationManager.cc:3197
ots::ConfigurationTree::isDisconnected
bool isDisconnected(void) const
Definition: ConfigurationTree.cc:1569
ots::ConfigurationTree::getNode
ConfigurationTree getNode(const std::string &nodeName, bool doNotThrowOnBrokenUIDLinks=false) const
navigating between nodes
Definition: ConfigurationTree.cc:1068
ots::ConfigurationTree::getValue
void getValue(T &value) const
ots::FiniteStateMachine::getTimeInState
time_t getTimeInState(void) const
Definition: FiniteStateMachine.cc:79
ots::IterateTable::COMMAND_BEGIN_LABEL
static const std::string COMMAND_BEGIN_LABEL
Definition: IterateTable.h:36
ots::Iterator::getIterationPlanStatus
void getIterationPlanStatus(HttpXmlDocument &xmldoc)
return state machine and iterator status
Definition: Iterator.cc:2537
ots::ProgressBar::readPercentageString
std::string readPercentageString()
return percentage complete as std::string
Definition: ProgressBar.cc:135
ots::SOAPMessenger::send
std::string send(XDAQ_CONST_CALL xdaq::ApplicationDescriptor *d, xoap::MessageReference message)
Definition: SOAPMessenger.cc:39
ots::StringMacros::getTimestampString
static std::string getTimestampString(const std::string &linuxTimeInSeconds)
Definition: StringMacros.cc:652
ots::StringMacros::getVectorFromString
static void getVectorFromString(const std::string &inputString, std::vector< std::string > &listToReturn, const std::set< char > &delimiter={',', '|', '&'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'}, std::vector< char > *listOfDelimiters=0, bool decodeURIComponents=false)
Definition: StringMacros.cc:765
ots::StringMacros::vectorToString
static std::string vectorToString(const std::vector< T > &setToReturn, const std::string &delimeter=", ")
vectorToString ~
ots::StringMacros::isNumber
static bool isNumber(const std::string &stringToCheck)
Note: before call consider use of stringToCheck = StringMacros::convertEnvironmentVariables(stringToC...
Definition: StringMacros.cc:464
ots::StringMacros::decodeURIComponent
static std::string decodeURIComponent(const std::string &data)
Definition: StringMacros.cc:116
ots::StringMacros::getNumber
static bool getNumber(const std::string &s, T &retValue)