otsdaq  3.11.00
Iterator.cc
1 #include "otsdaq/GatewaySupervisor/Iterator.h"
2 #include "otsdaq/CoreSupervisors/CoreSupervisorBase.h"
3 #include "otsdaq/GatewaySupervisor/GatewaySupervisor.h"
4 #include "otsdaq/Macros/CoutMacros.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/WebUsersUtilities/WebUsers.h"
7 
8 #include <iostream>
9 #include <thread> //for std::thread
10 
11 #undef __MF_SUBJECT__
12 #define __MF_SUBJECT__ "Iterator"
13 
14 using namespace ots;
15 
16 #define ITERATOR_PLAN_HISTORY_FILENAME \
17  ((getenv("SERVICE_DATA_PATH") == NULL) \
18  ? (std::string(__ENV__("USER_DATA")) + "/ServiceData") \
19  : (std::string(__ENV__("SERVICE_DATA_PATH")))) + \
20  "/IteratorPlanHistory.hist"
21 
22 const std::string Iterator::RESERVED_GEN_PLAN_NAME = "---GENERATED_PLAN---";
23 
24 //==============================================================================
25 Iterator::Iterator(GatewaySupervisor* supervisor)
26  : workloopRunning_(false)
27  , activePlanIsRunning_(false)
28  , iteratorBusy_(false)
29  , commandPlay_(false)
30  , commandPause_(false)
31  , commandHalt_(false)
32  , activePlanName_("")
33  , activeCommandIndex_(-1)
34  , activeCommandStartTime_(0)
35  , theSupervisor_(supervisor)
36 {
37  __COUT__ << "Iterator constructed." << __E__;
38 
39  //restore lastStartedPlanName_ and lastFinishedPlanName_ from file
40  __COUT__ << "Filename for iterator history: " << ITERATOR_PLAN_HISTORY_FILENAME
41  << __E__;
42  FILE* fp = fopen((ITERATOR_PLAN_HISTORY_FILENAME).c_str(), "r");
43  if(fp) // check for all core table names in file, and force their presence
44  {
45  char line[100];
46  int i = 0;
47  while(fgets(line, 100, fp))
48  {
49  if(strlen(line) < 3)
50  continue;
51  line[strlen(line) - 1] = '\0'; // remove endline
52  __COUTV__(line);
53  if(i == 0)
54  lastStartedPlanName_ = line;
55  else if(i == 1)
56  {
57  lastFinishedPlanName_ = line;
58  break; //only 2 lines in file (for now)
59  }
60  ++i;
61  }
62  fclose(fp);
63  __COUTV__(lastStartedPlanName_);
64  __COUTV__(lastFinishedPlanName_);
65  } //end get iterator plan history
66 } //end constructor()
67 
68 //==============================================================================
69 Iterator::~Iterator(void) {}
70 
71 //==============================================================================
72 void Iterator::IteratorWorkLoop(Iterator* iterator)
73 try
74 {
75  __COUT__ << "Iterator work loop starting..." << __E__;
76 
77  // mutex init scope
78  {
79  // lockout the messages array for the remainder of the scope
80  // this guarantees the reading thread can safely access the messages
81  if(iterator->theSupervisor_->VERBOSE_MUTEX)
82  __COUT__ << "Waiting for iterator access" << __E__;
83  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
84  if(iterator->theSupervisor_->VERBOSE_MUTEX)
85  __COUT__ << "Have iterator access" << __E__;
86 
87  iterator->errorMessage_ = ""; // clear error message
88  }
89 
90  ConfigurationManagerRW theConfigurationManager(
91  WebUsers::DEFAULT_ITERATOR_USERNAME); // this is a restricted username
92  // theConfigurationManager.init();
93  theConfigurationManager.getAllTableInfo(true /* refresh */, // to prep all info
94  0 /* accumulatedWarnings */,
95  "" /* errorFilterName */,
96  false /* getGroupKeys */,
97  false /* getGroupInfo */,
98  true /* initializeActiveGroups */);
99 
100  __COUT__ << "Iterator work loop starting..." << __E__;
101  IteratorWorkLoopStruct theIteratorStruct(iterator, &theConfigurationManager);
102  __COUT__ << "Iterator work loop starting..." << __E__;
103 
104  const IterateTable* itConfig;
105 
106  std::vector<IterateTable::Command> commands;
107 
108  try
109  {
110  while(1)
111  {
112  // Process:
113  // - always "listen" for commands
114  // - play: if no plan running, activePlanIsRunning_ = true,
115  // and start or continue plan based on name/commandIndex
116  // - pause: if plan playing, pause it, activePlanIsRunning_ = false
117  // and do not clear commandIndex or name, iteratorBusy_ = true
118  // - halt: if plan playing or not, activePlanIsRunning_ = false
119  // and clear commandIndex or name, iteratorBusy_ = false
120  // - when running
121  // - go through each command
122  // - start the command, commandBusy = true
123  // - check for complete, then commandBusy = false
124 
125  // start command handling
126  // define mutex scope
127  {
128  // lockout the messages array for the remainder of the scope
129  // this guarantees the reading thread can safely access the messages
130  if(iterator->theSupervisor_->VERBOSE_MUTEX)
131  __COUT__ << "Waiting for iterator access" << __E__;
132  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
133  if(iterator->theSupervisor_->VERBOSE_MUTEX)
134  __COUT__ << "Have iterator access" << __E__;
135 
136  if(iterator->commandPlay_)
137  {
138  iterator->commandPlay_ = false; // clear
139 
140  if(!iterator->activePlanIsRunning_)
141  {
142  // valid PLAY command!
143 
144  iterator->activePlanIsRunning_ = true;
145  iterator->iteratorBusy_ = true;
146  iterator->errorMessage_ = ""; // clear any previous error
147 
148  if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
149  {
150  __COUT__ << "New plan name encountered old="
151  << theIteratorStruct.activePlan_
152  << " vs new=" << iterator->activePlanName_ << __E__;
153  theIteratorStruct.commandIndex_ = -1; // reset for new plan
154  }
155 
156  theIteratorStruct.activePlan_ = iterator->activePlanName_;
157  iterator->lastStartedPlanName_ = iterator->activePlanName_;
158  FILE* fp = fopen((ITERATOR_PLAN_HISTORY_FILENAME).c_str(), "w");
159  if(fp)
160  {
161  fprintf(fp, "%s\n", iterator->lastStartedPlanName_.c_str());
162  fprintf(fp, "%s\n", iterator->lastFinishedPlanName_.c_str());
163  fclose(fp);
164  }
165  else
166  __COUT_WARN__ << "Could not open Iterator history file: "
167  << ITERATOR_PLAN_HISTORY_FILENAME << __E__;
168 
169  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
170  {
171  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_
172  << ".'" << __E__;
173  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_
174  << ".'" << __E__;
175  }
176  else
177  {
178  theIteratorStruct.doResumeAction_ = true;
179  __COUT__ << "Continuing plan '"
180  << theIteratorStruct.activePlan_
181  << "' at command index "
182  << theIteratorStruct.commandIndex_ << ". " << __E__;
183  __COUT__ << "Continuing plan '"
184  << theIteratorStruct.activePlan_
185  << "' at command index "
186  << theIteratorStruct.commandIndex_ << ". " << __E__;
187  }
188  }
189  }
190  else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
191  {
192  theIteratorStruct.doPauseAction_ = true;
193  iterator->commandPause_ = false; // clear
194  }
195  else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
196  {
197  theIteratorStruct.doHaltAction_ = true;
198  iterator->commandHalt_ = false; // clear
199  }
200 
201  theIteratorStruct.running_ = iterator->activePlanIsRunning_;
202 
203  if(iterator
204  ->activeCommandIndex_ != // update active command status if changed
205  theIteratorStruct.commandIndex_)
206  {
207  iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
208  if(theIteratorStruct.commandIndex_ <
209  theIteratorStruct.commands_.size())
210  iterator->activeCommandType_ =
211  theIteratorStruct.commands_[theIteratorStruct.commandIndex_]
212  .type_;
213  else
214  iterator->activeCommandType_ = "";
215  iterator->activeCommandStartTime_ = time(0); // reset on any change
216 
217  if(theIteratorStruct.commandIndex_ <
218  theIteratorStruct.commandIterations_.size())
219  iterator->activeCommandIteration_ =
220  theIteratorStruct
221  .commandIterations_[theIteratorStruct.commandIndex_];
222  else
223  iterator->activeCommandIteration_ = -1;
224 
225  iterator->depthIterationStack_.clear();
226  for(const auto& depthIteration : theIteratorStruct.stepIndexStack_)
227  iterator->depthIterationStack_.push_back(depthIteration);
228  // if(theIteratorStruct.stepIndexStack_.size())
229  // iterator->activeLoopIteration_ =
230  // theIteratorStruct.stepIndexStack_.back(); else
231  // iterator->activeLoopIteration_ = -1;
232 
233  iterator->activeNumberOfCommands_ =
234  theIteratorStruct.commands_.size();
235  }
236 
237  } // end command handling and iterator mutex
238 
241  // do halt or pause action outside of iterator mutex
242 
243  if(theIteratorStruct.doPauseAction_)
244  {
245  // valid PAUSE-iterator command!
246 
247  // safely pause plan!
248  // i.e. check that command is complete
249 
250  __COUT__ << "Waiting to pause..." << __E__;
251  while(!iterator->checkCommand(&theIteratorStruct))
252  __COUT__ << "Waiting to pause..." << __E__;
253 
254  __COUT__ << "Completing pause..." << __E__;
255 
256  theIteratorStruct.doPauseAction_ = false; // clear
257 
258  // lockout the messages array for the remainder of the scope
259  // this guarantees the reading thread can safely access the messages
260  if(iterator->theSupervisor_->VERBOSE_MUTEX)
261  __COUT__ << "Waiting for iterator access" << __E__;
262  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
263  if(iterator->theSupervisor_->VERBOSE_MUTEX)
264  __COUT__ << "Have iterator access" << __E__;
265 
266  iterator->activePlanIsRunning_ = false;
267 
268  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_
269  << "' at command index " << theIteratorStruct.commandIndex_
270  << ". " << __E__;
271  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_
272  << "' at command index " << theIteratorStruct.commandIndex_
273  << ". " << __E__;
274 
275  continue; // resume workloop
276  }
277  else if(theIteratorStruct.doHaltAction_)
278  {
279  // valid HALT-iterator command!
280 
281  // safely end plan!
282  // i.e. check that command is complete (if command throws, proceed to halt anyway)
283 
284  try
285  {
286  __COUT__ << "Waiting to halt..." << __E__;
287  while(!iterator->checkCommand(&theIteratorStruct))
288  __COUT__ << "Waiting to halt..." << __E__;
289  }
290  catch(...)
291  {
292  __COUT_INFO__ << "Current command threw during halt — "
293  << "proceeding to halt anyway." << __E__;
294  }
295 
296  __COUT__ << "Completing halt..." << __E__;
297 
298  theIteratorStruct.doHaltAction_ = false; // clear
299 
300  iterator->haltIterator(iterator, &theIteratorStruct);
301 
302  // //last ditch effort to make sure FSM is halted
303  // iterator->haltIterator(
304  // iterator->theSupervisor_,
305  // theIteratorStruct.fsmName_);
306  //
307  // //lockout the messages array for the remainder of the scope
308  // //this guarantees the reading thread can safely access the
309  // messages
310  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting
311  // for iterator access" << __E__; std::lock_guard<std::mutex>
312  // lock(iterator->accessMutex_);
313  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have
314  // iterator access" << __E__;
315  //
316  // iterator->activePlanIsRunning_ = false;
317  // iterator->iteratorBusy_ = false;
318  //
319  // __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
320  // at command index " << theIteratorStruct.commandIndex_ <<
321  //". " << __E__;
322  // __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
323  // at command index " << theIteratorStruct.commandIndex_ <<
324  //". " << __E__;
325  //
326  // theIteratorStruct.activePlan_ = ""; //clear
327  // theIteratorStruct.commandIndex_ = -1; //clear
328 
329  continue; // resume workloop
330  }
331 
334  // handle running
335  // __COUT__ << "thinking.." << theIteratorStruct.running_ << " " <<
336  // theIteratorStruct.activePlan_ << " cmd=" <<
337  // theIteratorStruct.commandIndex_ << __E__;
338  try // inner catch: on error, pause the plan instead of killing the thread
339  {
340  if(theIteratorStruct.running_ &&
341  theIteratorStruct.activePlan_ !=
342  "") // important, because after errors, still "running" until halt
343  {
344  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
345  {
346  // initialize the running plan
347  // commandIndex_ stays -1 until loading succeeds, so a failure
348  // here re-enters this block on retry instead of skipping it
349 
350  __COUT__ << "Get commands" << __E__;
351 
352  theIteratorStruct.cfgMgr_
353  ->init(); // completely reset to re-align with any changes
354 
355  if(theIteratorStruct.activePlan_ ==
356  Iterator::RESERVED_GEN_PLAN_NAME)
357  {
358  __COUT__ << "Using generated plan..." << __E__;
359  theIteratorStruct.onlyConfigIfNotConfigured_ =
360  iterator->genKeepConfiguration_;
361  theIteratorStruct.commands_ =
362  generateIterationPlan(iterator->genFsmName_,
363  iterator->genConfigAlias_,
364  iterator->genPlanDurationSeconds_,
365  iterator->genPlanNumberOfRuns_);
366  }
367  else
368  {
369  __COUT__ << "Getting iterator table..." << __E__;
370  itConfig =
371  theIteratorStruct.cfgMgr_->__GET_CONFIG__(IterateTable);
372  theIteratorStruct.onlyConfigIfNotConfigured_ = false;
373  theIteratorStruct.commands_ = itConfig->getPlanCommands(
374  theIteratorStruct.cfgMgr_, theIteratorStruct.activePlan_);
375  }
376 
377  // reset commandIteration counts and any label stacks left by a
378  // plan halted mid-loop
379  theIteratorStruct.commandIterations_.clear();
380  theIteratorStruct.stepIndexStack_.clear();
381  theIteratorStruct.stepLabelStack_.clear();
382  theIteratorStruct.outputFiles_.clear();
383  for(auto& command : theIteratorStruct.commands_)
384  {
385  theIteratorStruct.commandIterations_.push_back(0);
386  __COUT__ << "command " << command.type_ << __E__;
387  __COUT__ << "table "
388  << IterateTable::commandToTableMap_.at(command.type_)
389  << __E__;
390  __COUT__ << "param count = " << command.params_.size()
391  << __E__;
392 
393  for(auto& param : command.params_)
394  {
395  __COUT__ << "\t param " << param.first << " : "
396  << param.second << __E__;
397  }
398  }
399 
400  theIteratorStruct.originalTrackChanges_ =
401  ConfigurationInterface::isVersionTrackingEnabled();
402  theIteratorStruct.originalConfigGroup_ =
403  theIteratorStruct.cfgMgr_->getActiveGroupName();
404  theIteratorStruct.originalConfigKey_ =
405  theIteratorStruct.cfgMgr_->getActiveGroupKey();
406 
407  __COUT__ << "originalTrackChanges "
408  << theIteratorStruct.originalTrackChanges_ << __E__;
409  __COUT__ << "originalConfigGroup "
410  << theIteratorStruct.originalConfigGroup_ << __E__;
411  __COUT__ << "originalConfigKey "
412  << theIteratorStruct.originalConfigKey_ << __E__;
413 
414  theIteratorStruct.commandIndex_ = 0;
415 
416  } // end initial section
417 
418  if(!theIteratorStruct.commandBusy_)
419  {
420  if(theIteratorStruct.commandIndex_ <
421  theIteratorStruct.commands_.size())
422  {
423  // execute command
424  theIteratorStruct.commandBusy_ = true;
425 
426  __COUT__ << "Iterator starting command "
427  << theIteratorStruct.commandIndex_ + 1 << ": "
428  << theIteratorStruct
429  .commands_[theIteratorStruct.commandIndex_]
430  .type_
431  << __E__;
432  __COUT__ << "Iterator starting command "
433  << theIteratorStruct.commandIndex_ + 1 << ": "
434  << theIteratorStruct
435  .commands_[theIteratorStruct.commandIndex_]
436  .type_
437  << __E__;
438 
439  iterator->startCommand(&theIteratorStruct);
440  }
441  else if(theIteratorStruct.commandIndex_ ==
442  theIteratorStruct.commands_.size()) // Done!
443  {
444  __COUT__ << "Finished Iteration Plan '"
445  << theIteratorStruct.activePlan_ << __E__;
446  __COUT__ << "Finished Iteration Plan '"
447  << theIteratorStruct.activePlan_ << __E__;
448 
449  __COUT__ << "Reverting track changes." << __E__;
450  ConfigurationInterface::setVersionTrackingEnabled(
451  theIteratorStruct.originalTrackChanges_);
452 
453  //l
454  __COUT__ << "Activating original group..." << __E__;
455  try
456  {
457  theIteratorStruct.cfgMgr_->activateTableGroup(
458  theIteratorStruct.originalConfigGroup_,
459  theIteratorStruct.originalConfigKey_);
460  }
461  catch(...)
462  {
463  __COUT_WARN__ << "Original group could not be activated."
464  << __E__;
465  }
466 
467  // leave FSM halted
468  __COUT__ << "Completing Iteration Plan and cleaning up..."
469  << __E__;
470 
471  iterator->haltIterator(
472  iterator, &theIteratorStruct, true /* doNotHaltFSM */);
473  }
474  }
475  else if(theIteratorStruct.commandBusy_)
476  {
477  // check for command completion
478  if(iterator->checkCommand(&theIteratorStruct))
479  {
480  theIteratorStruct.commandBusy_ = false; // command complete
481 
482  ++theIteratorStruct.commandIndex_;
483 
484  __COUT__ << "Ready for next command. Done with "
485  << theIteratorStruct.commandIndex_ << " of "
486  << theIteratorStruct.commands_.size() << __E__;
487  __COUT__ << "Iterator ready for next command. Done with "
488  << theIteratorStruct.commandIndex_ << " of "
489  << theIteratorStruct.commands_.size() << __E__;
490  }
491 
492  // Note: check command gets one shot to resume
493  if(theIteratorStruct.doResumeAction_) // end resume action
494  theIteratorStruct.doResumeAction_ = false;
495  }
496 
497  } // end running
498  else
499  sleep(1); // when inactive sleep a lot
500 
501  } // end inner try
502  catch(const std::runtime_error& e)
503  {
504  __COUT_ERR__ << "Iterator command error (will pause for retry): "
505  << e.what() << __E__;
506  theIteratorStruct.commandBusy_ = false;
507 
508  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
509  iterator->activePlanIsRunning_ = false;
510  iterator->errorMessage_ =
511  std::string("Error at command ") +
512  std::to_string(theIteratorStruct.commandIndex_) + " (" +
513  (theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size()
514  ? theIteratorStruct.commands_[theIteratorStruct.commandIndex_]
515  .type_
516  : "?") +
517  "): " + e.what();
518  }
519  catch(...)
520  {
521  __COUT_ERR__ << "Iterator unknown error (will pause for retry)." << __E__;
522  theIteratorStruct.commandBusy_ = false;
523 
524  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
525  iterator->activePlanIsRunning_ = false;
526  iterator->errorMessage_ =
527  std::string("Unknown error at command ") +
528  std::to_string(theIteratorStruct.commandIndex_) + " (" +
529  (theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size()
530  ? theIteratorStruct.commands_[theIteratorStruct.commandIndex_]
531  .type_
532  : "?") +
533  ")";
534  }
535 
538 
539  // __COUT__ << "end loop.." << theIteratorStruct.running_ << " " <<
540  // theIteratorStruct.activePlan_ << " cmd=" <<
541  // theIteratorStruct.commandIndex_ << __E__;
542  } /* code */
543  } //end try/catch
544  catch(const std::runtime_error& e) //insert info about the state of the iterator
545  {
546  if(theIteratorStruct.activePlan_ != "")
547  {
548  __SS__ << "The active Iterator plan name is '"
549  << theIteratorStruct.activePlan_
550  << "'... Here was the error: " << e.what() << __E__;
551  __SS_THROW__;
552  }
553  else
554  throw;
555  }
556 
557  iterator->workloopRunning_ = false; // if we ever exit
558 } //end IteratorWorkLoop()
559 catch(const std::runtime_error& e)
560 {
561  __SS__ << "Encountered error in Iterator thread. Here is the error:\n"
562  << e.what() << __E__;
563  __COUT_ERR__ << ss.str();
564 
565  // lockout the messages array for the remainder of the scope
566  // this guarantees the reading thread can safely access the messages
567  if(iterator->theSupervisor_->VERBOSE_MUTEX)
568  __COUT__ << "Waiting for iterator access" << __E__;
569  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
570  if(iterator->theSupervisor_->VERBOSE_MUTEX)
571  __COUT__ << "Have iterator access" << __E__;
572 
573  iterator->workloopRunning_ = false; // if we ever exit
574  iterator->errorMessage_ = ss.str();
575 }
576 catch(...)
577 {
578  __SS__ << "Encountered unknown error in Iterator thread." << __E__;
579  try
580  {
581  throw;
582  } //one more try to printout extra info
583  catch(const std::exception& e)
584  {
585  ss << "Exception message: " << e.what();
586  }
587  catch(...)
588  {
589  }
590  __COUT_ERR__ << ss.str();
591 
592  // lockout the messages array for the remainder of the scope
593  // this guarantees the reading thread can safely access the messages
594  if(iterator->theSupervisor_->VERBOSE_MUTEX)
595  __COUT__ << "Waiting for iterator access" << __E__;
596  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
597  if(iterator->theSupervisor_->VERBOSE_MUTEX)
598  __COUT__ << "Have iterator access" << __E__;
599 
600  iterator->workloopRunning_ = false; // if we ever exit
601  iterator->errorMessage_ = ss.str();
602 } // end IteratorWorkLoop() exception handling
603 
604 //==============================================================================
605 void Iterator::startCommand(IteratorWorkLoopStruct* iteratorStruct)
606 try
607 {
608  {
609  int i = 0;
610  for(const auto& depthIteration : iteratorStruct->stepIndexStack_)
611  {
612  __COUT__ << i++ << ":" << depthIteration << __E__;
613  }
614  }
615 
616  // should be mutually exclusive with GatewaySupervisor main thread state machine
617  // accesses lockout the messages array for the remainder of the scope this
618  // guarantees the reading thread can safely access the messages
619  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
620  __COUT__ << "Waiting for FSM access" << __E__;
621  std::lock_guard<std::mutex> lock(
622  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
623  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
624  __COUT__ << "Have FSM access" << __E__;
625 
626  // for out of range, throw exception - should never happen
627  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
628  {
629  __SS__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
630  << " in size = " << iteratorStruct->commands_.size() << __E__;
631  __SS_THROW__;
632  }
633 
634  // increment iteration count for command
635  ++iteratorStruct->commandIterations_[iteratorStruct->commandIndex_];
636 
637  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
638  const std::string& targetSubsystem =
639  iteratorStruct->commands_[iteratorStruct->commandIndex_].targetSubsystem_;
640 
641  // fail loudly rather than silently acting on the local system
642  if(targetSubsystem.size() &&
643  (type == IterateTable::COMMAND_ACTIVATE_ALIAS ||
644  type == IterateTable::COMMAND_ACTIVATE_GROUP ||
645  type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP ||
646  type == IterateTable::COMMAND_RUN || type == IterateTable::COMMAND_WAIT ||
647  type == IterateTable::COMMAND_BEGIN_LABEL ||
648  type == IterateTable::COMMAND_REPEAT_LABEL ||
649  type == IterateTable::COMMAND_CHOOSE_FSM))
650  {
651  __SS__ << "Command type '" << type << "' can not target a remote subsystem ('"
652  << targetSubsystem << "'). Clear the TargetSubsystem for this command."
653  << __E__;
654  __SS_THROW__;
655  }
656 
657  if(type == IterateTable::COMMAND_BEGIN_LABEL)
658  {
659  return startCommandBeginLabel(iteratorStruct);
660  }
661  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
662  {
663  return startCommandChooseFSM(
664  iteratorStruct,
665  iteratorStruct->commands_[iteratorStruct->commandIndex_]
666  .params_[IterateTable::commandChooseFSMParams_.NameOfFSM_]);
667  }
668  else if(type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP)
669  {
670  if(targetSubsystem.size())
671  {
672  // use the remote subsystem's own active config group, not the top-level's
673  std::string groupAlias;
674  {
675  std::lock_guard<std::mutex> lock(
676  iteratorStruct->theIterator_->theSupervisor_
677  ->remoteGatewayAppsMutex_);
678  const auto& remoteApp = iteratorStruct->theIterator_->theSupervisor_
679  ->remoteGatewayApps_[findRemoteGatewayApp(
680  iteratorStruct, targetSubsystem)];
681  if(remoteApp.activeConfigGroupName == "" ||
682  remoteApp.activeConfigGroupKey.isInvalid())
683  {
684  __SS__ << "Remote subsystem '" << targetSubsystem
685  << "' has not reported an active configuration group yet, "
686  "so CONFIGURE_ACTIVE_GROUP can not be applied to it."
687  << __E__;
688  __SS_THROW__;
689  }
690  groupAlias = "GROUP:" + remoteApp.activeConfigGroupName + ":" +
691  remoteApp.activeConfigGroupKey.toString();
692  }
693  return startRemoteCommandConfigure(
694  iteratorStruct, groupAlias, targetSubsystem);
695  }
696  return startCommandConfigureActive(iteratorStruct);
697  }
698  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS)
699  {
700  const std::string& systemAlias =
701  iteratorStruct->commands_[iteratorStruct->commandIndex_]
702  .params_[IterateTable::commandConfigureAliasParams_.SystemAlias_];
703  if(targetSubsystem.size())
704  return startRemoteCommandConfigure(
705  iteratorStruct, systemAlias, targetSubsystem);
706  return startCommandConfigureAlias(iteratorStruct, systemAlias);
707  }
708  else if(type == IterateTable::COMMAND_CONFIGURE_GROUP)
709  {
710  if(targetSubsystem.size())
711  {
712  auto& params =
713  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_;
714  std::string groupAlias =
715  "GROUP:" + params[IterateTable::commandConfigureGroupParams_.GroupName_] +
716  ":" + params[IterateTable::commandConfigureGroupParams_.GroupKey_];
717  return startRemoteCommandConfigure(
718  iteratorStruct, groupAlias, targetSubsystem);
719  }
720  return startCommandConfigureGroup(iteratorStruct);
721  }
722  else if(type == IterateTable::COMMAND_ACTIVATE_ALIAS)
723  {
724  ConfigurationManagerRW* cfgMgr = iteratorStruct->cfgMgr_;
725  std::pair<std::string, TableGroupKey> newActiveGroup =
726  cfgMgr->getTableGroupFromAlias(
727  iteratorStruct->commands_[iteratorStruct->commandIndex_]
728  .params_[IterateTable::commandActivateAliasParams_.SystemAlias_]);
729  cfgMgr->loadTableGroup(
730  newActiveGroup.first, newActiveGroup.second, true /*activate*/);
731  }
732  else if(type == IterateTable::COMMAND_ACTIVATE_GROUP)
733  {
734  ConfigurationManagerRW* cfgMgr = iteratorStruct->cfgMgr_;
735  cfgMgr->loadTableGroup(
736  iteratorStruct->commands_[iteratorStruct->commandIndex_]
737  .params_[IterateTable::commandActivateGroupParams_.GroupName_],
738  TableGroupKey(
739  iteratorStruct->commands_[iteratorStruct->commandIndex_]
740  .params_[IterateTable::commandActivateGroupParams_.GroupKey_]),
741  true /*activate*/);
742  }
743  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
744  {
745  if(targetSubsystem.size())
746  return startRemoteCommandMacro(iteratorStruct, true /*isFEMacro*/);
747  return startCommandMacro(iteratorStruct, true /*isFEMacro*/);
748  }
749  else if(type == IterateTable::COMMAND_EXECUTE_MM_MACRO)
750  {
751  if(targetSubsystem.size())
752  return startRemoteCommandMacro(iteratorStruct, false /*isFEMacro*/);
753  return startCommandMacro(iteratorStruct, false /*isFEMacro*/);
754  }
755  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
756  {
757  return startCommandModifyActive(iteratorStruct);
758  }
759  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
760  {
761  return startCommandRepeatLabel(iteratorStruct);
762  }
763  else if(type == IterateTable::COMMAND_RUN)
764  {
765  return startCommandRun(iteratorStruct);
766  }
767  else if(type == IterateTable::COMMAND_WAIT)
768  {
769  return startCommandWait(iteratorStruct);
770  }
771  else if(type == IterateTable::COMMAND_START)
772  {
773  if(targetSubsystem.size())
774  return startRemoteCommandFSMTransition(
775  iteratorStruct,
776  RunControlStateMachine::START_TRANSITION_NAME,
777  targetSubsystem);
778  return startCommandFSMTransition(iteratorStruct,
779  RunControlStateMachine::START_TRANSITION_NAME);
780  }
781  else if(type == IterateTable::COMMAND_STOP)
782  {
783  if(targetSubsystem.size())
784  return startRemoteCommandFSMTransition(
785  iteratorStruct,
786  RunControlStateMachine::STOP_TRANSITION_NAME,
787  targetSubsystem);
788  return startCommandFSMTransition(iteratorStruct,
789  RunControlStateMachine::STOP_TRANSITION_NAME);
790  }
791  else if(type == IterateTable::COMMAND_PAUSE)
792  {
793  if(targetSubsystem.size())
794  return startRemoteCommandFSMTransition(
795  iteratorStruct,
796  RunControlStateMachine::PAUSE_TRANSITION_NAME,
797  targetSubsystem);
798  return startCommandFSMTransition(iteratorStruct,
799  RunControlStateMachine::PAUSE_TRANSITION_NAME);
800  }
801  else if(type == IterateTable::COMMAND_RESUME)
802  {
803  if(targetSubsystem.size())
804  return startRemoteCommandFSMTransition(
805  iteratorStruct,
806  RunControlStateMachine::RESUME_TRANSITION_NAME,
807  targetSubsystem);
808  return startCommandFSMTransition(iteratorStruct,
809  RunControlStateMachine::RESUME_TRANSITION_NAME);
810  }
811  else if(type == IterateTable::COMMAND_HALT)
812  {
813  if(targetSubsystem.size())
814  return startRemoteCommandFSMTransition(
815  iteratorStruct,
816  RunControlStateMachine::HALT_TRANSITION_NAME,
817  targetSubsystem);
818  return startCommandFSMTransition(iteratorStruct,
819  RunControlStateMachine::HALT_TRANSITION_NAME);
820  }
821  else
822  {
823  __SS__ << "Failed attempt to start unrecognized command type = " << type << __E__;
824  __COUT_ERR__ << ss.str();
825  __SS_THROW__;
826  }
827 }
828 catch(...)
829 {
830  __COUT__ << "Error caught. Reverting track changes." << __E__;
831  ConfigurationInterface::setVersionTrackingEnabled(
832  iteratorStruct->originalTrackChanges_);
833 
834  __COUT__ << "Activating original group..." << __E__;
835  try
836  {
837  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
838  iteratorStruct->originalConfigKey_);
839  }
840  catch(...)
841  {
842  __COUT_WARN__ << "Original group could not be activated." << __E__;
843  }
844  throw;
845 } // end startCommand()
846 
847 //==============================================================================
851 bool Iterator::checkCommand(IteratorWorkLoopStruct* iteratorStruct)
852 try
853 {
854  // for out of range, return done
855  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
856  {
857  __COUT__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
858  << " in size = " << iteratorStruct->commands_.size() << __E__;
859  return true;
860  }
861 
862  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
863  const std::string& targetSubsystem =
864  iteratorStruct->commands_[iteratorStruct->commandIndex_].targetSubsystem_;
865 
866  if(type == IterateTable::COMMAND_BEGIN_LABEL)
867  {
868  // do nothing
869  return true;
870  }
871  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
872  {
873  // do nothing
874  return true;
875  }
876  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS ||
877  type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP ||
878  type == IterateTable::COMMAND_CONFIGURE_GROUP)
879  {
880  if(targetSubsystem.size())
881  return checkRemoteCommandConfigure(iteratorStruct, targetSubsystem);
882  return checkCommandConfigure(iteratorStruct);
883  }
884  else if(type == IterateTable::COMMAND_ACTIVATE_ALIAS ||
885  type == IterateTable::COMMAND_ACTIVATE_GROUP)
886  {
887  // do nothing
888  return true;
889  }
890  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
891  {
892  if(targetSubsystem.size())
893  return checkRemoteCommandMacro(iteratorStruct, true /*isFEMacro*/);
894  return checkCommandMacro(iteratorStruct, true /*isFEMacro*/);
895  }
896  else if(type == IterateTable::COMMAND_EXECUTE_MM_MACRO)
897  {
898  if(targetSubsystem.size())
899  return checkRemoteCommandMacro(iteratorStruct, false /*isFEMacro*/);
900  return checkCommandMacro(iteratorStruct, false /*isFEMacro*/);
901  }
902  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
903  {
904  // do nothing
905  return true;
906  }
907  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
908  {
909  // do nothing
910  return true;
911  }
912  else if(type == IterateTable::COMMAND_RUN)
913  {
914  return checkCommandRun(iteratorStruct);
915  }
916  else if(type == IterateTable::COMMAND_WAIT)
917  {
918  return checkCommandWait(iteratorStruct);
919  }
920  else if(type == IterateTable::COMMAND_START)
921  {
922  if(targetSubsystem.size())
923  return checkRemoteCommandFSMTransition(
924  iteratorStruct, "Running", targetSubsystem);
925  return checkCommandFSMTransition(iteratorStruct, "Running");
926  }
927  else if(type == IterateTable::COMMAND_STOP)
928  {
929  if(targetSubsystem.size())
930  return checkRemoteCommandFSMTransition(
931  iteratorStruct, "Configured", targetSubsystem);
932  return checkCommandFSMTransition(iteratorStruct, "Configured");
933  }
934  else if(type == IterateTable::COMMAND_PAUSE)
935  {
936  if(targetSubsystem.size())
937  return checkRemoteCommandFSMTransition(
938  iteratorStruct, "Paused", targetSubsystem);
939  return checkCommandFSMTransition(iteratorStruct, "Paused");
940  }
941  else if(type == IterateTable::COMMAND_RESUME)
942  {
943  if(targetSubsystem.size())
944  return checkRemoteCommandFSMTransition(
945  iteratorStruct, "Running", targetSubsystem);
946  return checkCommandFSMTransition(iteratorStruct, "Running");
947  }
948  else if(type == IterateTable::COMMAND_HALT)
949  {
950  if(targetSubsystem.size())
951  return checkRemoteCommandFSMTransition(
952  iteratorStruct,
953  RunControlStateMachine::HALTED_STATE_NAME,
954  targetSubsystem);
955  return checkCommandFSMTransition(iteratorStruct,
956  RunControlStateMachine::HALTED_STATE_NAME);
957  }
958  else
959  {
960  __SS__ << "Attempt to check unrecognized command type = " << type << __E__;
961  __COUT_ERR__ << ss.str();
962  __SS_THROW__;
963  }
964 }
965 catch(...)
966 {
967  __COUT__ << "Error caught. Reverting track changes." << __E__;
968  ConfigurationInterface::setVersionTrackingEnabled(
969  iteratorStruct->originalTrackChanges_);
970 
971  __COUT__ << "Activating original group..." << __E__;
972  try
973  {
974  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
975  iteratorStruct->originalConfigKey_);
976  }
977  catch(...)
978  {
979  __COUT_WARN__ << "Original group could not be activated." << __E__;
980  }
981 
982  throw;
983 } // end checkCommand()
984 
985 //==============================================================================
986 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct* iteratorStruct,
987  const std::string& fsmName)
988 {
989  __COUT__ << "fsmName " << fsmName << __E__;
990 
991  iteratorStruct->fsmName_ = fsmName;
992  iteratorStruct->theIterator_->lastFsmName_ = fsmName;
993 
994  // Translate fsmName
995  // to gives us run alias (fsmRunAlias_) and next run number (fsmNextRunNumber_)
996 
997  // CAREFUL?? Threads
998 
1000 
1001  iteratorStruct->fsmRunAlias_ = "Run"; // default to "Run"
1002 
1003  // get stateMachineAliasFilter if possible
1004  ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorTableNode(
1005  iteratorStruct->theIterator_->theSupervisor_->getContextUID(),
1006  iteratorStruct->theIterator_->theSupervisor_->getSupervisorUID());
1007 
1008  if(!configLinkNode.isDisconnected())
1009  {
1010  try // for backwards compatibility
1011  {
1012  ConfigurationTree fsmLinkNode =
1013  configLinkNode.getNode("LinkToStateMachineTable");
1014  if(!fsmLinkNode.isDisconnected())
1015  iteratorStruct->fsmRunAlias_ =
1016  fsmLinkNode.getNode(fsmName + "/RunDisplayAlias")
1017  .getValue<std::string>();
1018  else
1019  __COUT_INFO__ << "FSM Link disconnected." << __E__;
1020  }
1021  catch(std::runtime_error& e)
1022  {
1023  //__COUT_INFO__ << e.what() << __E__;
1024  __COUT_INFO__
1025  << "No state machine Run alias. Ignoring and assuming alias of '"
1026  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
1027  }
1028  catch(...)
1029  {
1030  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
1031 
1032  __COUT_INFO__
1033  << "No state machine Run alias. Ignoring and assuming alias of '"
1034  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
1035  }
1036  }
1037  else
1038  __COUT_INFO__ << "FSM Link disconnected." << __E__;
1039 
1040  __COUT__ << "fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
1041 
1043 
1044  iteratorStruct->fsmNextRunNumber_ =
1045  iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
1046  iteratorStruct->fsmName_);
1047 
1048  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1049  .getCurrentStateName() == "Running" ||
1050  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1051  .getCurrentStateName() == "Paused")
1052  --iteratorStruct->fsmNextRunNumber_; // current run number is one back
1053 
1054  __COUT__ << "fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
1055 } // end startCommandChooseFSM()
1056 
1057 //==============================================================================
1059 bool Iterator::haltIterator(Iterator* iterator,
1060  IteratorWorkLoopStruct* iteratorStruct /* = 0 */,
1061  bool doNotHaltFSM /* = false */)
1062 
1063 {
1064  GatewaySupervisor* theSupervisor = iterator->theSupervisor_;
1065  const std::string& fsmName = iterator->lastFsmName_;
1066 
1067  std::vector<std::string> fsmCommandParameters;
1068  std::string errorStr = "";
1069  std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
1070 
1071  __COUTV__(currentState);
1072 
1073  bool haltAttempted = true;
1074  if(doNotHaltFSM)
1075  {
1076  __COUT_INFO__ << "Iterator is leaving FSM in current state: " << currentState
1077  << ". If this is undesireable, add a Halt command, for example, to "
1078  "the end of your Iteration plan."
1079  << __E__;
1080  haltAttempted = false;
1081  }
1082  else if(currentState == RunControlStateMachine::INITIAL_STATE_NAME ||
1083  currentState == RunControlStateMachine::HALTED_STATE_NAME)
1084  {
1085  __COUT__ << "Do nothing. Already halted." << __E__;
1086  haltAttempted = false;
1087  }
1088  else if(currentState == "Running")
1089  errorStr = theSupervisor->attemptStateMachineTransition(
1090  0,
1091  0,
1092  RunControlStateMachine::ABORT_TRANSITION_NAME,
1093  fsmName,
1094  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1095  WebUsers::DEFAULT_ITERATOR_USERNAME,
1096  fsmCommandParameters);
1097  else
1098  errorStr = theSupervisor->attemptStateMachineTransition(
1099  0,
1100  0,
1101  RunControlStateMachine::HALT_TRANSITION_NAME,
1102  fsmName,
1103  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1104  WebUsers::DEFAULT_ITERATOR_USERNAME,
1105  fsmCommandParameters);
1106 
1107  if(haltAttempted)
1108  {
1109  if(errorStr != "")
1110  {
1111  __SS__ << "Iterator failed to halt because of the following error: "
1112  << errorStr;
1113  __SS_THROW__;
1114  }
1115 
1116  // else successfully launched
1117  __COUT__ << "FSM in transition = "
1118  << theSupervisor->theStateMachine_.isInTransition() << __E__;
1119  __COUT__ << "halting state machine launched." << __E__;
1120  }
1121 
1122  // finish up cleanup of the iterator
1123  __COUT__ << "Conducting Iterator cleanup." << __E__;
1124 
1125  if(iteratorStruct)
1126  {
1127  __COUT__ << "Reverting track changes." << __E__;
1128  ConfigurationInterface::setVersionTrackingEnabled(
1129  iteratorStruct->originalTrackChanges_);
1130 
1131  if(!doNotHaltFSM)
1132  {
1133  __COUT__ << "Activating original group..." << __E__;
1134  try
1135  {
1136  iteratorStruct->cfgMgr_->activateTableGroup(
1137  iteratorStruct->originalConfigGroup_,
1138  iteratorStruct->originalConfigKey_);
1139  }
1140  catch(...)
1141  {
1142  __COUT_WARN__ << "Original group could not be activated." << __E__;
1143  }
1144  }
1145  }
1146 
1147  // lockout the messages array for the remainder of the scope
1148  // this guarantees the reading thread can safely access the messages
1149  if(iterator->theSupervisor_->VERBOSE_MUTEX)
1150  __COUT__ << "Waiting for iterator access" << __E__;
1151  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
1152  if(iterator->theSupervisor_->VERBOSE_MUTEX)
1153  __COUT__ << "Have iterator access" << __E__;
1154 
1155  iterator->activePlanIsRunning_ = false;
1156  iterator->iteratorBusy_ = false;
1157 
1158  // clear
1159  iterator->activePlanName_ = "";
1160  iterator->activeCommandIndex_ = -1;
1161 
1162  if(iteratorStruct)
1163  {
1164  __COUT__ << "Iterator cleanup complete of plan '" << iteratorStruct->activePlan_
1165  << "' at command index " << iteratorStruct->commandIndex_ << ". "
1166  << __E__;
1167 
1168  iterator->lastFinishedPlanName_ = iteratorStruct->activePlan_;
1169  // publish the output files written during this plan for the GUI completion popup
1170  iterator->lastPlanOutputFiles_ = iteratorStruct->outputFiles_;
1171  for(const auto& f : iterator->lastPlanOutputFiles_)
1172  __COUT_INFO__ << "Plan '" << iteratorStruct->activePlan_ << "' output file ("
1173  << (f.subsystem == "" ? std::string("Self") : f.subsystem)
1174  << ", " << f.label << "): " << f.path << __E__;
1175  FILE* fp = fopen((ITERATOR_PLAN_HISTORY_FILENAME).c_str(), "w");
1176  if(fp)
1177  {
1178  fprintf(fp, "%s\n", iterator->lastStartedPlanName_.c_str());
1179  fprintf(fp, "%s\n", iterator->lastFinishedPlanName_.c_str());
1180  fclose(fp);
1181  }
1182  else
1183  __COUT_WARN__ << "Could not open Iterator history file: "
1184  << ITERATOR_PLAN_HISTORY_FILENAME << __E__;
1185 
1186  iteratorStruct->activePlan_ = ""; // clear
1187  iteratorStruct->commandIndex_ = -1; // clear
1188  }
1189 
1190  return haltAttempted;
1191 } // end haltIterator()
1192 
1193 //==============================================================================
1194 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct* iteratorStruct)
1195 {
1196  __COUT__ << "Entering label '"
1197  << iteratorStruct->commands_[iteratorStruct->commandIndex_]
1198  .params_[IterateTable::commandBeginLabelParams_.Label_]
1199  << "'..." << std::endl;
1200 
1201  // add new step index to stack
1202  iteratorStruct->stepIndexStack_.push_back(0);
1203  iteratorStruct->stepLabelStack_.push_back(
1204  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1205  .params_[IterateTable::commandBeginLabelParams_.Label_]);
1206 } // end startCommandBeginLabel()
1207 
1208 //==============================================================================
1209 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct* iteratorStruct)
1210 {
1211  // search for first matching label backward and set command to there
1212 
1213  int numOfRepetitions;
1214  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1215  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_]
1216  .c_str(),
1217  "%d",
1218  &numOfRepetitions);
1219  __COUT__ << "numOfRepetitions remaining = " << numOfRepetitions << __E__;
1220 
1221  char repStr[200];
1222 
1223  if(numOfRepetitions <= 0)
1224  {
1225  // write original number of repetitions value back
1226  sprintf(repStr, "%d", iteratorStruct->stepIndexStack_.back());
1227  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1228  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
1229  repStr; // re-store as string
1230 
1231  // remove step index from stack
1232  iteratorStruct->stepIndexStack_.pop_back();
1233  if(iteratorStruct->stepLabelStack_.size())
1234  iteratorStruct->stepLabelStack_.pop_back();
1235 
1236  return; // no more repetitions
1237  }
1238 
1239  --numOfRepetitions;
1240 
1241  // increment step index in stack
1242  ++(iteratorStruct->stepIndexStack_.back());
1243 
1244  unsigned int i;
1245  for(i = iteratorStruct->commandIndex_; i > 0;
1246  --i) // assume 0 is always the fallback option
1247  if(iteratorStruct->commands_[i].type_ == IterateTable::COMMAND_BEGIN_LABEL &&
1248  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1249  .params_[IterateTable::commandRepeatLabelParams_.Label_] ==
1250  iteratorStruct->commands_[i]
1251  .params_[IterateTable::commandBeginLabelParams_.Label_])
1252  break;
1253 
1254  sprintf(repStr, "%d", numOfRepetitions);
1255  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1256  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
1257  repStr; // re-store as string
1258 
1259  iteratorStruct->commandIndex_ = i;
1260  __COUT__ << "Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
1261 } // end startCommandRepeatLabel()
1262 
1263 //==============================================================================
1264 std::string Iterator::buildIteratorLogEntry(IteratorWorkLoopStruct* iteratorStruct)
1265 {
1266  std::string entry = "Iterator plan '" + iteratorStruct->activePlan_ + "'";
1267  if(!iteratorStruct->stepIndexStack_.empty())
1268  {
1269  entry += " [";
1270  for(size_t i = 0; i < iteratorStruct->stepIndexStack_.size(); ++i)
1271  {
1272  if(i)
1273  entry += ", ";
1274  if(i < iteratorStruct->stepLabelStack_.size() &&
1275  !iteratorStruct->stepLabelStack_[i].empty())
1276  entry += iteratorStruct->stepLabelStack_[i] + " ";
1277  entry += "pass " + std::to_string(iteratorStruct->stepIndexStack_[i]);
1278  }
1279  entry += "]";
1280  }
1281  return entry;
1282 }
1283 
1284 //==============================================================================
1285 void Iterator::startCommandRun(IteratorWorkLoopStruct* iteratorStruct)
1286 {
1287  __COUT__ << "startCommandRun " << __E__;
1288 
1289  iteratorStruct->runIsDone_ = false;
1290  iteratorStruct->fsmCommandParameters_.clear();
1291 
1292  bool writeToEcl = false;
1293  try
1294  {
1295  writeToEcl = iteratorStruct->commands_[iteratorStruct->commandIndex_].params_.at(
1296  IterateTable::commandRunParams_.WriteToECL_) == "1";
1297  }
1298  catch(...)
1299  {
1300  }
1301  iteratorStruct->theIterator_->theSupervisor_->activeStateMachineWriteToEcl_ =
1302  writeToEcl;
1303 
1304  std::string errorStr = "";
1305  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1306  ->theStateMachine_.getCurrentStateName();
1307 
1308  // execute first transition (may need two)
1309 
1310  __COUTTV__(iteratorStruct->theIterator_->genLogEntry_);
1311 
1312  if(currentState == RunControlStateMachine::CONFIGURED_STATE_NAME)
1313  errorStr =
1314  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1315  0,
1316  0,
1317  RunControlStateMachine::START_TRANSITION_NAME,
1318  iteratorStruct->fsmName_,
1319  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1320  WebUsers::DEFAULT_ITERATOR_USERNAME,
1321  iteratorStruct->fsmCommandParameters_,
1322  iteratorStruct->activePlan_ == Iterator::RESERVED_GEN_PLAN_NAME
1323  ? iteratorStruct->theIterator_->genLogEntry_
1324  : buildIteratorLogEntry(iteratorStruct));
1325  else
1326  errorStr = "Can only Run from the Configured state. The current state is " +
1327  currentState;
1328 
1329  if(errorStr != "")
1330  {
1331  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1332  __SS_THROW__;
1333  }
1334 
1335  // save original duration
1336  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1337  .params_[IterateTable::commandRunParams_.DurationInSeconds_]
1338  .c_str(),
1339  "%ld",
1340  &iteratorStruct->originalDurationInSeconds_);
1341  __COUTV__(iteratorStruct->originalDurationInSeconds_);
1342 
1343  // else successfully launched
1344  __COUT__
1345  << "FSM in transition = "
1346  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1347  << __E__;
1348  __COUT__ << "startCommandRun success." << __E__;
1349 } // end startCommandRun()
1350 
1351 //==============================================================================
1352 void Iterator::startCommandWait(IteratorWorkLoopStruct* iteratorStruct)
1353 {
1354  __COUT__ << "startCommandWait " << __E__;
1355 
1356  iteratorStruct->waitIsDone_ = false;
1357 
1358  // Get original duration
1359  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1360  .params_[IterateTable::commandWaitParams_.DurationInSeconds_]
1361  .c_str(),
1362  "%ld",
1363  &iteratorStruct->originalDurationInSeconds_);
1364  __COUTV__(iteratorStruct->originalDurationInSeconds_);
1365 
1366  __COUT__ << "startCommandWait success." << __E__;
1367 }
1368 
1369 //==============================================================================
1374 bool Iterator::commandSkipsIfConfigured(IteratorWorkLoopStruct* iteratorStruct)
1375 {
1376  if(iteratorStruct->onlyConfigIfNotConfigured_)
1377  return true;
1378  auto& params = iteratorStruct->commands_[iteratorStruct->commandIndex_].params_;
1379  auto it =
1380  params.find(IterateTable::commandConfigureAliasParams_.SkipIfAlreadyConfigured_);
1381  return it != params.end() && (it->second == "1" || it->second == "Yes" ||
1382  it->second == "True" || it->second == "On");
1383 } // end commandSkipsIfConfigured()
1384 
1385 //==============================================================================
1386 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct* iteratorStruct)
1387 {
1388  __COUT__ << "startCommandConfigureActive " << __E__;
1389 
1390  // steps:
1391  // get active config group
1392  // transition to configure with parameters describing group
1393 
1394  std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
1395  TableGroupKey key = iteratorStruct->cfgMgr_->getActiveGroupKey();
1396 
1397  __COUT__ << "group " << group << __E__;
1398  __COUT__ << "key " << key << __E__;
1399 
1400  // create special alias for this group using : separators
1401 
1402  std::stringstream systemAlias;
1403  systemAlias << "GROUP:" << group << ":" << key;
1404  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
1405 } // end startCommandConfigureActive()
1406 
1407 //==============================================================================
1408 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct* iteratorStruct)
1409 {
1410  __COUT__ << "startCommandConfigureGroup " << __E__;
1411 
1412  // steps:
1413  // transition to configure with parameters describing group
1414 
1415  std::string group =
1416  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1417  .params_[IterateTable::commandConfigureGroupParams_.GroupName_];
1418  TableGroupKey key =
1419  TableGroupKey(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1420  .params_[IterateTable::commandConfigureGroupParams_.GroupKey_]);
1421 
1422  __COUT__ << "group " << group << __E__;
1423  __COUT__ << "key " << key << __E__;
1424 
1425  // create special alias for this group using : separators
1426 
1427  std::stringstream systemAlias;
1428  systemAlias << "GROUP:" << group << ":" << key;
1429  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
1430 } // end startCommandConfigureGroup()
1431 
1432 //==============================================================================
1433 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct* iteratorStruct,
1434  const std::string& systemAlias)
1435 {
1436  __COUT__ << "systemAlias " << systemAlias << __E__;
1437 
1438  iteratorStruct->fsmCommandParameters_.clear();
1439  iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
1440 
1441  std::string errorStr = "";
1442  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1443  ->theStateMachine_.getCurrentStateName();
1444 
1445  // execute first transition (may need two in conjunction with checkCommandConfigure())
1446 
1447  if(currentState == RunControlStateMachine::INITIAL_STATE_NAME)
1448  errorStr =
1449  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1450  0,
1451  0,
1452  RunControlStateMachine::INIT_TRANSITION_NAME,
1453  iteratorStruct->fsmName_,
1454  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1455  WebUsers::DEFAULT_ITERATOR_USERNAME,
1456  iteratorStruct->fsmCommandParameters_);
1457  else if(currentState == RunControlStateMachine::HALTED_STATE_NAME)
1458  errorStr =
1459  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1460  0,
1461  0,
1462  RunControlStateMachine::CONFIGURE_TRANSITION_NAME,
1463  iteratorStruct->fsmName_,
1464  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1465  WebUsers::DEFAULT_ITERATOR_USERNAME,
1466  iteratorStruct->fsmCommandParameters_);
1467  else if(currentState == RunControlStateMachine::CONFIGURED_STATE_NAME ||
1468  currentState == RunControlStateMachine::FAILED_STATE_NAME)
1469  {
1470  if(commandSkipsIfConfigured(iteratorStruct) &&
1471  currentState != RunControlStateMachine::FAILED_STATE_NAME)
1472  __COUT_INFO__ << "Already configured and SkipIfAlreadyConfigured is set, so "
1473  "leaving the configuration as-is."
1474  << __E__;
1475  else
1476  errorStr = iteratorStruct->theIterator_->theSupervisor_
1477  ->attemptStateMachineTransition(
1478  0,
1479  0,
1480  RunControlStateMachine::HALT_TRANSITION_NAME,
1481  iteratorStruct->fsmName_,
1482  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1483  WebUsers::DEFAULT_ITERATOR_USERNAME,
1484  iteratorStruct->fsmCommandParameters_);
1485  }
1486  else
1487  errorStr =
1488  "Can only Configure from the Initial or Halted state. The current state is " +
1489  currentState;
1490 
1491  if(errorStr != "")
1492  {
1493  __SS__ << "Iterator failed to configure with system alias '"
1494  << (iteratorStruct->fsmCommandParameters_.size()
1495  ? iteratorStruct->fsmCommandParameters_[0]
1496  : "UNKNOWN")
1497  << "' because of the following error: " << errorStr;
1498  __SS_THROW__;
1499  }
1500 
1501  // else successfully launched
1502  __COUT__
1503  << "FSM in transition = "
1504  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1505  << __E__;
1506  __COUT__ << "startCommandConfigureAlias success." << __E__;
1507 } // end startCommandConfigureAlias()
1508 
1509 //==============================================================================
1510 void Iterator::startCommandFSMTransition(IteratorWorkLoopStruct* iteratorStruct,
1511  const std::string& transitionCommand)
1512 {
1513  __COUTV__(transitionCommand);
1514 
1515  iteratorStruct->fsmCommandParameters_.clear();
1516 
1517  std::string errorStr = "";
1518  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1519  ->theStateMachine_.getCurrentStateName();
1520 
1521  // execute first transition (may need two in conjunction with checkCommandConfigure())
1522 
1523  __COUTV__(currentState);
1524 
1525  errorStr =
1526  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1527  0,
1528  0,
1529  transitionCommand,
1530  iteratorStruct->fsmName_,
1531  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1532  WebUsers::DEFAULT_ITERATOR_USERNAME,
1533  iteratorStruct->fsmCommandParameters_,
1534  (transitionCommand == RunControlStateMachine::START_TRANSITION_NAME)
1535  ? (iteratorStruct->activePlan_ == Iterator::RESERVED_GEN_PLAN_NAME
1536  ? iteratorStruct->theIterator_->genLogEntry_
1537  : buildIteratorLogEntry(iteratorStruct))
1538  : (transitionCommand == RunControlStateMachine::STOP_TRANSITION_NAME)
1539  ? "Stopped by iterator: " + buildIteratorLogEntry(iteratorStruct)
1540  : "");
1541 
1542  if(errorStr != "")
1543  {
1544  __SS__ << "Iterator failed to " << transitionCommand << " with ";
1545  if(iteratorStruct->fsmCommandParameters_.size() == 0)
1546  ss << "no parameters ";
1547  else
1548  ss << "parameters '"
1549  << StringMacros::vectorToString(iteratorStruct->fsmCommandParameters_)
1550  << "' ";
1551  ss << "' because of the following error: " << errorStr;
1552  __SS_THROW__;
1553  }
1554 
1555  // else successfully launched
1556  __COUT__
1557  << "FSM in transition = "
1558  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1559  << __E__;
1560  __COUT__ << "startCommandFSMTransition success." << __E__;
1561 } // end startCommandFSMTransition()
1562 
1563 //==============================================================================
1564 void Iterator::startCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1565  bool isFrontEndMacro)
1566 {
1567  // Steps:
1568  // 4 parameters CommandExecuteFEMacroParams:
1569  // //targets
1570  // const std::string FEMacroName_ = "FEMacroName";
1571  // //macro parameters (table/groupID)
1572 
1573  const std::string& macroName =
1574  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1575  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1576  const std::string& enableSavingOutput =
1577  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1578  .params_[IterateTable::commandExecuteMacroParams_.EnableSavingOutput_];
1579  const std::string& outputFilePath =
1580  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1581  .params_[IterateTable::commandExecuteMacroParams_.OutputFilePath_];
1582  const std::string& outputFileRadix =
1583  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1584  .params_[IterateTable::commandExecuteMacroParams_.OutputFileRadix_];
1585  const std::string inputArgs = applyStepIndexToMacroArgs(
1586  iteratorStruct,
1587  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1588  .params_[IterateTable::commandExecuteMacroParams_.MacroArgumentString_],
1589  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1590  .params_[IterateTable::commandExecuteMacroParams_.MacroArgumentLabels_]);
1591  iteratorStruct->macroArgsSummary_ =
1592  macroArgsSummary(inputArgs); // for output file labels
1593 
1594  __COUTV__(macroName);
1595  __COUTV__(enableSavingOutput);
1596  __COUTV__(outputFilePath);
1597  __COUTV__(outputFileRadix);
1598  __COUTV__(inputArgs);
1599 
1600  // send request to MacroMaker a la FEVInterface::runFrontEndMacro
1601  // but need to pass iteration information, so that the call is launched by just one
1602  // send to each front end. Front-ends must immediately respond that is started
1603  // FEVInterfacesManager.. must start a thread for running the macro iterations
1604  // Then check for complete.
1605 
1606  iteratorStruct->targetsDone_.clear(); // reset
1607 
1608  __COUTV__(iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size());
1609  for(const auto& target :
1610  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_)
1611  {
1612  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1613 
1614  // for each target, init to not done
1615  iteratorStruct->targetsDone_.push_back(false);
1616 
1617  xoap::MessageReference message =
1618  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1619 
1620  SOAPParameters parameters;
1621  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalStart"
1622  : "macroMultiDimensionalStart";
1623  parameters.addParameter("type", type);
1624  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1625  parameters.addParameter("targetInterfaceID", target.UID_);
1626  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1627  parameters.addParameter("enableSavingOutput", enableSavingOutput);
1628  parameters.addParameter("outputFilePath", outputFilePath);
1629  parameters.addParameter("outputFileRadix", outputFileRadix);
1630  parameters.addParameter("inputArgs", inputArgs);
1631  SOAPUtilities::addParameters(message, parameters);
1632 
1633  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1634  << __E__;
1635 
1636  xoap::MessageReference replyMessage =
1637  iteratorStruct->theIterator_->theSupervisor_
1638  ->SOAPMessenger::sendWithSOAPReply(
1639  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1640  .getAllMacroMakerTypeSupervisorInfo()
1641  .begin()
1642  ->second.getDescriptor(),
1643  message);
1644 
1645  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1646  << __E__;
1647 
1648  SOAPParameters rxParameters;
1649  rxParameters.addParameter("Error");
1650  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1651 
1652  std::string error = rxParameters.getValue("Error");
1653 
1654  if(response != type + "Done" || error != "")
1655  {
1656  // error occurred!
1657  __SS__ << "Error transmitting request to target interface '" << target.UID_
1658  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1659  << response << "' with error: " << error << __E__;
1660  __SS_THROW__;
1661  }
1662  } // end target loop
1663 
1664 } // end startCommandMacro()
1665 
1666 //==============================================================================
1667 bool Iterator::checkCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1668  bool isFrontEndMacro)
1669 {
1670  sleep(1);
1671 
1672  // Steps:
1673  // 4 parameters CommandExecuteFEMacroParams:
1674  // //targets
1675  // const std::string FEMacroName_ = "FEMacroName";
1676  // //macro parameters (table/groupID)
1677 
1678  const std::string& macroName =
1679  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1680  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1681 
1682  __COUTV__(macroName);
1683 
1684  // send request to MacroMaker to check completion of macro
1685  // as targets are identified complete, remove targets_ from vector
1686 
1687  for(unsigned int i = 0;
1688  i < iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size();
1689  ++i)
1690  {
1691  ots::IterateTable::CommandTarget& target =
1692  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_[i];
1693 
1694  if(i < iteratorStruct->targetsDone_.size() && iteratorStruct->targetsDone_[i])
1695  continue; // already reported Done (the FE forgets the launch once reported)
1696 
1697  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1698 
1699  xoap::MessageReference message =
1700  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1701 
1702  SOAPParameters parameters;
1703  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalCheck"
1704  : "macroMultiDimensionalCheck";
1705  parameters.addParameter("type", type);
1706  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1707  parameters.addParameter("targetInterfaceID", target.UID_);
1708  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1709  SOAPUtilities::addParameters(message, parameters);
1710 
1711  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1712  << __E__;
1713 
1714  xoap::MessageReference replyMessage =
1715  iteratorStruct->theIterator_->theSupervisor_
1716  ->SOAPMessenger::sendWithSOAPReply(
1717  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1718  .getAllMacroMakerTypeSupervisorInfo()
1719  .begin()
1720  ->second.getDescriptor(),
1721  message);
1722 
1723  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1724  << __E__;
1725 
1726  SOAPParameters rxParameters;
1727  rxParameters.addParameter("Error");
1728  rxParameters.addParameter("Done");
1729  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1730 
1731  std::string error = rxParameters.getValue("Error");
1732  bool done = rxParameters.getValue("Done") == "1";
1733 
1734  if(response != type + "Done" || error != "")
1735  {
1736  // error occurred!
1737  __SS__ << "Error transmitting request to target interface '" << target.UID_
1738  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1739  << response << "' with error: " << error << __E__;
1740  __SS_THROW__;
1741  }
1742 
1743  if(!done) // still more to do so give up checking
1744  return false;
1745 
1746  // mark target done
1747  iteratorStruct->targetsDone_[i] = true;
1748 
1749  // record the saved-output file (if any) for the plan completion report.
1750  // Separate receive so front-ends built before "OutputFile" was added still work.
1751  try
1752  {
1753  SOAPParameters fileParameters;
1754  fileParameters.addParameter("OutputFile");
1755  SOAPUtilities::receive(replyMessage, fileParameters);
1756  const std::string outputFile = fileParameters.getValue("OutputFile");
1757  if(outputFile != "")
1758  {
1759  __COUT_INFO__ << "Macro '" << macroName << "' on '" << target.UID_
1760  << "' saved outputs to: " << outputFile << __E__;
1761  // label: "<macro> on <FE> iteration #<pass> [arg = val (0xhex), ...]"
1762  // pass = how many times this command has started in this plan (1-based)
1763  std::string label =
1764  macroName + " on " + target.UID_ + " iteration #" +
1765  std::to_string(
1766  iteratorStruct
1767  ->commandIterations_[iteratorStruct->commandIndex_]);
1768  if(iteratorStruct->macroArgsSummary_ != "")
1769  label += " [" + iteratorStruct->macroArgsSummary_ + "]";
1770  iteratorStruct->outputFiles_.push_back(
1771  {"" /*Self*/, outputFile, label, ""});
1772  }
1773  }
1774  catch(...)
1775  {
1776  __COUT__ << "Front-end did not report an OutputFile (older build?)" << __E__;
1777  }
1778 
1779  // iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.erase(
1780  // targetIt--); //go back after delete
1781 
1782  } // end target loop
1783 
1784  // if here all targets are done
1785  return true;
1786 } // end checkCommandMacro()
1787 
1788 //==============================================================================
1789 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct* iteratorStruct)
1790 {
1791  // Steps:
1792  // 4 parameters commandModifyActiveParams_:
1793  // const std::string DoTrackGroupChanges_ TrueFalse
1794  // //targets
1795  // const std::string RelativePathToField_ = "RelativePathToField";
1796  // const std::string FieldStartValue_ = "FieldStartValue";
1797  // const std::string FieldIterationStepSize_ = "FieldIterationStepSize";
1798  //
1799  // if tracking changes,
1800  // create a new group
1801  // for every enabled FE
1802  // set field = start value + stepSize * currentStepIndex_
1803  // activate group
1804  // else
1805  // load scratch group
1806  // for every enabled FE
1807  // set field = start value + stepSize * stepIndex
1808  // activate group
1809 
1810  bool doTrackGroupChanges = false;
1811  if("True" ==
1812  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1813  .params_[IterateTable::commandModifyActiveParams_.DoTrackGroupChanges_])
1814  doTrackGroupChanges = true;
1815 
1816  const std::string& startValueStr =
1817  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1818  .params_[IterateTable::commandModifyActiveParams_.FieldStartValue_];
1819  const std::string& stepSizeStr =
1820  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1821  .params_[IterateTable::commandModifyActiveParams_.FieldIterationStepSize_];
1822 
1823  const unsigned int stepIndex = getStepIndexForLabel(iteratorStruct, "" /*innermost*/);
1824 
1825  __COUT__ << "doTrackGroupChanges " << (doTrackGroupChanges ? "yes" : "no")
1826  << std::endl;
1827  __COUTV__(startValueStr);
1828  __COUTV__(stepSizeStr);
1829  __COUTV__(stepIndex);
1830 
1831  ConfigurationInterface::setVersionTrackingEnabled(doTrackGroupChanges);
1832 
1833  // two approaches: double or long handling
1834  // OR TODO -- if step is 0 and startValue is NaN.. then handle as string
1835 
1836  if(((stepSizeStr.size() && stepSizeStr[0] == '0') || !stepSizeStr.size()) &&
1837  !StringMacros::isNumber(
1838  startValueStr)) //if not a number and no step size, then interpret as string
1839  {
1840  __COUT__ << "Treating start value as string: " << startValueStr << __E__;
1841  helpCommandModifyActive(iteratorStruct, startValueStr, doTrackGroupChanges);
1842  }
1843  else if(startValueStr.size() && (startValueStr[startValueStr.size() - 1] == 'f' ||
1844  startValueStr.find('.') != std::string::npos))
1845  {
1846  // handle as double
1847  double startValue = strtod(startValueStr.c_str(), 0);
1848  double stepSize = strtod(stepSizeStr.c_str(), 0);
1849 
1850  __COUT__ << "startValue " << startValue << std::endl;
1851  __COUT__ << "stepSize " << stepSize << std::endl;
1852  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1853 
1854  helpCommandModifyActive(
1855  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1856  }
1857  else // handle as long
1858  {
1859  long int startValue;
1860  long int stepSize;
1861 
1862  StringMacros::getNumber(startValueStr, startValue);
1863  StringMacros::getNumber(startValueStr, stepSize);
1864 
1865  __COUT__ << "startValue " << startValue << std::endl;
1866  __COUT__ << "stepSize " << stepSize << std::endl;
1867  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1868 
1869  helpCommandModifyActive(
1870  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1871  }
1872 
1873 } // end startCommandModifyActive()
1874 
1875 //==============================================================================
1884 bool Iterator::checkCommandRun(IteratorWorkLoopStruct* iteratorStruct)
1885 {
1886  sleep(1); // sleep to give FSM time to transition
1887 
1888  // all RunControlStateMachine access commands should be mutually exclusive with
1889  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
1890  // with GatewaySupervisor main thread state machine accesses lockout the messages
1891  // array for the remainder of the scope this guarantees the reading thread can safely
1892  // access the messages
1893  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1894  __COUT__ << "Waiting for FSM access" << __E__;
1895  std::lock_guard<std::mutex> lock(
1896  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1897  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1898  __COUT__ << "Have FSM access" << __E__;
1899 
1900  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1901  return false;
1902 
1903  iteratorStruct->fsmCommandParameters_.clear();
1904 
1905  std::string errorStr = "";
1906  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1907  ->theStateMachine_.getCurrentStateName();
1908 
1910  // check for imposed actions and forced exits
1911  if(iteratorStruct->doPauseAction_)
1912  {
1913  // transition to pause state
1914  __COUT__ << "Transitioning FSM to Paused..." << __E__;
1915 
1916  if(currentState == "Paused")
1917  {
1918  // done with early pause exit!
1919  __COUT__ << "Transition to Paused complete." << __E__;
1920  return true;
1921  }
1922  else if(currentState == "Running") // launch transition to pause
1923  errorStr = iteratorStruct->theIterator_->theSupervisor_
1924  ->attemptStateMachineTransition(
1925  0,
1926  0,
1927  "Pause",
1928  iteratorStruct->fsmName_,
1929  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1930  WebUsers::DEFAULT_ITERATOR_USERNAME,
1931  iteratorStruct->fsmCommandParameters_);
1932  else if(currentState == "Configured")
1933  {
1934  // no need to pause state machine, no run going on
1935  __COUT__ << "In Configured state. No need to transition to Paused." << __E__;
1936  return true;
1937  }
1938  else
1939  errorStr = "Expected to be in Paused. Unexpectedly, the current state is " +
1940  currentState +
1941  ". Last State Machine error message was as follows: " +
1942  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1943  .getErrorMessage();
1944 
1945  if(errorStr != "")
1946  {
1947  __SS__ << "Iterator failed to pause because of the following error: "
1948  << errorStr;
1949  __SS_THROW__;
1950  }
1951  return false;
1952  }
1953  else if(iteratorStruct->doHaltAction_)
1954  {
1955  // transition to halted state
1956  __COUT__ << "Transitioning FSM to Halted..." << __E__;
1957 
1958  if(currentState == RunControlStateMachine::HALTED_STATE_NAME)
1959  {
1960  // done with early halt exit!
1961  __COUT__ << "Transition to Halted complete." << __E__;
1962  return true;
1963  }
1964  else if(currentState == RunControlStateMachine::
1965  RUNNING_STATE_NAME || // launch transition to halt
1966  currentState == RunControlStateMachine::PAUSED_STATE_NAME)
1967  errorStr = iteratorStruct->theIterator_->theSupervisor_
1968  ->attemptStateMachineTransition(
1969  0,
1970  0,
1971  RunControlStateMachine::ABORT_TRANSITION_NAME,
1972  iteratorStruct->fsmName_,
1973  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1974  WebUsers::DEFAULT_ITERATOR_USERNAME,
1975  iteratorStruct->fsmCommandParameters_);
1976  else if(currentState == RunControlStateMachine::
1977  CONFIGURED_STATE_NAME) // launch transition to halt
1978  errorStr = iteratorStruct->theIterator_->theSupervisor_
1979  ->attemptStateMachineTransition(
1980  0,
1981  0,
1982  RunControlStateMachine::HALT_TRANSITION_NAME,
1983  iteratorStruct->fsmName_,
1984  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1985  WebUsers::DEFAULT_ITERATOR_USERNAME,
1986  iteratorStruct->fsmCommandParameters_);
1987  else
1988  errorStr = "Expected to be in Halted. Unexpectedly, the current state is " +
1989  currentState +
1990  ". Last State Machine error message was as follows: " +
1991  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1992  .getErrorMessage();
1993 
1994  if(errorStr != "")
1995  {
1996  __SS__ << "Iterator failed to halt because of the following error: "
1997  << errorStr;
1998  __SS_THROW__;
1999  }
2000  return false;
2001  }
2002  else if(iteratorStruct->doResumeAction_)
2003  {
2004  // Note: check command gets one shot to resume
2005 
2006  // transition to running state
2007  __COUT__ << "Transitioning FSM to Running..." << __E__;
2008 
2009  if(currentState == "Paused") // launch transition to running
2010  errorStr = iteratorStruct->theIterator_->theSupervisor_
2011  ->attemptStateMachineTransition(
2012  0,
2013  0,
2014  RunControlStateMachine::RESUME_TRANSITION_NAME,
2015  iteratorStruct->fsmName_,
2016  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
2017  WebUsers::DEFAULT_ITERATOR_USERNAME,
2018  iteratorStruct->fsmCommandParameters_);
2019 
2020  if(errorStr != "")
2021  {
2022  __SS__ << "Iterator failed to run because of the following error: "
2023  << errorStr;
2024  __SS_THROW__;
2025  }
2026  return false;
2027  }
2028 
2030  // normal running
2031 
2032  if(currentState != "Running")
2033  {
2034  if(iteratorStruct->runIsDone_ && currentState == "Configured")
2035  {
2036  // indication of done
2037  __COUT__ << "Reached end of run " << iteratorStruct->fsmNextRunNumber_
2038  << __E__;
2039  return true;
2040  }
2041 
2042  errorStr = "Expected to be in Running. Unexpectedly, the current state is " +
2043  currentState + ". Last State Machine error message was as follows: " +
2044  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
2045  .getErrorMessage();
2046  }
2047  else // else in Running state! Check for end of run
2048  {
2049  bool waitOnRunningThreads = false;
2050  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_]
2051  .params_[IterateTable::commandRunParams_.WaitOnRunningThreads_])
2052  waitOnRunningThreads = true;
2053 
2054  time_t remainingDurationInSeconds;
2055  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
2056  .params_[IterateTable::commandRunParams_.DurationInSeconds_]
2057  .c_str(),
2058  "%ld",
2059  &remainingDurationInSeconds);
2060 
2061  __COUT__ << "waitOnRunningThreads " << waitOnRunningThreads << __E__;
2062  __COUT__ << "remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
2063 
2065  // priority 1 is waiting on running threads
2066  if(waitOnRunningThreads)
2067  {
2068  // get status of all running FE workloops
2069  GatewaySupervisor* theSupervisor =
2070  iteratorStruct->theIterator_->theSupervisor_;
2071 
2072  bool allFrontEndsAreDone = true;
2073  for(auto& it : theSupervisor->allSupervisorInfo_.getAllFETypeSupervisorInfo())
2074  {
2075  try
2076  {
2077  std::string status = theSupervisor->send(it.second.getDescriptor(),
2078  "WorkLoopStatusRequest");
2079 
2080  __COUT__ << "FESupervisor instance " << it.first
2081  << " has status = " << status << std::endl;
2082 
2083  if(status != CoreSupervisorBase::WORK_LOOP_DONE)
2084  {
2085  allFrontEndsAreDone = false;
2086  break;
2087  }
2088  }
2089  catch(xdaq::exception::Exception& e)
2090  {
2091  __SS__ << "Could not retrieve status from FESupervisor instance "
2092  << it.first << "." << std::endl;
2093  __COUT_WARN__ << ss.str();
2094  errorStr = ss.str();
2095 
2096  if(errorStr != "")
2097  {
2098  __SS__
2099  << "Iterator failed to run because of the following error: "
2100  << errorStr;
2101  __SS_THROW__;
2102  }
2103  }
2104  }
2105 
2106  if(allFrontEndsAreDone)
2107  {
2108  // need to end run!
2109  __COUT__ << "FE workloops all complete! Stopping run..." << __E__;
2110 
2111  errorStr = iteratorStruct->theIterator_->theSupervisor_
2112  ->attemptStateMachineTransition(
2113  0,
2114  0,
2115  "Stop",
2116  iteratorStruct->fsmName_,
2117  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
2118  WebUsers::DEFAULT_ITERATOR_USERNAME,
2119  iteratorStruct->fsmCommandParameters_,
2120  "Stopped by iterator: " +
2121  buildIteratorLogEntry(iteratorStruct));
2122 
2123  if(errorStr != "")
2124  {
2125  __SS__
2126  << "Iterator failed to stop run because of the following error: "
2127  << errorStr;
2128  __SS_THROW__;
2129  }
2130 
2131  // write indication of run done into duration
2132  iteratorStruct->runIsDone_ = true;
2133 
2134  return false;
2135  }
2136  } //end waitOnRunningThreads
2137 
2139  // priority 2 is duration, if <= 0 it is ignored
2140  if(remainingDurationInSeconds > 1)
2141  {
2142  //reset command start time (for displays) once running is stable
2143  if(remainingDurationInSeconds == iteratorStruct->originalDurationInSeconds_)
2144  {
2145  iteratorStruct->theIterator_->activeCommandStartTime_ =
2146  time(0); // reset on any change
2147  __COUT__ << "Starting run duration of " << remainingDurationInSeconds
2148  << " [s] at time = "
2149  << iteratorStruct->theIterator_->activeCommandStartTime_ << " "
2150  << StringMacros::getTimestampString(
2151  iteratorStruct->theIterator_->activeCommandStartTime_)
2152  << __E__;
2153  }
2154 
2155  --remainingDurationInSeconds;
2156 
2157  // write back to string
2158  char str[200];
2159  sprintf(str, "%ld", remainingDurationInSeconds);
2160  iteratorStruct->commands_[iteratorStruct->commandIndex_]
2161  .params_[IterateTable::commandRunParams_.DurationInSeconds_] =
2162  str; // re-store as string
2163  }
2164  else if(remainingDurationInSeconds == 1)
2165  {
2166  // need to end run!
2167  __COUT__ << "Time duration reached! Stopping run..." << __E__;
2168 
2169  errorStr =
2170  iteratorStruct->theIterator_->theSupervisor_
2171  ->attemptStateMachineTransition(
2172  0,
2173  0,
2174  "Stop",
2175  iteratorStruct->fsmName_,
2176  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
2177  WebUsers::DEFAULT_ITERATOR_USERNAME,
2178  iteratorStruct->fsmCommandParameters_,
2179  "Stopped by iterator: " + buildIteratorLogEntry(iteratorStruct));
2180 
2181  if(errorStr != "")
2182  {
2183  __SS__ << "Iterator failed to stop run because of the following error: "
2184  << errorStr;
2185  __SS_THROW__;
2186  }
2187 
2188  // write indication of run done
2189  iteratorStruct->runIsDone_ = true;
2190 
2191  // write original duration back to string
2192  char str[200];
2193  sprintf(str, "%ld", iteratorStruct->originalDurationInSeconds_);
2194  iteratorStruct->commands_[iteratorStruct->commandIndex_]
2195  .params_[IterateTable::commandRunParams_.DurationInSeconds_] =
2196  str; // re-store as string
2197  return false;
2198  }
2199  }
2200 
2201  if(errorStr != "")
2202  {
2203  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
2204  __SS_THROW__;
2205  }
2206  return false;
2207 } // end checkCommandRun()
2208 
2209 //==============================================================================
2210 bool Iterator::checkCommandWait(IteratorWorkLoopStruct* iteratorStruct)
2211 {
2212  sleep(1); // sleep for a second
2213 
2214  // Get the remaining time
2215  long remainingDurationInSeconds;
2216  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
2217  .params_[IterateTable::commandWaitParams_.DurationInSeconds_]
2218  .c_str(),
2219  "%ld",
2220  &remainingDurationInSeconds);
2221 
2222  __COUT__ << "Wait remaining time: " << remainingDurationInSeconds << " seconds."
2223  << __E__;
2224 
2225  // If this is the first check, reset the command start time for the display
2226  if(remainingDurationInSeconds == iteratorStruct->originalDurationInSeconds_)
2227  {
2228  iteratorStruct->theIterator_->activeCommandStartTime_ = time(0);
2229  __COUT__ << "Starting wait duration of " << remainingDurationInSeconds
2230  << " [s] at time = "
2231  << iteratorStruct->theIterator_->activeCommandStartTime_ << " "
2232  << StringMacros::getTimestampString(
2233  iteratorStruct->theIterator_->activeCommandStartTime_)
2234  << __E__;
2235  }
2236 
2237  // Handle pause and halt actions if requested
2238  if(iteratorStruct->doPauseAction_ || iteratorStruct->doHaltAction_)
2239  {
2240  // Restore the original duration before exiting
2241  char str[200];
2242  sprintf(str, "%ld", iteratorStruct->originalDurationInSeconds_);
2243  iteratorStruct->commands_[iteratorStruct->commandIndex_]
2244  .params_[IterateTable::commandWaitParams_.DurationInSeconds_] = str;
2245 
2246  return true; // Command is done when pause or halt is requested
2247  }
2248 
2249  // Check if we're done waiting
2250  if(remainingDurationInSeconds <= 1)
2251  {
2252  __COUT__ << "Wait duration complete!" << __E__;
2253 
2254  // Restore the original duration
2255  char str[200];
2256  sprintf(str, "%ld", iteratorStruct->originalDurationInSeconds_);
2257  iteratorStruct->commands_[iteratorStruct->commandIndex_]
2258  .params_[IterateTable::commandWaitParams_.DurationInSeconds_] = str;
2259 
2260  iteratorStruct->waitIsDone_ = true;
2261  return true; // Command is done
2262  }
2263  else
2264  {
2265  // Decrement the remaining time
2266  --remainingDurationInSeconds;
2267 
2268  // Write back to string
2269  char str[200];
2270  sprintf(str, "%ld", remainingDurationInSeconds);
2271  iteratorStruct->commands_[iteratorStruct->commandIndex_]
2272  .params_[IterateTable::commandWaitParams_.DurationInSeconds_] = str;
2273  }
2274 
2275  return false; // Command still in progress
2276 }
2277 
2278 //==============================================================================
2280 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct* iteratorStruct)
2281 {
2282  sleep(1); // sleep to give FSM time to transition
2283 
2284  // all RunControlStateMachine access commands should be mutually exclusive with
2285  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
2286  // with GatewaySupervisor main thread state machine accesses lockout the messages
2287  // array for the remainder of the scope this guarantees the reading thread can safely
2288  // access the messages
2289  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2290  __COUT__ << "Waiting for FSM access" << __E__;
2291  std::lock_guard<std::mutex> lock(
2292  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
2293  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2294  __COUT__ << "Have FSM access" << __E__;
2295 
2296  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
2297  return false;
2298 
2299  std::string errorStr = "";
2300  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
2301  ->theStateMachine_.getCurrentStateName();
2302 
2303  if(currentState == RunControlStateMachine::HALTED_STATE_NAME)
2304  errorStr =
2305  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
2306  0,
2307  0,
2308  "Configure",
2309  iteratorStruct->fsmName_,
2310  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
2311  WebUsers::DEFAULT_ITERATOR_USERNAME,
2312  iteratorStruct->fsmCommandParameters_);
2313  else if(currentState != RunControlStateMachine::CONFIGURED_STATE_NAME)
2314  errorStr = "Expected to be in '" + RunControlStateMachine::CONFIGURED_STATE_NAME +
2315  "' state. Unexpectedly, the current state is " + currentState + "." +
2316  ". Last State Machine error message was as follows: " +
2317  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
2318  .getErrorMessage();
2319  else // else successfully done (in Configured state!)
2320  {
2321  __COUT__ << "checkCommandConfigureAlias complete." << __E__;
2322 
2323  // also activate the Iterator config manager to mimic active config
2324  std::pair<std::string, TableGroupKey> newActiveGroup =
2325  iteratorStruct->cfgMgr_->getTableGroupFromAlias(
2326  iteratorStruct->fsmCommandParameters_[0]);
2327  iteratorStruct->cfgMgr_->loadTableGroup(
2328  newActiveGroup.first, newActiveGroup.second, true /*activate*/);
2329 
2330  __COUT__ << "originalTrackChanges " << iteratorStruct->originalTrackChanges_
2331  << __E__;
2332  __COUT__ << "originalConfigGroup " << iteratorStruct->originalConfigGroup_
2333  << __E__;
2334  __COUT__ << "originalConfigKey " << iteratorStruct->originalConfigKey_ << __E__;
2335 
2336  __COUT__ << "currentTrackChanges "
2337  << ConfigurationInterface::isVersionTrackingEnabled() << __E__;
2338  __COUT__ << "originalConfigGroup "
2339  << iteratorStruct->cfgMgr_->getActiveGroupName() << __E__;
2340  __COUT__ << "originalConfigKey " << iteratorStruct->cfgMgr_->getActiveGroupKey()
2341  << __E__;
2342 
2343  return true;
2344  }
2345 
2346  if(errorStr != "")
2347  {
2348  __SS__ << "Iterator failed to configure with system alias '"
2349  << (iteratorStruct->fsmCommandParameters_.size()
2350  ? iteratorStruct->fsmCommandParameters_[0]
2351  : "UNKNOWN")
2352  << "' because of the following error: " << errorStr;
2353  __SS_THROW__;
2354  }
2355  return false;
2356 } // end checkCommandConfigure()
2357 
2358 //==============================================================================
2360 bool Iterator::checkCommandFSMTransition(IteratorWorkLoopStruct* iteratorStruct,
2361  const std::string& finalState)
2362 {
2363  __COUTV__(finalState);
2364 
2365  sleep(1); // sleep to give FSM time to transition
2366 
2367  // all RunControlStateMachine access commands should be mutually exclusive with
2368  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
2369  // with GatewaySupervisor main thread state machine accesses lockout the messages
2370  // array for the remainder of the scope this guarantees the reading thread can safely
2371  // access the messages
2372  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2373  __COUT__ << "Waiting for FSM access" << __E__;
2374  std::lock_guard<std::mutex> lock(
2375  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
2376  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
2377  __COUT__ << "Have FSM access" << __E__;
2378 
2379  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
2380  return false;
2381 
2382  std::string errorStr = "";
2383  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
2384  ->theStateMachine_.getCurrentStateName();
2385 
2386  if(currentState != finalState)
2387  errorStr = "Expected to be in " + finalState +
2388  ". Unexpectedly, the current state is " + currentState + "." +
2389  ". Last State Machine error message was as follows: " +
2390  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
2391  .getErrorMessage();
2392  else // else successfully done (in Configured state!)
2393  {
2394  __COUT__ << "checkCommandFSMTransition complete." << __E__;
2395 
2396  return true;
2397  }
2398 
2399  if(errorStr != "")
2400  {
2401  __SS__ << "Iterator failed to reach final state '" << finalState
2402  << "' because of the following error: " << errorStr;
2403  __SS_THROW__;
2404  }
2405  return false;
2406 } // end checkCommandConfigure()
2407 
2408 //==============================================================================
2410 std::vector<IterateTable::Command> Iterator::generateIterationPlan(
2411  const std::string& fsmName,
2412  const std::string& configAlias,
2413  uint64_t durationSeconds /* = -1 */,
2414  unsigned int numberOfRuns /* = 1 */)
2415 {
2416  std::vector<IterateTable::Command> commands;
2417 
2418  //example commands:
2419  // Choose FSM name
2420  // Configure if needed to selected system alias
2421  // Start run
2422  // count seconds
2423  // Stop run numberOfRuns
2424  // loop
2425 
2426  if(durationSeconds == (uint64_t)-1)
2427  numberOfRuns = 1;
2428  __COUTV__(durationSeconds);
2429  __COUTV__(numberOfRuns);
2430 
2431  // -------- Choose FSM name
2432  {
2433  commands.push_back(IterateTable::Command());
2434  commands.back().type_ = IterateTable::COMMAND_CHOOSE_FSM;
2435 
2436  commands.back().params_.emplace(
2437  std::pair<std::string /*param name*/, std::string /*param value*/>(
2438  IterateTable::commandChooseFSMParams_.NameOfFSM_, fsmName));
2439  }
2440 
2441  // -------- Configure if needed to selected system alias
2442  {
2443  commands.push_back(IterateTable::Command());
2444  commands.back().type_ = IterateTable::COMMAND_CONFIGURE_ALIAS;
2445 
2446  commands.back().params_.emplace(
2447  std::pair<std::string /*param name*/, std::string /*param value*/>(
2448  IterateTable::commandConfigureAliasParams_.SystemAlias_, configAlias));
2449  // generated plans keep using the plan-wide onlyConfigIfNotConfigured_ flag
2450  commands.back().params_.emplace(
2451  std::pair<std::string /*param name*/, std::string /*param value*/>(
2452  IterateTable::commandConfigureAliasParams_.SkipIfAlreadyConfigured_,
2453  "0"));
2454  }
2455 
2456  if(durationSeconds == (uint64_t)-1)
2457  {
2458  __COUT__ << "Open ended run..." << __E__;
2459 
2460  // -------- Start FSM transtion
2461  {
2462  commands.push_back(IterateTable::Command());
2463  commands.back().type_ = IterateTable::COMMAND_START;
2464  }
2465  }
2466  else
2467  {
2468  __COUT__ << "Finite duration run(s)..." << __E__;
2469 
2470  if(numberOfRuns > 1)
2471  {
2472  __COUT__ << "Setting up iterator loop for " << numberOfRuns << " runs."
2473  << __E__;
2474 
2475  commands.push_back(IterateTable::Command());
2476  commands.back().type_ = IterateTable::COMMAND_BEGIN_LABEL;
2477 
2478  commands.back().params_.emplace(
2479  std::pair<std::string /*param name*/, std::string /*param value*/>(
2480  IterateTable::commandBeginLabelParams_.Label_, "GENERATED_LABEL"));
2481  }
2482 
2483  // -------- Start managed run
2484  {
2485  commands.push_back(IterateTable::Command());
2486  commands.back().type_ = IterateTable::COMMAND_RUN;
2487 
2488  commands.back().params_.emplace(
2489  std::pair<std::string /*param name*/, std::string /*param value*/>(
2490  IterateTable::commandRunParams_.DurationInSeconds_,
2491  std::to_string(durationSeconds)));
2492  }
2493 
2494  if(numberOfRuns > 1)
2495  {
2496  commands.push_back(IterateTable::Command());
2497  commands.back().type_ = IterateTable::COMMAND_REPEAT_LABEL;
2498 
2499  commands.back().params_.emplace(
2500  std::pair<std::string /*param name*/, std::string /*param value*/>(
2501  IterateTable::commandRepeatLabelParams_.Label_, "GENERATED_LABEL"));
2502  commands.back().params_.emplace(
2503  std::pair<std::string /*param name*/, std::string /*param value*/>(
2504  IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_,
2505  std::to_string(numberOfRuns -
2506  1) //number of repeats (i.e., 1 repeat, gives 2 runs)
2507  ));
2508  }
2509  }
2510 
2511  __COUTV__(commands.size());
2512 
2513  return commands;
2514 } //end generateIterationPlan()
2515 
2516 //==============================================================================
2517 bool Iterator::handleCommandRequest(HttpXmlDocument& xmldoc,
2518  const std::string& command,
2519  const std::string& parameter)
2520 try
2521 {
2522  bool handledCommand = false;
2523  __COUTTV__(command);
2524  if(command == "iteratePlay")
2525  {
2526  playIterationPlan(xmldoc, parameter);
2527  handledCommand = true;
2528  }
2529  else if(command == "iteratePlayGenerated")
2530  {
2531  playGeneratedIterationPlan(xmldoc, parameter);
2532  handledCommand = true;
2533  }
2534  else if(command == "iteratePause")
2535  {
2536  pauseIterationPlan(xmldoc);
2537  handledCommand = true;
2538  }
2539  else if(command == "iterateHalt")
2540  {
2541  haltIterationPlan(xmldoc);
2542  handledCommand = true;
2543  }
2544  else if(command == "getIterationPlanStatus")
2545  {
2546  if(activePlanName_ == "" &&
2547  parameter !=
2548  "") //take parameter to set active plan name from GUI manipulations
2549  {
2550  activePlanName_ = parameter;
2551  __COUTV__(activePlanName_);
2552  }
2553  getIterationPlanStatus(xmldoc);
2554  handledCommand = true;
2555  }
2556  else // return true if iterator has control of state machine
2557  {
2558  // lockout the messages array for the remainder of the scope
2559  // this guarantees the reading thread can safely access the messages
2560  if(theSupervisor_->VERBOSE_MUTEX)
2561  __COUT__ << "Waiting for iterator access" << __E__;
2562  std::lock_guard<std::mutex> lock(accessMutex_);
2563  if(theSupervisor_->VERBOSE_MUTEX)
2564  __COUT__ << "Have iterator access" << __E__;
2565 
2566  if(iteratorBusy_)
2567  {
2568  __SS__ << "Error - Can not accept request because the Iterator "
2569  << "is currently "
2570  << "in control of State Machine progress. ";
2571  __COUT_ERR__ << "\n" << ss.str();
2572  __COUT_ERR__ << "\n" << ss.str();
2573 
2574  xmldoc.addTextElementToData("state_transition_attempted",
2575  "0"); // indicate to GUI transition NOT attempted
2576  xmldoc.addTextElementToData(
2577  "state_transition_attempted_err",
2578  ss.str()); // indicate to GUI transition NOT attempted
2579  theSupervisor_->theStateMachine_.setErrorMessage(ss.str());
2580  return true; // to block other commands
2581  }
2582  }
2583 
2584  if(handledCommand)
2585  {
2586  xmldoc.addTextElementToData("state_transition_attempted",
2587  "1"); // indicate to GUI iterator attempted
2588  return true;
2589  }
2590  //else not an iterator command
2591 
2592  return false;
2593 } //end handleCommandRequest()
2594 catch(...)
2595 {
2596  __SS__ << "Error caught by Iterator command handling!" << __E__;
2597  try
2598  {
2599  throw;
2600  }
2601  catch(const std::runtime_error& e)
2602  {
2603  ss << "\nHere is the error: " << e.what() << __E__;
2604  }
2605  catch(...)
2606  {
2607  ss << "Uknown error caught." << __E__;
2608  }
2609 
2610  __COUT_ERR__ << "\n" << ss.str();
2611 
2612  xmldoc.addTextElementToData("state_transition_attempted",
2613  "0"); // indicate to GUI transition NOT attempted
2614  xmldoc.addTextElementToData("state_transition_attempted_err",
2615  ss.str()); // indicate to GUI transition NOT attempted
2616  theSupervisor_->theStateMachine_.setErrorMessage(ss.str());
2617  return true;
2618 } // end handleCommandRequest() error handling
2619 
2620 //==============================================================================
2621 void Iterator::playIterationPlan(HttpXmlDocument& xmldoc, const std::string& planName)
2622 {
2623  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
2624 
2625  if(planName == Iterator::RESERVED_GEN_PLAN_NAME)
2626  {
2627  __SS__ << "Illegal use of reserved iteration plan name '"
2628  << Iterator::RESERVED_GEN_PLAN_NAME << "!' Please select a different name."
2629  << __E__;
2630  __SS_THROW__;
2631  }
2632 
2633  playIterationPlanPrivate(xmldoc, planName);
2634 
2635 } //end playIterationPlan()
2636 
2637 //==============================================================================
2638 void Iterator::playGeneratedIterationPlan(HttpXmlDocument& xmldoc,
2639  const std::string& parametersCSV)
2640 {
2641  __COUTV__(parametersCSV);
2642  std::vector<std::string> parameters =
2643  StringMacros::getVectorFromString(parametersCSV, {','});
2644 
2645  if(parameters.size() != 6)
2646  {
2647  __SS__ << "Malformed CSV parameters to playGeneratedIterationPlan(), must be 6 "
2648  "arguments and there were "
2649  << parameters.size() << ": " << parametersCSV << __E__;
2650  __SS_THROW__;
2651  }
2652  // parameters[0] /*fsmName*/,
2653  // parameters[1] /*configAlias*/,
2654  // parameters[2] /*durationSeconds*/,
2655  // parameters[3] /*numberOfRuns*/,
2656  // parameters[4] /*keepConfiguration*/,
2657  // parameters[5] /*logEntry*/ double encoded
2658  parameters[5] = StringMacros::decodeURIComponent(parameters[5]);
2659 
2660  uint64_t durationSeconds;
2661  sscanf(parameters[2].c_str(), "%lu", &durationSeconds);
2662  unsigned int numberOfRuns;
2663  sscanf(parameters[3].c_str(), "%u", &numberOfRuns);
2664  unsigned int keepConfiguration;
2665  sscanf(parameters[4].c_str(), "%u", &keepConfiguration);
2666  playGeneratedIterationPlan(xmldoc,
2667  parameters[0] /*fsmName*/,
2668  parameters[1] /*configAlias*/,
2669  durationSeconds,
2670  numberOfRuns,
2671  keepConfiguration,
2672  parameters[5] /*logEntry*/
2673  );
2674 
2675 } //end playGeneratedIterationPlan()
2676 
2677 //==============================================================================
2678 void Iterator::playGeneratedIterationPlan(HttpXmlDocument& xmldoc,
2679  const std::string& fsmName,
2680  const std::string& configAlias,
2681  uint64_t durationSeconds /* = -1 */,
2682  unsigned int numberOfRuns /* = 1 */,
2683  bool keepConfiguration /* = false */,
2684  const std::string& logEntry)
2685 {
2686  std::string planName = Iterator::RESERVED_GEN_PLAN_NAME;
2687  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
2688 
2689  genFsmName_ = fsmName;
2690  genConfigAlias_ = configAlias;
2691  genPlanDurationSeconds_ = durationSeconds;
2692  genPlanNumberOfRuns_ = numberOfRuns;
2693  genKeepConfiguration_ = keepConfiguration;
2694  genLogEntry_ = logEntry;
2695 
2696  __COUTV__(genFsmName_);
2697  __COUTV__(genConfigAlias_);
2698  __COUTV__(genPlanDurationSeconds_);
2699  __COUTV__(genPlanNumberOfRuns_);
2700  __COUTV__(genKeepConfiguration_);
2701  __COUTV__(genLogEntry_);
2702 
2703  playIterationPlanPrivate(xmldoc, planName);
2704 
2705 } //end playGeneratedIterationPlan()
2706 
2707 //==============================================================================
2709 void Iterator::playIterationPlanPrivate(HttpXmlDocument& xmldoc,
2710  const std::string& planName)
2711 {
2712  // setup "play" command
2713 
2714  // lockout the messages array for the remainder of the scope
2715  // this guarantees the reading thread can safely access the messages
2716  if(theSupervisor_->VERBOSE_MUTEX)
2717  __COUT__ << "Waiting for iterator access" << __E__;
2718  std::lock_guard<std::mutex> lock(accessMutex_);
2719  if(theSupervisor_->VERBOSE_MUTEX)
2720  __COUT__ << "Have iterator access" << __E__;
2721 
2722  if(!activePlanIsRunning_ && !commandPlay_)
2723  {
2724  if(!workloopRunning_)
2725  {
2726  // start thread with member variables initialized
2727 
2728  workloopRunning_ = true;
2729 
2730  // must start thread first
2731  std::thread([](Iterator* iterator) { Iterator::IteratorWorkLoop(iterator); },
2732  this)
2733  .detach();
2734  }
2735 
2736  activePlanName_ = planName;
2737  commandPlay_ = true;
2738  }
2739  else
2740  {
2741  __SS__ << "Invalid play command attempted. Can only play when the Iterator is "
2742  "inactive or paused."
2743  << " If you would like to restart an iteration plan, first try halting "
2744  "the Iterator."
2745  << __E__;
2746  __COUT__ << ss.str();
2747 
2748  xmldoc.addTextElementToData("error_message", ss.str());
2749 
2750  __COUT__ << "Invalid play command attempted. " << activePlanIsRunning_ << " "
2751  << commandPlay_ << " " << activePlanName_ << __E__;
2752  }
2753 } //end playIterationPlan()
2754 
2755 //==============================================================================
2756 void Iterator::pauseIterationPlan(HttpXmlDocument& xmldoc)
2757 {
2758  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
2759  << __E__;
2760  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
2761  << __E__;
2762 
2763  // setup "pause" command
2764 
2765  // lockout the messages array for the remainder of the scope
2766  // this guarantees the reading thread can safely access the messages
2767  if(theSupervisor_->VERBOSE_MUTEX)
2768  __COUT__ << "Waiting for iterator access" << __E__;
2769  std::lock_guard<std::mutex> lock(accessMutex_);
2770  if(theSupervisor_->VERBOSE_MUTEX)
2771  __COUT__ << "Have iterator access" << __E__;
2772 
2773  if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
2774  {
2775  commandPause_ = true;
2776  }
2777  else
2778  {
2779  __SS__ << "Invalid pause command attempted. Can only pause when running."
2780  << __E__;
2781  __COUT__ << ss.str();
2782 
2783  xmldoc.addTextElementToData("error_message", ss.str());
2784 
2785  __COUT__ << "Invalid pause command attempted. " << workloopRunning_ << " "
2786  << activePlanIsRunning_ << " " << commandPause_ << " " << activePlanName_
2787  << __E__;
2788  }
2789 } //end pauseIterationPlan()
2790 
2791 //==============================================================================
2792 void Iterator::haltIterationPlan(HttpXmlDocument& /*xmldoc*/)
2793 {
2794  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
2795  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
2796 
2797  // setup "halt" command
2798 
2799  if(workloopRunning_)
2800  {
2801  // lockout the messages array for the remainder of the scope
2802  // this guarantees the reading thread can safely access the messages
2803  if(theSupervisor_->VERBOSE_MUTEX)
2804  __COUT__ << "Waiting for iterator access" << __E__;
2805  std::lock_guard<std::mutex> lock(accessMutex_);
2806  if(theSupervisor_->VERBOSE_MUTEX)
2807  __COUT__ << "Have iterator access" << __E__;
2808 
2809  __COUT__ << "activePlanIsRunning_: " << activePlanIsRunning_ << __E__;
2810  __COUT__ << "Passing halt command to iterator thread." << __E__;
2811  commandHalt_ = true;
2812 
2813  // clear
2814  activePlanName_ = "";
2815  activeCommandIndex_ = -1;
2816  }
2817  else // no thread, so halt (and reset Error') without command to thread
2818  {
2819  __COUT__ << "No thread, so conducting halt. lastFsmName_ = " << lastFsmName_
2820  << __E__;
2821  lastFsmName_ =
2822  theSupervisor_
2823  ->activeStateMachineName_; //force haltIterator to be successful
2824  __COUTV__(lastFsmName_);
2825  Iterator::haltIterator(this);
2826  }
2827 } //end haltIterationPlan()
2828 
2829 //==============================================================================
2831 void Iterator::getIterationPlanStatus(HttpXmlDocument& xmldoc)
2832 {
2833  xmldoc.addTextElementToData(
2834  "current_state",
2835  theSupervisor_->theStateMachine_.isInTransition()
2836  ? theSupervisor_->theStateMachine_.getCurrentTransitionName(
2837  theSupervisor_->stateMachineLastCommandInput_)
2838  : theSupervisor_->theStateMachine_.getCurrentStateName());
2839 
2840  // xmldoc.addTextElementToData("in_transition",
2841  // theSupervisor_->theStateMachine_.isInTransition() ? "1" : "0");
2842  if(theSupervisor_->theStateMachine_.isInTransition())
2843  xmldoc.addTextElementToData(
2844  "transition_progress",
2845  theSupervisor_->theProgressBar_.readPercentageString());
2846  else
2847  xmldoc.addTextElementToData("transition_progress", "100");
2848 
2849  xmldoc.addNumberElementToData("time_in_state",
2850  theSupervisor_->theStateMachine_.getTimeInState());
2851 
2852  // lockout the messages array for the remainder of the scope
2853  // this guarantees the reading thread can safely access the messages
2854  if(theSupervisor_->VERBOSE_MUTEX)
2855  __COUT__ << "Waiting for iterator access" << __E__;
2856  std::lock_guard<std::mutex> lock(accessMutex_);
2857  if(theSupervisor_->VERBOSE_MUTEX)
2858  __COUT__ << "Have iterator access" << __E__;
2859 
2860  xmldoc.addTextElementToData("active_plan", activePlanName_);
2861  xmldoc.addTextElementToData("last_started_plan", lastStartedPlanName_);
2862  xmldoc.addTextElementToData("last_finished_plan", lastFinishedPlanName_);
2863  // output files of the last finished plan: parallel arrays, one entry per file
2864  for(const auto& f : lastPlanOutputFiles_)
2865  {
2866  xmldoc.addTextElementToData("output_file_subsystem", f.subsystem);
2867  xmldoc.addTextElementToData("output_file_path", f.path);
2868  xmldoc.addTextElementToData("output_file_label", f.label);
2869  xmldoc.addTextElementToData("output_file_icon", f.iconName);
2870  }
2871 
2872  xmldoc.addNumberElementToData("current_command_index", activeCommandIndex_);
2873  xmldoc.addNumberElementToData("current_number_of_commands", activeNumberOfCommands_);
2874  xmldoc.addTextElementToData("current_command_type", activeCommandType_);
2875  xmldoc.addNumberElementToData("current_command_duration",
2876  time(0) - activeCommandStartTime_);
2877  xmldoc.addNumberElementToData("current_command_iteration", activeCommandIteration_);
2878  for(const auto& depthIteration : depthIterationStack_)
2879  xmldoc.addNumberElementToData("depth_iteration", depthIteration);
2880 
2881  if(activePlanName_ == Iterator::RESERVED_GEN_PLAN_NAME)
2882  {
2883  xmldoc.addNumberElementToData("generated_number_of_runs", genPlanNumberOfRuns_);
2884  xmldoc.addNumberElementToData("generated_duration_of_runs",
2885  genPlanDurationSeconds_);
2886  }
2887 
2888  if(activePlanIsRunning_ && iteratorBusy_)
2889  {
2890  if(workloopRunning_)
2891  xmldoc.addTextElementToData("active_plan_status", "Running");
2892  else
2893  xmldoc.addTextElementToData("active_plan_status",
2894  "Error"); // thread died (safety net)
2895  }
2896  else if(!activePlanIsRunning_ && iteratorBusy_)
2897  {
2898  if(errorMessage_.size())
2899  xmldoc.addTextElementToData(
2900  "active_plan_status",
2901  "Error"); // paused after error — press Play to retry
2902  else
2903  xmldoc.addTextElementToData("active_plan_status", "Paused");
2904  }
2905  else
2906  xmldoc.addTextElementToData("active_plan_status", "Inactive");
2907 
2908  xmldoc.addTextElementToData("error_message", errorMessage_);
2909 } //end getIterationPlanStatus()
2910 
2911 //==============================================================================
2912 size_t Iterator::findRemoteGatewayApp(IteratorWorkLoopStruct* iteratorStruct,
2913  const std::string& targetSubsystem)
2914 {
2915  auto& remoteApps = iteratorStruct->theIterator_->theSupervisor_->remoteGatewayApps_;
2916  for(size_t i = 0; i < remoteApps.size(); ++i)
2917  if(remoteApps[i].appInfo.name == targetSubsystem)
2918  return i;
2919 
2920  __SS__ << "Target subsystem '" << targetSubsystem
2921  << "' not found in remote gateway apps." << __E__;
2922  __SS_THROW__;
2923 } // end findRemoteGatewayApp()
2924 
2925 //==============================================================================
2926 void Iterator::queueRemoteGatewayCommand(IteratorWorkLoopStruct* iteratorStruct,
2927  size_t remoteAppIndex,
2928  const std::string& command,
2929  const std::string& statusLabel)
2930 {
2931  auto& remoteApp =
2932  iteratorStruct->theIterator_->theSupervisor_->remoteGatewayApps_[remoteAppIndex];
2933 
2934  if(remoteApp.command != "")
2935  {
2936  __SS__ << "Remote subsystem '" << remoteApp.appInfo.name
2937  << "' already has a pending command: " << remoteApp.command << __E__;
2938  __SS_THROW__;
2939  }
2940 
2941  remoteApp.clearError();
2942  remoteApp.command = command;
2943  remoteApp.fsmName = iteratorStruct->fsmName_;
2944  remoteApp.appInfo.status = "Launching " + statusLabel;
2945  remoteApp.appInfo.progress = 0;
2946  remoteApp.commandSentTime = time(0);
2947 
2948  __COUT__ << "Remote command '" << command << "' queued for subsystem '"
2949  << remoteApp.appInfo.name << "'" << __E__;
2950 } // end queueRemoteGatewayCommand()
2951 
2952 //==============================================================================
2955 std::string Iterator::buildRemoteConfigureCommand(IteratorWorkLoopStruct* iteratorStruct,
2956  const std::string& systemAlias)
2957 {
2958  GatewaySupervisor* gw = iteratorStruct->theIterator_->theSupervisor_;
2959 
2960  std::string command =
2961  RunControlStateMachine::CONFIGURE_TRANSITION_NAME + "," + systemAlias;
2962 
2963  std::string subsystemCommonList =
2964  StringMacros::setToString(gw->theConfigurationManager_->getVersionAliases(
2965  ConfigurationManager::SUBSYSTEM_COMMON_VERSION_ALIAS));
2966  if(subsystemCommonList.size())
2967  command += "," + GatewaySupervisor::COMMAND_PARAM_SUBSYSTEM_COMMON_PREAMBLE +
2968  StringMacros::encodeURIComponent(subsystemCommonList);
2969 
2970  std::string subsystemCommonOverrideList =
2971  StringMacros::setToString(gw->theConfigurationManager_->getVersionAliases(
2972  ConfigurationManager::SUBSYSTEM_COMMON_OVERRIDE_VERSION_ALIAS));
2973  if(subsystemCommonOverrideList.size())
2974  command += "," +
2975  GatewaySupervisor::COMMAND_PARAM_SUBSYSTEM_COMMON_OVERRIDE_PREAMBLE +
2976  StringMacros::encodeURIComponent(subsystemCommonOverrideList);
2977 
2978  std::string logEntry = gw->getLastLogEntry(
2979  RunControlStateMachine::CONFIGURE_TRANSITION_NAME, iteratorStruct->fsmName_);
2980  if(logEntry.size())
2981  command += "," + GatewaySupervisor::COMMAND_PARAM_LOG_ENTRY_PREAMBLE +
2982  StringMacros::encodeURIComponent(logEntry);
2983 
2984  return command;
2985 } // end buildRemoteConfigureCommand()
2986 
2987 //==============================================================================
2988 void Iterator::startRemoteCommandFSMTransition(IteratorWorkLoopStruct* iteratorStruct,
2989  const std::string& transitionCommand,
2990  const std::string& targetSubsystem)
2991 {
2992  __COUT__ << "startRemoteCommandFSMTransition: " << transitionCommand << " targeting '"
2993  << targetSubsystem << "'" << __E__;
2994 
2995  std::lock_guard<std::mutex> lock(
2996  iteratorStruct->theIterator_->theSupervisor_->remoteGatewayAppsMutex_);
2997 
2998  queueRemoteGatewayCommand(iteratorStruct,
2999  findRemoteGatewayApp(iteratorStruct, targetSubsystem),
3000  transitionCommand,
3001  transitionCommand);
3002 } // end startRemoteCommandFSMTransition()
3003 
3004 //==============================================================================
3005 bool Iterator::checkRemoteCommandFSMTransition(IteratorWorkLoopStruct* iteratorStruct,
3006  const std::string& finalState,
3007  const std::string& targetSubsystem)
3008 {
3009  sleep(1);
3010 
3011  std::lock_guard<std::mutex> lock(
3012  iteratorStruct->theIterator_->theSupervisor_->remoteGatewayAppsMutex_);
3013 
3014  const auto& remoteApp =
3015  iteratorStruct->theIterator_->theSupervisor_
3016  ->remoteGatewayApps_[findRemoteGatewayApp(iteratorStruct, targetSubsystem)];
3017 
3018  if(remoteApp.getError() != "")
3019  {
3020  __SS__ << "Remote subsystem '" << targetSubsystem
3021  << "' reported error: " << remoteApp.getError() << __E__;
3022  __SS_THROW__;
3023  }
3024 
3025  if(remoteApp.command != "")
3026  {
3027  __COUT__ << "Waiting for command to be sent to '" << targetSubsystem << "'..."
3028  << __E__;
3029  return false;
3030  }
3031 
3032  const std::string& remoteStatus = remoteApp.appInfo.status;
3033  __COUT__ << "Remote subsystem '" << targetSubsystem << "' status: " << remoteStatus
3034  << " (waiting for '" << finalState << "')" << __E__;
3035 
3036  // Halt is the recovery path out of Failed, so a (possibly stale) Failed status
3037  // is not an error while waiting to reach Halted
3038  if(finalState != RunControlStateMachine::HALTED_STATE_NAME &&
3039  remoteStatus.find(RunControlStateMachine::FAILED_STATE_NAME) == 0)
3040  {
3041  __SS__ << "Remote subsystem '" << targetSubsystem << "' entered '" << remoteStatus
3042  << "' while waiting for '" << finalState << "'" << __E__;
3043  __SS_THROW__;
3044  }
3045 
3046  if(remoteStatus == finalState)
3047  {
3048  __COUT__ << "checkRemoteCommandFSMTransition complete for '" << targetSubsystem
3049  << "'" << __E__;
3050  return true;
3051  }
3052  return false;
3053 } // end checkRemoteCommandFSMTransition()
3054 
3055 //==============================================================================
3058 void Iterator::startRemoteCommandConfigure(IteratorWorkLoopStruct* iteratorStruct,
3059  const std::string& systemAlias,
3060  const std::string& targetSubsystem)
3061 {
3062  __COUT__ << "startRemoteCommandConfigure: alias '" << systemAlias << "' targeting '"
3063  << targetSubsystem << "'" << __E__;
3064 
3065  iteratorStruct->fsmCommandParameters_.clear();
3066  iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
3067  iteratorStruct->remoteConfigureQueued_ = false;
3068 
3069  std::lock_guard<std::mutex> lock(
3070  iteratorStruct->theIterator_->theSupervisor_->remoteGatewayAppsMutex_);
3071 
3072  size_t remoteAppIndex = findRemoteGatewayApp(iteratorStruct, targetSubsystem);
3073  std::string currentState =
3074  iteratorStruct->theIterator_->theSupervisor_->remoteGatewayApps_[remoteAppIndex]
3075  .appInfo.status;
3076  __COUTV__(currentState);
3077 
3078  if(currentState == RunControlStateMachine::INITIAL_STATE_NAME ||
3079  currentState == RunControlStateMachine::HALTED_STATE_NAME)
3080  {
3081  queueRemoteGatewayCommand(
3082  iteratorStruct,
3083  remoteAppIndex,
3084  buildRemoteConfigureCommand(iteratorStruct, systemAlias),
3085  RunControlStateMachine::CONFIGURE_TRANSITION_NAME);
3086  }
3087  else if(currentState == RunControlStateMachine::CONFIGURED_STATE_NAME ||
3088  currentState.find(RunControlStateMachine::FAILED_STATE_NAME) == 0)
3089  {
3090  if(commandSkipsIfConfigured(iteratorStruct) &&
3091  currentState == RunControlStateMachine::CONFIGURED_STATE_NAME)
3092  {
3093  __COUT_INFO__
3094  << "Remote subsystem '" << targetSubsystem
3095  << "' already configured and SkipIfAlreadyConfigured is set, so "
3096  "leaving its configuration as-is."
3097  << __E__;
3098  return; // check sees Configured with nothing queued and completes
3099  }
3100 
3101  queueRemoteGatewayCommand(iteratorStruct,
3102  remoteAppIndex,
3103  RunControlStateMachine::HALT_TRANSITION_NAME,
3104  RunControlStateMachine::HALT_TRANSITION_NAME);
3105  iteratorStruct->remoteConfigureQueued_ = true;
3106  }
3107  else
3108  {
3109  __SS__ << "Iterator failed to configure remote subsystem '" << targetSubsystem
3110  << "' with system alias '" << systemAlias
3111  << "': Can only Configure from the Initial or Halted state. The current "
3112  "state is "
3113  << currentState << __E__;
3114  __SS_THROW__;
3115  }
3116 
3117  __COUT__ << "startRemoteCommandConfigure success." << __E__;
3118 } // end startRemoteCommandConfigure()
3119 
3120 //==============================================================================
3121 bool Iterator::checkRemoteCommandConfigure(IteratorWorkLoopStruct* iteratorStruct,
3122  const std::string& targetSubsystem)
3123 {
3124  sleep(1);
3125 
3126  std::lock_guard<std::mutex> lock(
3127  iteratorStruct->theIterator_->theSupervisor_->remoteGatewayAppsMutex_);
3128 
3129  size_t remoteAppIndex = findRemoteGatewayApp(iteratorStruct, targetSubsystem);
3130  auto& remoteApp =
3131  iteratorStruct->theIterator_->theSupervisor_->remoteGatewayApps_[remoteAppIndex];
3132 
3133  if(remoteApp.getError() != "")
3134  {
3135  __SS__ << "Iterator failed to configure remote subsystem '" << targetSubsystem
3136  << "' with system alias '"
3137  << (iteratorStruct->fsmCommandParameters_.size()
3138  ? iteratorStruct->fsmCommandParameters_[0]
3139  : "UNKNOWN")
3140  << "' because of the following error: " << remoteApp.getError() << __E__;
3141  __SS_THROW__;
3142  }
3143 
3144  if(remoteApp.command != "")
3145  {
3146  __COUT__ << "Waiting for command to be sent to '" << targetSubsystem << "'..."
3147  << __E__;
3148  return false;
3149  }
3150 
3151  const std::string& currentState = remoteApp.appInfo.status;
3152  __COUT__ << "Remote subsystem '" << targetSubsystem << "' status: " << currentState
3153  << __E__;
3154 
3155  if(iteratorStruct->remoteConfigureQueued_)
3156  {
3157  // Halt was sent first; once Halted, send the real Configure.
3158  // A stale 'Failed' status is expected briefly here, so do not treat it as
3159  // an error; a Halt that actually fails surfaces through getError() above.
3160  if(currentState == RunControlStateMachine::HALTED_STATE_NAME)
3161  {
3162  queueRemoteGatewayCommand(
3163  iteratorStruct,
3164  remoteAppIndex,
3165  buildRemoteConfigureCommand(iteratorStruct,
3166  iteratorStruct->fsmCommandParameters_[0]),
3167  RunControlStateMachine::CONFIGURE_TRANSITION_NAME);
3168  iteratorStruct->remoteConfigureQueued_ = false;
3169  }
3170  return false;
3171  }
3172 
3173  if(currentState.find(RunControlStateMachine::FAILED_STATE_NAME) == 0)
3174  {
3175  __SS__ << "Remote subsystem '" << targetSubsystem << "' entered '" << currentState
3176  << "' while configuring." << __E__;
3177  __SS_THROW__;
3178  }
3179 
3180  if(currentState == RunControlStateMachine::CONFIGURED_STATE_NAME)
3181  {
3182  __COUT__ << "checkRemoteCommandConfigure complete for '" << targetSubsystem << "'"
3183  << __E__;
3184  return true;
3185  }
3186  return false;
3187 } // end checkRemoteCommandConfigure()
3188 
3189 //==============================================================================
3197 Iterator::MacroLoopSpec Iterator::parseMacroLoopSpec(const std::string& inputArgs)
3198 {
3199  MacroLoopSpec spec;
3200 
3201  std::vector<std::string> dimensions;
3202  StringMacros::getVectorFromString(inputArgs, dimensions, {';'});
3203 
3204  if(dimensions.size() == 0 || (dimensions.size() == 1 && dimensions[0] == ""))
3205  {
3206  spec.dimIterations.push_back(1);
3207  spec.dimArgs.push_back({});
3208  }
3209  else
3210  for(unsigned int d = 0; d < dimensions.size(); ++d)
3211  {
3212  std::vector<std::string> args;
3213  StringMacros::getVectorFromString(dimensions[d], args, {','});
3214  if(args.size() == 0 || args[0] == "")
3215  {
3216  __SS__ << "Invalid dimensional arguments! Need number of iterations at "
3217  "dimension "
3218  << d << __E__;
3219  __SS_THROW__;
3220  }
3221  unsigned long numOfIterations;
3222  StringMacros::getNumber(args[0], numOfIterations);
3223  if(numOfIterations == 0)
3224  {
3225  __SS__ << "Illegal number of iterations '" << args[0] << "' at dimension "
3226  << d << ". Must be a positive integer!" << __E__;
3227  __SS_THROW__;
3228  }
3229  spec.dimIterations.push_back(numOfIterations);
3230  spec.dimArgs.push_back({});
3231 
3232  for(unsigned int a = 1; a < args.size(); ++a)
3233  {
3234  // name may contain ':' (e.g. "Target Link (Default := -1)"), so split from the right
3235  std::string name, init, step;
3236  if(!StringMacros::splitMacroArgTriple(args[a], name, init, step))
3237  {
3238  __SS__ << "Invalid argument '" << args[a]
3239  << "'! Expected name:initialValue:stepSize." << __E__;
3240  __SS_THROW__;
3241  }
3242  name = StringMacros::decodeURIComponent(name);
3243  init = StringMacros::decodeURIComponent(init);
3244  step = StringMacros::decodeURIComponent(step);
3245  MacroLoopSpec::Arg arg;
3246  arg.name = name;
3247  if(isConstantMacroStep(step))
3248  {
3249  // constant (step DEFAULT or 0): keep the value exactly as written
3250  arg.type = MacroLoopSpec::Arg::STRING;
3251  arg.sVal = init;
3252  }
3253  else if((init.size() &&
3254  (init.back() == 'f' || init.find('.') != std::string::npos)) ||
3255  (step.size() &&
3256  (step.back() == 'f' || step.find('.') != std::string::npos)))
3257  {
3258  arg.type = MacroLoopSpec::Arg::DOUBLE;
3259  arg.dInit = strtod(init.c_str(), 0);
3260  arg.dStep = strtod(step.c_str(), 0);
3261  }
3262  else
3263  {
3264  arg.type = MacroLoopSpec::Arg::LONG;
3265  StringMacros::getNumber(init, arg.lInit);
3266  StringMacros::getNumber(step, arg.lStep);
3267  }
3268  spec.dimArgs.back().push_back(arg);
3269  }
3270  }
3271 
3272  // total = product of dimension counts, guarding overflow
3273  spec.totalIterations = 1;
3274  for(unsigned long n : spec.dimIterations)
3275  if(__builtin_mul_overflow(
3276  spec.totalIterations, (uint64_t)n, &spec.totalIterations))
3277  {
3278  __SS__ << "Dimensional loop '" << inputArgs
3279  << "' has more iterations than can be counted (product overflows)."
3280  << __E__;
3281  __SS_THROW__;
3282  }
3283 
3284  // emit-order argument names, de-duplicated (lower dimension wins)
3285  for(const auto& dim : spec.dimArgs)
3286  for(const auto& arg : dim)
3287  {
3288  bool clash = false;
3289  for(const auto& existing : spec.argNames)
3290  if(existing == arg.name)
3291  {
3292  clash = true;
3293  break;
3294  }
3295  if(!clash)
3296  spec.argNames.push_back(arg.name);
3297  }
3298 
3299  return spec;
3300 } // end parseMacroLoopSpec()
3301 
3302 //==============================================================================
3306 std::string Iterator::macroArgValueForLabel(const std::string& value)
3307 {
3308  if(value.empty())
3309  return value;
3310  char* end = nullptr;
3311  long long v = strtoll(value.c_str(), &end, 0 /*auto base: 0x, 0 octal, decimal*/);
3312  if(end == value.c_str() || *end != '\0')
3313  return value; // not (only) an integer
3314  std::stringstream ss;
3315  ss << v << " (0x" << std::hex << std::uppercase << v << ")";
3316  return ss.str();
3317 } // end macroArgValueForLabel()
3318 
3319 //==============================================================================
3324 std::string Iterator::macroArgsSummary(const std::string& inputArgs)
3325 {
3326  try
3327  {
3328  const MacroLoopSpec spec = parseMacroLoopSpec(inputArgs);
3329  const auto values = macroLoopIteration(spec, 0);
3330  std::string str;
3331  for(size_t a = 0; a < values.size(); ++a)
3332  str += (a ? ", " : "") + feMacroArgBaseName(values[a].first) + " = " +
3333  macroArgValueForLabel(values[a].second);
3334  if(spec.totalIterations > 1)
3335  str += " (x" + std::to_string(spec.totalIterations) + ")";
3336  return str;
3337  }
3338  catch(...)
3339  {
3340  return "";
3341  }
3342 } // end macroArgsSummary()
3343 
3344 //==============================================================================
3348 std::vector<std::pair<std::string, std::string>> Iterator::macroLoopIteration(
3349  const MacroLoopSpec& spec, uint64_t index)
3350 {
3351  // odometer: innermost (last) dimension turns fastest
3352  std::vector<uint64_t> counters(spec.dimIterations.size(), 0);
3353  for(size_t d = spec.dimIterations.size(); d-- > 0;)
3354  {
3355  counters[d] = index % spec.dimIterations[d];
3356  index /= spec.dimIterations[d];
3357  }
3358 
3359  std::vector<std::pair<std::string, std::string>> argsIn;
3360  for(size_t d = 0; d < spec.dimArgs.size(); ++d)
3361  for(const auto& arg : spec.dimArgs[d])
3362  {
3363  bool clash = false;
3364  for(const auto& existing : argsIn)
3365  if(existing.first == arg.name)
3366  {
3367  clash = true;
3368  break;
3369  }
3370  if(clash)
3371  continue; // lower dimension wins
3372  std::string value =
3373  arg.type == MacroLoopSpec::Arg::LONG
3374  ? std::to_string(arg.lInit + arg.lStep * (long)counters[d])
3375  : arg.type == MacroLoopSpec::Arg::DOUBLE
3376  ? std::to_string(arg.dInit + arg.dStep * (double)counters[d])
3377  : arg.sVal;
3378  argsIn.emplace_back(arg.name, value);
3379  }
3380  return argsIn;
3381 } // end macroLoopIteration()
3382 
3383 //==============================================================================
3384 unsigned int Iterator::getStepIndexForLabel(IteratorWorkLoopStruct* iteratorStruct,
3385  const std::string& label)
3386 {
3387  if(iteratorStruct->stepIndexStack_.empty())
3388  return 0;
3389 
3390  // blank, or the table's unset-column sentinel, both mean the innermost open label
3391  if(label == "" || label == TableViewColumnInfo::DATATYPE_STRING_DEFAULT ||
3392  label == TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT)
3393  return iteratorStruct->stepIndexStack_.back();
3394 
3395  // stacks are parallel; search from the innermost outward
3396  for(size_t i = iteratorStruct->stepLabelStack_.size(); i-- > 0;)
3397  if(iteratorStruct->stepLabelStack_[i] == label &&
3398  i < iteratorStruct->stepIndexStack_.size())
3399  return iteratorStruct->stepIndexStack_[i];
3400 
3401  __COUT_WARN__ << "Step label '" << label
3402  << "' is not an open BEGIN_LABEL at this command (open labels: "
3403  << StringMacros::vectorToString(iteratorStruct->stepLabelStack_)
3404  << "). Using pass index 0." << __E__;
3405  return 0;
3406 } // end getStepIndexForLabel()
3407 
3408 //==============================================================================
3412 bool Iterator::isConstantMacroStep(const std::string& step)
3413 {
3414  if(step == TableViewColumnInfo::DATATYPE_STRING_DEFAULT ||
3415  step == TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT)
3416  return true;
3417  double stepValue = 1;
3418  return step.size() && StringMacros::getNumber(step, stepValue) && stepValue == 0;
3419 } // end isConstantMacroStep()
3420 
3421 //==============================================================================
3426 std::string Iterator::applyStepIndexToMacroArgs(IteratorWorkLoopStruct* iteratorStruct,
3427  const std::string& inputArgs,
3428  const std::string& labelsStr)
3429 {
3430  if(inputArgs == "")
3431  return inputArgs;
3432 
3433  std::vector<std::string> dimensions, labels;
3434  StringMacros::getVectorFromString(inputArgs, dimensions, {';'});
3435  StringMacros::getVectorFromString(labelsStr, labels, {';'});
3436 
3437  std::string out;
3438  for(size_t d = 0; d < dimensions.size(); ++d)
3439  {
3440  std::string label = d < labels.size() ? labels[d] : "";
3441  unsigned int stepIndex = getStepIndexForLabel(iteratorStruct, label);
3442 
3443  std::vector<std::string> args;
3444  StringMacros::getVectorFromString(dimensions[d], args, {','});
3445 
3446  if(d)
3447  out += ";";
3448  for(size_t a = 0; a < args.size(); ++a)
3449  {
3450  if(a)
3451  out += ",";
3452  if(a == 0) // iteration count
3453  {
3454  out += args[0];
3455  continue;
3456  }
3457 
3458  std::vector<std::string> pieces(3);
3459  if(!StringMacros::splitMacroArgTriple(
3460  args[a], pieces[0], pieces[1], pieces[2]))
3461  {
3462  out += args[a]; // leave malformed entries for the FE-side error
3463  continue;
3464  }
3465 
3466  // pieces arrive URI-encoded from IterateTable::getPlanCommands(); decode before
3467  // any numeric handling ('-' and '.' would otherwise be %2D / %2E and parse as 0)
3468  const std::string name = StringMacros::decodeURIComponent(pieces[0]);
3469  const std::string init = StringMacros::decodeURIComponent(pieces[1]);
3470  const std::string step = StringMacros::decodeURIComponent(pieces[2]);
3471 
3472  if(isConstantMacroStep(step))
3473  {
3474  // constant argument (step DEFAULT or 0): pass through untouched
3475  out += args[a];
3476  continue;
3477  }
3478 
3479  std::string newInit;
3480  if((init.size() &&
3481  (init.back() == 'f' || init.find('.') != std::string::npos)) ||
3482  (step.size() &&
3483  (step.back() == 'f' || step.find('.') != std::string::npos)))
3484  newInit = std::to_string(strtod(init.c_str(), 0) +
3485  strtod(step.c_str(), 0) * stepIndex);
3486  else
3487  {
3488  long initValue = 0, stepValue = 0;
3489  StringMacros::getNumber(init, initValue);
3490  StringMacros::getNumber(step, stepValue);
3491  newInit = std::to_string(initValue + stepValue * (long)stepIndex);
3492  }
3493 
3494  __COUT_INFO__ << "Macro arg '" << name << "' pass " << stepIndex
3495  << (label.size() ? (" of label '" + label + "'") : "") << ": "
3496  << init << " + " << step << "*" << stepIndex << " = " << newInit
3497  << __E__;
3498 
3499  // re-encode so the output has the same wire format as the input
3500  out += StringMacros::encodeURIComponent(name) + ":" +
3501  StringMacros::encodeURIComponent(newInit) + ":" +
3502  StringMacros::encodeURIComponent(step);
3503  }
3504  }
3505  return out;
3506 } // end applyStepIndexToMacroArgs()
3507 
3508 //==============================================================================
3513 std::string Iterator::feMacroArgBaseName(const std::string& argName)
3514 {
3515  std::string base = argName.substr(0, argName.find('('));
3516  size_t end = base.find_last_not_of(" \t");
3517  return end == std::string::npos ? std::string() : base.substr(0, end + 1);
3518 } // end feMacroArgBaseName()
3519 
3520 //==============================================================================
3526 void Iterator::startRemoteCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
3527  bool isFEMacro)
3528 {
3529  auto& command = iteratorStruct->commands_[iteratorStruct->commandIndex_];
3530  const std::string targetSubsystem = command.targetSubsystem_;
3531  const std::string macroName =
3532  command.params_[IterateTable::commandExecuteMacroParams_.MacroName_];
3533  const bool saveOutputs =
3534  command.params_[IterateTable::commandExecuteMacroParams_.EnableSavingOutput_] ==
3535  "1";
3536  const std::string inputArgs = applyStepIndexToMacroArgs(
3537  iteratorStruct,
3538  command.params_[IterateTable::commandExecuteMacroParams_.MacroArgumentString_],
3539  command.params_[IterateTable::commandExecuteMacroParams_.MacroArgumentLabels_]);
3540 
3541  __COUT__ << "startRemoteCommandMacro: '" << macroName << "' targeting '"
3542  << targetSubsystem << "' isFEMacro=" << isFEMacro << __E__;
3543  __COUTV__(inputArgs);
3544 
3545  if(command.targets_.size() == 0)
3546  {
3547  __SS__ << "No target front-ends defined for remote macro '" << macroName
3548  << "' on subsystem '" << targetSubsystem << "'" << __E__;
3549  __SS_THROW__;
3550  }
3551 
3552  GatewaySupervisor* gw = iteratorStruct->theIterator_->theSupervisor_;
3553 
3554  // resolve MacroMaker UDP address (throws with env-var / Configured guidance)
3555  std::string ipPort = gw->getRemoteMacroMakerUDPAddress(targetSubsystem);
3556  __COUTV__(ipPort);
3557 
3558  // discover live FEs and macros on the remote
3559  // Iterator is a friend of GatewaySupervisor, so the UDP bind address is reachable
3560  const std::string localIpAddress = gw->ipAddressForStateChangesOverUDP_;
3561 
3562  GatewaySupervisor::RemoteFEMacroInfo info =
3563  GatewaySupervisor::parseFEMacroInfo(GatewaySupervisor::queryRemoteMacroMaker(
3564  ipPort, "GetFrontendMacroInfo", 10 /*inactivity s*/, localIpAddress));
3565  __COUT__ << "Remote subsystem '" << targetSubsystem << "' has " << info.fes.size()
3566  << " live front-end(s) and " << info.publicMacros.size()
3567  << " public MacroMaker macro(s)." << __E__;
3568 
3569  // validate targets and macro; collect input/output names
3570  std::vector<std::string> inputNames, outputNames;
3571  std::string uidCSV;
3572  for(size_t t = 0; t < command.targets_.size(); ++t)
3573  {
3574  const std::string& uid = command.targets_[t].UID_;
3575  auto feIt = info.fes.find(uid);
3576  if(feIt == info.fes.end())
3577  {
3578  __SS__ << "Front-end '" << uid << "' is not live on remote subsystem '"
3579  << targetSubsystem
3580  << "'. Is it enabled and is the subsystem Configured? "
3581  << "Live front-ends: ";
3582  for(const auto& fe : info.fes)
3583  ss << fe.first << " ";
3584  ss << __E__;
3585  __SS_THROW__;
3586  }
3587  if(isFEMacro)
3588  {
3589  auto macroIt = feIt->second.macros.find(macroName);
3590  if(macroIt == feIt->second.macros.end())
3591  {
3592  __SS__ << "FE Macro '" << macroName << "' not found on front-end '" << uid
3593  << "' of remote subsystem '" << targetSubsystem
3594  << "'. Available: ";
3595  for(const auto& m : feIt->second.macros)
3596  ss << m.first << " ";
3597  ss << __E__;
3598  __SS_THROW__;
3599  }
3600  if(t == 0)
3601  {
3602  inputNames = macroIt->second.inputs;
3603  outputNames = macroIt->second.outputs;
3604  }
3605  else
3606  {
3607  // one RunFrontendMacro call carries a single ordered input list and a
3608  // single output list for all target FEs, so every target must declare
3609  // the same signature. Inputs match on base name (the remote ignores the
3610  // "(Default/Note)" suffix); outputs must match exactly, as the remote
3611  // compares output names verbatim (FEVInterfacesManager::runFEMacro).
3612  const auto& otherInputs = macroIt->second.inputs;
3613  bool same = otherInputs.size() == inputNames.size();
3614  for(size_t k = 0; same && k < inputNames.size(); ++k)
3615  same = feMacroArgBaseName(otherInputs[k]) ==
3616  feMacroArgBaseName(inputNames[k]);
3617  if(!same)
3618  {
3619  __SS__ << "FE Macro '" << macroName << "' on front-end '" << uid
3620  << "' of remote subsystem '" << targetSubsystem
3621  << "' declares inputs ["
3622  << StringMacros::vectorToString(otherInputs)
3623  << "] which differ from front-end '"
3624  << command.targets_[0].UID_ << "' inputs ["
3625  << StringMacros::vectorToString(inputNames)
3626  << "]. All targets of one remote macro command must share "
3627  "the same input signature."
3628  << __E__;
3629  __SS_THROW__;
3630  }
3631  const auto& otherOutputs = macroIt->second.outputs;
3632  if(otherOutputs != outputNames)
3633  {
3634  __SS__ << "FE Macro '" << macroName << "' on front-end '" << uid
3635  << "' of remote subsystem '" << targetSubsystem
3636  << "' declares outputs ["
3637  << StringMacros::vectorToString(otherOutputs)
3638  << "] which differ from front-end '"
3639  << command.targets_[0].UID_ << "' outputs ["
3640  << StringMacros::vectorToString(outputNames)
3641  << "]. All targets of one remote macro command must share "
3642  "the same output signature; use separate commands for "
3643  "front-ends with different signatures."
3644  << __E__;
3645  __SS_THROW__;
3646  }
3647  }
3648  }
3649  if(t)
3650  uidCSV += ",";
3651  uidCSV += uid;
3652  }
3653  if(!isFEMacro)
3654  {
3655  auto macroIt = info.publicMacros.find(macroName);
3656  if(macroIt == info.publicMacros.end())
3657  {
3658  __SS__ << "MacroMaker macro '" << macroName
3659  << "' not found among the PUBLIC macros of remote subsystem '"
3660  << targetSubsystem
3661  << "' (only public macros can be run remotely; make a private macro "
3662  "public in MacroMaker first). Available: ";
3663  for(const auto& m : info.publicMacros)
3664  ss << m.first << " ";
3665  ss << __E__;
3666  __SS_THROW__;
3667  }
3668  inputNames = macroIt->second.inputs;
3669  outputNames = macroIt->second.outputs;
3670  }
3671 
3672  // Parse the dimensional loop (iterations are computed one at a time in the worker,
3673  // so a large product never has to fit in memory) and map the remote macro's
3674  // declared inputs, in its order, onto the loop's argument names once.
3675  // The remote (FEVInterfacesManager::runFEMacro) validates inputs positionally
3676  // and ignores any "(Default/Note)" suffix, so match on the base name here too;
3677  // this keeps saved plans working when a macro's default/note text changes.
3678  const MacroLoopSpec spec = parseMacroLoopSpec(inputArgs);
3679  std::vector<size_t>
3680  inputToArgIndex; // inputNames[k] takes spec.argNames[inputToArgIndex[k]]
3681  {
3682  std::vector<bool> used(spec.argNames.size(), false);
3683  for(const auto& inputName : inputNames)
3684  {
3685  const std::string inputBase = feMacroArgBaseName(inputName);
3686  bool bound = false;
3687  for(size_t a = 0; a < spec.argNames.size(); ++a)
3688  if(!used[a] && feMacroArgBaseName(spec.argNames[a]) == inputBase)
3689  {
3690  inputToArgIndex.push_back(a);
3691  used[a] = true;
3692  bound = true;
3693  break;
3694  }
3695  if(!bound)
3696  {
3697  // Only FE macros understand a literal "Default" (FEVInterface substitutes the
3698  // declared default). Public MacroMaker macros parse every input as a number,
3699  // so "Default" would silently become 0: keep that a hard error.
3700  if(!isFEMacro)
3701  {
3702  __SS__
3703  << "ArgIn '" << inputName
3704  << "' was not assigned a value by any dimensional loop parameter "
3705  "sets. This is illegal. Macro '"
3706  << macroName << "' requires '" << inputName
3707  << "' as an input argument. Either remove the input argument "
3708  "from "
3709  "the macro, or define a value as a dimensional loop parameter."
3710  << __E__;
3711  __SS_THROW__;
3712  }
3713  inputToArgIndex.push_back(SIZE_MAX);
3714  __COUT_INFO__
3715  << "ArgIn '" << inputName
3716  << "' was not specified by the Iterator command for FE macro '"
3717  << macroName << "'; using 'Default' for this argument." << __E__;
3718  }
3719  }
3720  for(size_t a = 0; a < spec.argNames.size(); ++a)
3721  if(!used[a])
3722  __COUT_WARN__ << "Dimensional loop parameter '" << spec.argNames[a]
3723  << "' is not an input of macro '" << macroName
3724  << "' on remote subsystem '" << targetSubsystem
3725  << "'; it will not be sent." << __E__;
3726  }
3727 
3728  {
3729  std::stringstream hdr;
3730  hdr << "Remote macro '" << macroName << "' on subsystem '" << targetSubsystem
3731  << "' targets [" << uidCSV << "]: " << spec.totalIterations
3732  << " iteration(s) from loop spec '" << inputArgs << "'";
3733  if(saveOutputs)
3734  hdr << ". Output saving is enabled: the remote MacroMaker writes "
3735  "macroOutput_<time>.txt under its OTSDAQ_DATA (OutputFilePath/Radix "
3736  "are "
3737  "not applied to remote targets)";
3738  __COUT_INFO__ << hdr.str() << __E__;
3739  }
3740 
3741  std::string outputCSV;
3742  for(size_t i = 0; i < outputNames.size(); ++i)
3743  outputCSV += (i ? "," : "") + StringMacros::encodeURIComponent(outputNames[i]);
3744 
3745  auto run = std::make_shared<IteratorWorkLoopStruct::RemoteMacroRun>();
3746  run->iterationsTotal = spec.totalIterations;
3747  iteratorStruct->remoteMacroRun_ = run;
3748 
3749  // The thread captures only value copies and the shared run state: no Iterator,
3750  // IteratorWorkLoopStruct or GatewaySupervisor pointer, so it can safely outlive
3751  // all of them (it is detached and may block on the remote for up to the
3752  // inactivity timeout). run->abort is checked inside every UDP receive poll.
3753  std::thread([run,
3754  localIpAddress,
3755  ipPort,
3756  uidCSV,
3757  macroName,
3758  macroType = std::string(isFEMacro ? "fe" : "public"),
3759  spec,
3760  inputNames,
3761  inputToArgIndex,
3762  outputCSV,
3763  saveOutputs,
3764  targetSubsystem]() {
3765  try
3766  {
3767  const uint64_t total = spec.totalIterations;
3768  for(uint64_t i = 0; i < total; ++i)
3769  {
3770  if(run->abort)
3771  {
3772  __COUT_INFO__ << "Remote macro '" << macroName
3773  << "' aborted before iteration " << i + 1 << " of "
3774  << total << __E__;
3775  break;
3776  }
3777 
3778  // compute this iteration's values and emit them in the remote macro's
3779  // declared input order, under the remote's current input names
3780  const auto values = macroLoopIteration(spec, i);
3781  std::string inputStr,
3782  argsSummary; // wire format / "arg = val, ..." for labels
3783  for(size_t k = 0; k < inputNames.size(); ++k)
3784  {
3785  const std::string val = inputToArgIndex[k] == SIZE_MAX
3786  ? std::string("Default")
3787  : values[inputToArgIndex[k]].second;
3788  inputStr += (k ? ";" : "") +
3789  StringMacros::encodeURIComponent(inputNames[k]) + "," +
3790  StringMacros::encodeURIComponent(val);
3791  argsSummary += (k ? ", " : "") + feMacroArgBaseName(inputNames[k]) +
3792  " = " + macroArgValueForLabel(val);
3793  }
3794 
3795  // RunFrontendMacro;feClass;feUIDs;macroType;macroName;inputArgs;outputArgs;saveOutputs
3796  std::string cmd = "RunFrontendMacro;*;" + uidCSV + ";" + macroType + ";" +
3797  StringMacros::encodeURIComponent(macroName) + ";" +
3798  StringMacros::encodeURIComponent(inputStr) + ";" +
3799  StringMacros::encodeURIComponent(outputCSV) + ";" +
3800  (saveOutputs ? "1" : "0");
3801 
3802  __COUT_INFO__ << "Remote macro '" << macroName << "' iteration " << i + 1
3803  << " of " << total << " inputs: " << inputStr << __E__;
3804 
3805  std::string response = GatewaySupervisor::queryRemoteMacroMaker(
3806  ipPort,
3807  cmd,
3808  30 /*inactivity s*/,
3809  localIpAddress,
3810  [&](int pct) {
3811  std::lock_guard<std::mutex> lock(run->mutex);
3812  run->progress = pct;
3813  },
3814  &run->abort);
3815 
3816  if(response.find("Error") == 0)
3817  {
3818  std::lock_guard<std::mutex> lock(run->mutex);
3819  run->error = "iteration " + std::to_string(i + 1) + " of " +
3820  std::to_string(total) + ": " + response;
3821  break;
3822  }
3823 
3824  // output file written by the remote MacroMaker (when saving outputs it
3825  // reports feMacroRunArgs_name=Filename / feMacroRunArgs_value=$OTSDAQ_DATA/...
3826  // instead of per-FE output values)
3827  {
3828  size_t fAfter = 0;
3829  std::string argName;
3830  while((argName = StringMacros::extractXmlField(
3831  response, "feMacroRunArgs_name", 0, fAfter, &fAfter)) !=
3832  "")
3833  {
3834  fAfter += strlen("feMacroRunArgs_name");
3835  std::string argValue = StringMacros::extractXmlField(
3836  response, "feMacroRunArgs_value", 0, fAfter, &fAfter);
3837  fAfter += strlen("feMacroRunArgs_value");
3838  if(argName == "Filename" && argValue != "")
3839  {
3840  __COUT_INFO__ << "Remote macro '" << macroName
3841  << "' iteration " << i + 1
3842  << " output file on '" << targetSubsystem
3843  << "': " << argValue << __E__;
3844  std::lock_guard<std::mutex> lock(run->mutex);
3845  run->outputFiles.emplace_back(argValue, argsSummary);
3846  }
3847  }
3848  }
3849 
3850  // log per-FE outputs
3851  size_t after = 0;
3852  std::string feUid;
3853  while((feUid = StringMacros::extractXmlField(
3854  response, "fe_uid", 0, after, &after)) != "")
3855  {
3856  after += strlen("fe_uid");
3857  size_t argAfter = after;
3858  std::string outName, outValue, outputs;
3859  // outputArgs_name/value pairs follow fe_* fields within this feMacroExec
3860  size_t nextFe = response.find("<fe_uid", after);
3861  while((outName = StringMacros::extractXmlField(
3862  response, "outputArgs_name", 0, argAfter, &argAfter)) !=
3863  "" &&
3864  (nextFe == std::string::npos || argAfter < nextFe))
3865  {
3866  argAfter += strlen("outputArgs_name");
3867  outValue = StringMacros::extractXmlField(
3868  response, "outputArgs_value", 0, argAfter, &argAfter);
3869  argAfter += strlen("outputArgs_value");
3870  outputs += " " + outName + "=" +
3871  StringMacros::decodeURIComponent(outValue);
3872  }
3873  __COUT_INFO__ << "Remote macro '" << macroName << "' iteration "
3874  << i + 1 << " FE '" << feUid << "' outputs:" << outputs
3875  << __E__;
3876  }
3877 
3878  std::lock_guard<std::mutex> lock(run->mutex);
3879  ++run->iterationsDone;
3880  run->progress = 0;
3881  }
3882  }
3883  catch(const std::runtime_error& e)
3884  {
3885  if(run->abort) // a Halt interrupted the in-flight UDP call: not an error
3886  __COUT_INFO__ << "Remote macro '" << macroName
3887  << "' aborted mid-iteration: " << e.what() << __E__;
3888  else
3889  {
3890  std::lock_guard<std::mutex> lock(run->mutex);
3891  run->error = e.what();
3892  }
3893  }
3894  catch(...)
3895  {
3896  std::lock_guard<std::mutex> lock(run->mutex);
3897  run->error = "unknown error";
3898  }
3899  run->done = true;
3900  }).detach();
3901 
3902  __COUT__ << "startRemoteCommandMacro launched for '" << targetSubsystem << "'"
3903  << __E__;
3904 } // end startRemoteCommandMacro()
3905 
3906 //==============================================================================
3911 bool Iterator::checkRemoteCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
3912  bool /*isFEMacro*/)
3913 {
3914  sleep(1);
3915 
3916  auto run = iteratorStruct->remoteMacroRun_;
3917  if(!run)
3918  {
3919  __SS__ << "No remote macro run in progress!?" << __E__;
3920  __SS_THROW__;
3921  }
3922 
3923  if(iteratorStruct->doHaltAction_ && !run->abort)
3924  {
3925  __COUT_INFO__ << "Halt requested: interrupting the remote macro run." << __E__;
3926  run->abort = true;
3927  }
3928 
3929  std::string error;
3930  uint64_t itDone, itTotal;
3931  int progress;
3932  {
3933  std::lock_guard<std::mutex> lock(run->mutex);
3934  error = run->error;
3935  itDone = run->iterationsDone;
3936  itTotal = run->iterationsTotal;
3937  progress = run->progress;
3938  }
3939 
3940  if(error != "")
3941  {
3942  iteratorStruct->remoteMacroRun_.reset();
3943  auto& command = iteratorStruct->commands_[iteratorStruct->commandIndex_];
3944  __SS__ << "Remote macro '"
3945  << command.params_[IterateTable::commandExecuteMacroParams_.MacroName_]
3946  << "' on subsystem '" << command.targetSubsystem_ << "' failed: " << error
3947  << __E__;
3948  __SS_THROW__;
3949  }
3950 
3951  __COUT__ << "Remote macro progress: " << itDone << " of " << itTotal
3952  << " iteration(s) done, current iteration " << progress << "%" << __E__;
3953 
3954  if(run->done)
3955  {
3956  // record output files for the plan completion report
3957  std::vector<std::pair<std::string, std::string>> files;
3958  {
3959  std::lock_guard<std::mutex> lock(run->mutex);
3960  files = run->outputFiles;
3961  }
3962  if(files.size())
3963  {
3964  auto& command = iteratorStruct->commands_[iteratorStruct->commandIndex_];
3965  const std::string macroName =
3966  command.params_[IterateTable::commandExecuteMacroParams_.MacroName_];
3967  std::string uids;
3968  for(size_t t = 0; t < command.targets_.size(); ++t)
3969  uids += (t ? "," : "") + command.targets_[t].UID_;
3970  const std::string iconName = findRemoteCodeEditorIconName(
3971  iteratorStruct->theIterator_->theSupervisor_, command.targetSubsystem_);
3972  // label: "<macro> on <FEs> iteration #<pass>[.<i>] [arg = val (0xhex), ...]"
3973  // pass = how many times this command has started in this plan (1-based);
3974  // .<i> only when the command looped internally and wrote several files
3975  const std::string pass = std::to_string(
3976  iteratorStruct->commandIterations_[iteratorStruct->commandIndex_]);
3977  for(size_t i = 0; i < files.size(); ++i)
3978  {
3979  std::string label = macroName + " on " + uids + " iteration #" + pass;
3980  if(files.size() > 1)
3981  label += "." + std::to_string(i + 1);
3982  if(files[i].second != "")
3983  label += " [" + files[i].second + "]";
3984  iteratorStruct->outputFiles_.push_back(
3985  {command.targetSubsystem_, files[i].first, label, iconName});
3986  }
3987  }
3988 
3989  iteratorStruct->remoteMacroRun_.reset();
3990  __COUT__ << "checkRemoteCommandMacro complete." << __E__;
3991  return true;
3992  }
3993  return false;
3994 } // end checkRemoteCommandMacro()
3995 
3996 //==============================================================================
4001 std::string Iterator::findRemoteCodeEditorIconName(GatewaySupervisor* supervisor,
4002  const std::string& targetSubsystem)
4003 {
4004  std::string iconString, folderPath;
4005  {
4006  std::lock_guard<std::mutex> lock(supervisor->remoteGatewayAppsMutex_);
4007  for(const auto& app : supervisor->remoteGatewayApps_)
4008  if(app.appInfo.name == targetSubsystem)
4009  {
4010  iconString = app.iconString;
4011  folderPath = app.parentIconFolderPath;
4012  break;
4013  }
4014  }
4015  if(iconString == "")
4016  return "";
4017 
4018  // 7 comma-separated fields per icon: caption, altText, uniqueWin, permissions,
4019  // picfn, linkurl, folderPath (see GatewaySupervisor::GetRemoteGatewayIcons)
4020  std::vector<std::string> parts = StringMacros::getVectorFromString(iconString, {','});
4021  for(size_t i = 0; i + 6 < parts.size(); i += 7)
4022  {
4023  const std::string& caption = parts[i];
4024  const std::string& linkurl = parts[i + 5];
4025  if(linkurl.find("CodeEditor") != std::string::npos ||
4026  caption.find("Code Editor") != std::string::npos)
4027  {
4028  // Desktop keys icons as "<folder>/<caption>" (folder without leading '/')
4029  std::string folder = parts[i + 6];
4030  while(folder.size() && folder[0] == '/')
4031  folder = folder.substr(1);
4032  while(folder.size() && folder.back() == '/')
4033  folder.pop_back();
4034  return folder.size() ? folder + "/" + caption : caption;
4035  }
4036  }
4037  __COUT_WARN__ << "No Code Editor desktop icon found for remote subsystem '"
4038  << targetSubsystem << "' (folder '" << folderPath
4039  << "'); output file links will show the path only." << __E__;
4040  return "";
4041 } // end findRemoteCodeEditorIconName()
ots::ConfigurationManagerRW::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false)
Definition: ConfigurationManagerRW.cc:901
ots::ConfigurationManager::getVersionAliases
std::map< std::string, std::map< std::string, TableVersion > > getVersionAliases(void) const
Definition: ConfigurationManager.cc:4092
ots::ConfigurationManager::getTableGroupFromAlias
std::pair< std::string, TableGroupKey > getTableGroupFromAlias(std::string systemAlias, ProgressBar *progressBar=0)
Getters.
Definition: ConfigurationManager.cc:3978
ots::ConfigurationTree::isDisconnected
bool isDisconnected(void) const
Definition: ConfigurationTree.cc:1773
ots::ConfigurationTree::getNode
ConfigurationTree getNode(const std::string &nodeName, bool doNotThrowOnBrokenUIDLinks=false) const
navigating between nodes
Definition: ConfigurationTree.cc:1269
ots::ConfigurationTree::getValue
void getValue(T &value) const
ots::FiniteStateMachine::getTimeInState
time_t getTimeInState(void) const
Definition: FiniteStateMachine.cc:81
ots::GatewaySupervisor::ipAddressForStateChangesOverUDP_
std::string ipAddressForStateChangesOverUDP_
IP used for UDP reverse-login propagation to remote gateways.
Definition: GatewaySupervisor.h:539
ots::IterateTable::COMMAND_BEGIN_LABEL
static const std::string COMMAND_BEGIN_LABEL
Definition: IterateTable.h:37
ots::Iterator::getIterationPlanStatus
void getIterationPlanStatus(HttpXmlDocument &xmldoc)
return state machine and iterator status
Definition: Iterator.cc:2831
ots::ProgressBar::readPercentageString
std::string readPercentageString()
return percentage complete as std::string
Definition: ProgressBar.cc:135
ots::SOAPMessenger::send
std::string send(XDAQ_CONST_CALL xdaq::ApplicationDescriptor *d, xoap::MessageReference message)
Definition: SOAPMessenger.cc:39
ots
defines used also by OtsConfigurationWizardSupervisor
Definition: ArtdaqOtsBuildInfo_module.cc:10
ots::StringMacros::getTimestampString
static std::string getTimestampString(const std::string &linuxTimeInSeconds)
Definition: StringMacros.cc:990
ots::StringMacros::extractXmlField
static std::string extractXmlField(const std::string &xml, const std::string &field, uint32_t occurrence, size_t after, size_t *returnFindPos=nullptr, const std::string &valueField="value=", const std::string &quoteType="'")
Definition: StringMacros.cc:2034
ots::StringMacros::getVectorFromString
static void getVectorFromString(const std::string &inputString, std::vector< std::string > &listToReturn, const std::set< char > &delimiter={',', '|', '&'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'}, std::vector< char > *listOfDelimiters=0, bool decodeURIComponents=false)
Definition: StringMacros.cc:1104
ots::StringMacros::splitMacroArgTriple
static bool splitMacroArgTriple(const std::string &inputString, std::string &name, std::string &initValue, std::string &stepValue, char delimiter=':')
Definition: StringMacros.cc:1242
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
setToString ~
ots::StringMacros::vectorToString
static std::string vectorToString(const std::vector< T > &setToReturn, const std::string &delimeter=", ")
vectorToString ~
ots::StringMacros::isNumber
static bool isNumber(const std::string &stringToCheck)
Note: before call consider use of stringToCheck = StringMacros::convertEnvironmentVariables(stringToC...
Definition: StringMacros.cc:802
ots::StringMacros::decodeURIComponent
static std::string decodeURIComponent(const std::string &data)
Definition: StringMacros.cc:230
ots::StringMacros::getNumber
static bool getNumber(const std::string &s, T &retValue)