otsdaq  3.09.00
ConfigurationManagerRW.cc
1 #include "otsdaq/ConfigurationInterface/ConfigurationManagerRW.h"
2 
3 #include <dirent.h>
4 #include <chrono>
5 
6 using namespace ots;
7 
8 #undef __MF_SUBJECT__
9 #define __MF_SUBJECT__ "ConfigurationManagerRW"
10 
11 #define TABLE_INFO_PATH std::string(__ENV__("TABLE_INFO_PATH")) + "/"
12 #define TABLE_INFO_EXT "Info.xml"
13 
14 #define CORE_TABLE_INFO_FILENAME \
15  ((getenv("SERVICE_DATA_PATH") == NULL) \
16  ? (std::string(__ENV__("USER_DATA")) + "/ServiceData") \
17  : (std::string(__ENV__("SERVICE_DATA_PATH")))) + \
18  "/CoreTableInfoNames.dat"
19 
20 std::atomic<bool> ConfigurationManagerRW::firstTimeConstructed_ = true;
21 
22 std::mutex ConfigurationManagerRW::versionCreationTimeCacheMutex_;
23 std::map<std::string, std::map<TableVersion, time_t>>
24  ConfigurationManagerRW::versionCreationTimeCache_;
25 
26 //==============================================================================
28 ConfigurationManagerRW::ConfigurationManagerRW(const std::string& username)
29  : ConfigurationManager(username) // for use as author of new views
30 {
31  __GEN_COUT__ << "Instantiating Config Manager with Write Access! (for " << username
32  << ") time=" << time(0) << " runTimeSeconds()=" << runTimeSeconds()
33  << __E__;
34 
35  theInterface_ = ConfigurationInterface::getInstance(
36  ConfigurationInterface::CONFIGURATION_MODE::
37  ARTDAQ_DATABASE); // false to use artdaq DB
38 
39  //=========================
40  // dump names of core tables (so UpdateOTS.sh can copy core tables for user)
41  // only if table does not exist
42  if(firstTimeConstructed_)
43  {
44  firstTimeConstructed_ = false;
45 
46  // make table group history directory here and at Gateway Supervisor (just in case)
47  mkdir((ConfigurationManager::LAST_TABLE_GROUP_SAVE_PATH).c_str(), 0755);
48 
49  const std::set<std::string>& contextMemberNames = getFixedContextMemberNames();
50  const std::set<std::string>& backboneMemberNames = getBackboneMemberNames();
51  const std::set<std::string>& iterateMemberNames = getIterateMemberNames();
52 
53  FILE* fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "r");
54 
55  if(fp) // check for all core table names in file, and force their presence
56  {
57  std::vector<unsigned int> foundVector;
58  char line[100];
59  for(const auto& name : contextMemberNames)
60  {
61  foundVector.push_back(false);
62  rewind(fp);
63  while(fgets(line, 100, fp))
64  {
65  if(strlen(line) < 1)
66  continue;
67  line[strlen(line) - 1] = '\0'; // remove endline
68  if(strcmp(line, ("ContextGroup/" + name).c_str()) == 0) // is match?
69  {
70  foundVector.back() = true;
71  break;
72  }
73  }
74  }
75 
76  for(const auto& name : backboneMemberNames)
77  {
78  foundVector.push_back(false);
79  rewind(fp);
80  while(fgets(line, 100, fp))
81  {
82  if(strlen(line) < 1)
83  continue;
84  line[strlen(line) - 1] = '\0'; // remove endline
85  if(strcmp(line, ("BackboneGroup/" + name).c_str()) == 0) // is match?
86  {
87  foundVector.back() = true;
88  break;
89  }
90  }
91  }
92 
93  for(const auto& name : iterateMemberNames)
94  {
95  foundVector.push_back(false);
96  rewind(fp);
97  while(fgets(line, 100, fp))
98  {
99  if(strlen(line) < 1)
100  continue;
101  line[strlen(line) - 1] = '\0'; // remove endline
102  if(strcmp(line, ("IterateGroup/" + name).c_str()) == 0) // is match?
103  {
104  foundVector.back() = true;
105  break;
106  }
107  }
108  }
109 
110  //look for optional Context table ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE
111  {
112  foundVector.push_back(false);
113  rewind(fp);
114  while(fgets(line, 100, fp))
115  {
116  if(strlen(line) < 1)
117  continue;
118  line[strlen(line) - 1] = '\0'; // remove endline
119  if(strcmp(line,
120  ("ContextGroup/" +
121  ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE)
122  .c_str()) == 0) // is match?
123  {
124  foundVector.back() = true;
125  break;
126  }
127  }
128  }
129 
130  fclose(fp);
131 
132  // open file for appending the missing names
133  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "a");
134  if(fp)
135  {
136  unsigned int i = 0;
137  for(const auto& name : contextMemberNames)
138  {
139  if(!foundVector[i])
140  fprintf(fp, "\nContextGroup/%s", name.c_str());
141 
142  ++i;
143  }
144  for(const auto& name : backboneMemberNames)
145  {
146  if(!foundVector[i])
147  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
148 
149  ++i;
150  }
151  for(const auto& name : iterateMemberNames)
152  {
153  if(!foundVector[i])
154  fprintf(fp, "\nIterateGroup/%s", name.c_str());
155 
156  ++i;
157  }
158 
159  //last is optional Context table ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE
160  if(!foundVector[i])
161  fprintf(
162  fp,
163  "\nContextGroup/%s",
164  ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE.c_str());
165 
166  fclose(fp);
167  }
168  else
169  {
170  __SS__ << "Failed to open core table info file for appending: "
171  << CORE_TABLE_INFO_FILENAME << __E__;
172  __SS_THROW__;
173  }
174  }
175  else
176  {
177  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "w");
178  if(fp)
179  {
180  fprintf(fp, "ARTDAQ/*");
181  fprintf(fp, "\nConfigCore/*");
182  for(const auto& name : contextMemberNames)
183  fprintf(fp, "\nContextGroup/%s", name.c_str());
184  for(const auto& name : backboneMemberNames)
185  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
186  for(const auto& name : iterateMemberNames)
187  fprintf(fp, "\nIterateGroup/%s", name.c_str());
188  fclose(fp);
189  }
190  else
191  {
192  __SS__ << "Failed to open core table info file: "
193  << CORE_TABLE_INFO_FILENAME << __E__;
194  __SS_THROW__;
195  }
196  }
197  } // end dump names of core tables
198 
199  __GEN_COUTV__(runTimeSeconds());
200 } // end constructor
201 
202 //==============================================================================
209 const std::map<std::string, TableInfo>& ConfigurationManagerRW::getAllTableInfo(
210  bool refresh /* = false */,
211  std::string* accumulatedWarnings /* = 0 */,
212  const std::string& errorFilterName /* = "" */,
213  bool getGroupKeys /* = false */,
214  bool getGroupInfo /* = false */,
215  bool initializeActiveGroups /* = false */)
216 {
217  // allTableInfo_ is container to be returned
218 
219  if(!refresh)
220  return allTableInfo_;
221 
222  // else refresh!
223  allTableInfo_.clear();
224 
225  TableBase* table;
226 
227  // existing configurations are defined by which infos are in TABLE_INFO_PATH
228  // can test that the class exists based on this
229  // and then which versions
230  __GEN_COUT__ << "======================================================== "
231  "getAllTableInfo start runTimeSeconds()="
232  << runTimeSeconds() << __E__;
233  {
234  __GEN_COUT__ << "Refreshing all! Extracting list of tables..." << __E__;
235  DIR* pDIR;
236  struct dirent* entry;
237  std::string path = TABLE_INFO_PATH;
238  char fileExt[] = TABLE_INFO_EXT;
239  const unsigned char MIN_TABLE_NAME_SZ = 3;
240 
241  const int numOfThreads = StringMacros::getConcurrencyCount() / 2;
242  __GEN_COUT__ << " getConcurrencyCount " << StringMacros::getConcurrencyCount()
243  << " ==> " << numOfThreads << " threads." << __E__;
244  if(numOfThreads < 2) // no multi-threading
245  {
246  if((pDIR = opendir(path.c_str())) != 0)
247  {
248  while((entry = readdir(pDIR)) != 0)
249  {
250  // enforce table name length
251  if(strlen(entry->d_name) < strlen(fileExt) + MIN_TABLE_NAME_SZ)
252  continue;
253 
254  // find file names with correct file extenstion
255  if(strcmp(&(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
256  fileExt) != 0)
257  continue; // skip different extentions
258 
259  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] =
260  '\0'; // remove file extension to get table name
261 
262  // 0 will force the creation of new instance (and reload from Info)
263  table = 0;
264 
265  try // only add valid table instances to maps
266  {
267  theInterface_->get(table,
268  entry->d_name,
269  0,
270  0,
271  true); // dont fill
272  }
273  catch(cet::exception const&)
274  {
275  if(table)
276  delete table;
277  table = 0;
278 
279  __GEN_COUT__ << "Skipping! No valid class found for... "
280  << entry->d_name << "\n";
281  continue;
282  }
283  catch(std::runtime_error& e)
284  {
285  if(table)
286  delete table;
287  table = 0;
288 
289  __GEN_COUT__ << "Skipping! No valid class found for... "
290  << entry->d_name << "\n";
291  __GEN_COUT__ << "Error: " << e.what() << __E__;
292 
293  // for a runtime_error, it is likely that columns are the problem
294  // the Table Editor needs to still fix these.. so attempt to
295  // proceed.
296  if(accumulatedWarnings)
297  {
298  if(errorFilterName == "" || errorFilterName == entry->d_name)
299  {
300  *accumulatedWarnings += std::string("\nIn table '") +
301  entry->d_name + "'..." +
302  e.what(); // global accumulate
303 
304  __SS__ << "Attempting to allow illegal columns!" << __E__;
305  *accumulatedWarnings += ss.str();
306  }
307 
308  // attempt to recover and build a mock-up
309  __GEN_COUT__ << "Attempting to allow illegal columns!"
310  << __E__;
311 
312  std::string returnedAccumulatedErrors;
313  try
314  {
315  table = new TableBase(entry->d_name,
316  &returnedAccumulatedErrors);
317  }
318  catch(...)
319  {
320  __GEN_COUT__ << "Skipping! Allowing illegal columns "
321  "didn't work either... "
322  << entry->d_name << "\n";
323  continue;
324  }
325  __GEN_COUT__
326  << "Error (but allowed): " << returnedAccumulatedErrors
327  << __E__;
328 
329  if(errorFilterName == "" || errorFilterName == entry->d_name)
330  *accumulatedWarnings +=
331  std::string("\nIn table '") + entry->d_name + "'..." +
332  returnedAccumulatedErrors; // global accumulate
333  }
334  else
335  continue;
336  }
337 
338  if(nameToTableMap_[entry->d_name]) // handle if instance existed
339  {
340  // copy the existing temporary versions! (or else all is lost)
341  std::set<TableVersion> versions =
342  nameToTableMap_[entry->d_name]->getStoredVersions();
343  for(auto& version : versions)
344  if(version.isTemporaryVersion())
345  {
346  try // do NOT let TableView::init() throw here
347  {
348  nameToTableMap_[entry->d_name]->setActiveView(
349  version);
350  table->copyView( // this calls TableView::init()
351  nameToTableMap_[entry->d_name]->getView(),
352  version,
353  username_);
354  }
355  catch(
356  ...) // do NOT let invalid temporary version throw at this
357  // point
358  {
359  } // just trust configurationBase throws out the failed version
360  }
361 
362  delete nameToTableMap_[entry->d_name];
363  nameToTableMap_[entry->d_name] = 0;
364  }
365 
366  nameToTableMap_[entry->d_name] = table;
367 
368  allTableInfo_[entry->d_name].tablePtr_ = table;
369  allTableInfo_[entry->d_name].versions_ =
370  theInterface_->getVersions(table);
371 
372  // also add any existing temporary versions to all table info
373  // because the interface wont find those versions
374  std::set<TableVersion> versions =
375  nameToTableMap_[entry->d_name]->getStoredVersions();
376  for(auto& version : versions)
377  if(version.isTemporaryVersion())
378  {
379  allTableInfo_[entry->d_name].versions_.emplace(version);
380  }
381  } //end table name handling from directory
382  closedir(pDIR);
383  }
384  }
385  else //multi-threading
386  {
387  int threadsLaunched = 0;
388  int foundThreadIndex = 0;
389  std::string tableName;
390 
391  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
392  for(int i = 0; i < numOfThreads; ++i)
393  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
394 
395  std::vector<std::shared_ptr<ots::TableInfo>> sharedTableInfoPtrs;
396 
397  if((pDIR = opendir(path.c_str())) != 0)
398  {
399  while((entry = readdir(pDIR)) != 0)
400  {
401  // enforce table name length
402  if(strlen(entry->d_name) < strlen(fileExt) + MIN_TABLE_NAME_SZ)
403  continue;
404 
405  // find file names with correct file extenstion
406  if(strcmp(&(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
407  fileExt) != 0)
408  continue; // skip different extentions
409 
410  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] =
411  '\0'; // remove file extension to get table name
412  tableName =
413  entry
414  ->d_name; //copy immediate, thread does not like using pointer to char*, corrupts immediately on wraparound
415 
416  //make temporary table info for thread
417  sharedTableInfoPtrs.push_back(std::make_shared<ots::TableInfo>());
418  sharedTableInfoPtrs.back()->accumulatedWarnings_ =
419  accumulatedWarnings ? "ALLOW"
420  : ""; //mark to allow accumulated warnings
421 
422  if(threadsLaunched >= numOfThreads)
423  {
424  //find availableThreadIndex
425  foundThreadIndex = -1;
426  size_t ii = 0;
427  while(foundThreadIndex == -1)
428  {
429  for(int i = 0; i < numOfThreads; ++i)
430  if(*(threadDone[i]))
431  {
432  foundThreadIndex = i;
433  break;
434  }
435  if(foundThreadIndex == -1)
436  {
437  __GEN_COUT_TYPE__(TLVL_DEBUG + 2)
438  << __COUT_HDR__
439  << "Waiting for available thread... iteration # "
440  << ii << __E__;
441  if(++ii > 100 /* 1s */ * 10)
442  {
443  __GEN_SS__ << "Threads seem to be stuck getting "
444  "table info! Timeout while waiting..."
445  << __E__;
446  __GEN_SS_THROW__;
447  }
448  usleep(10000);
449  }
450  } //end thread search loop
451  threadsLaunched = numOfThreads - 1;
452  }
453  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex
454  << " for table " << tableName << __E__;
455 
456  *(threadDone[foundThreadIndex]) = false;
457  std::thread(
458  [](ConfigurationManagerRW* theCfgMgr,
459  std::string theTableName,
460  TableBase* existingTable,
461  std::shared_ptr<ots::TableInfo> theTableInfo,
462  std::shared_ptr<std::atomic<bool>> theThreadDone) {
463  ConfigurationManagerRW::loadTableInfoThread(theCfgMgr,
464  theTableName,
465  existingTable,
466  theTableInfo,
467  theThreadDone);
468  },
469  this,
470  tableName,
471  nameToTableMap_[tableName],
472  sharedTableInfoPtrs.back(),
473  threadDone[foundThreadIndex])
474  .detach();
475 
476  ++threadsLaunched;
477  ++foundThreadIndex;
478  } //end table name handling from directory
479  closedir(pDIR);
480  } //end tableInfo thread loop
481 
482  //check for all threads done
483  do
484  {
485  foundThreadIndex = -1;
486  for(int i = 0; i < numOfThreads; ++i)
487  if(!*(threadDone[i]))
488  {
489  foundThreadIndex = i;
490  break;
491  }
492  if(foundThreadIndex != -1)
493  {
494  __GEN_COUTT__ << "Waiting for thread to finish... "
495  << foundThreadIndex << __E__;
496  usleep(10000);
497  }
498  } while(foundThreadIndex != -1); //end thread done search loop
499 
500  //threads done now, so copy table info
501  for(auto& tableInfo : sharedTableInfoPtrs)
502  {
503  if(tableInfo->tablePtr_ == nullptr)
504  {
505  __SS__ << "Fatal error occurred loading table info into cache! "
506  "Perhaps there is an illegal Table schema definition? "
507  "Check logs to resolve."
508  << __E__;
509  __SS_THROW__;
510  }
511  __GEN_COUT_TYPE__(TLVL_DEBUG + 3)
512  << __COUT_HDR__ << "Copying table info for "
513  << tableInfo->tablePtr_->getTableName() << __E__;
514  nameToTableMap_[tableInfo->tablePtr_->getTableName()] =
515  tableInfo->tablePtr_;
516  allTableInfo_[tableInfo->tablePtr_->getTableName()].tablePtr_ =
517  tableInfo->tablePtr_;
518  allTableInfo_[tableInfo->tablePtr_->getTableName()].versions_ =
519  tableInfo->versions_;
520  } //end copy group info loop
521  }
522  __GEN_COUT__ << "Extracting list of tables complete." << __E__;
523  } //end Extracting list of tables
524 
525  // call init to load active versions by default, activate with warnings allowed (assuming development going on)
526  if(initializeActiveGroups)
527  {
528  __GEN_COUT__ << "Now initializing..." << __E__;
529  // if there is a filter name, do not include init warnings (it just scares people in the table editor)
530  std::string tmpAccumulateWarnings;
531  init(0 /*accumulatedErrors*/,
532  false /*initForWriteAccess*/,
533  accumulatedWarnings ? &tmpAccumulateWarnings : nullptr);
534 
535  if(accumulatedWarnings && errorFilterName == "")
536  *accumulatedWarnings += tmpAccumulateWarnings;
537  }
538  __GEN_COUT__ << "======================================================== "
539  "getAllTableInfo end runTimeSeconds()="
540  << runTimeSeconds() << __E__;
541 
542  // get Group Info too!
543  if(getGroupKeys || getGroupInfo)
544  {
545  allGroupInfo_.clear();
546  try
547  {
548  // build allGroupInfo_ for the ConfigurationManagerRW
549 
550  std::set<std::string /*name*/> tableGroups =
551  theInterface_->getAllTableGroupNames();
552  __GEN_COUT__ << "Number of Groups: " << tableGroups.size() << __E__;
553 
554  __GEN_COUTT__ << "Group Info start runTimeSeconds()=" << runTimeSeconds()
555  << __E__;
556 
557  TableGroupKey key;
558  std::string name;
559  for(const auto& fullName : tableGroups)
560  {
561  TableGroupKey::getGroupNameAndKey(fullName, name, key);
562  allGroupInfo_[name].keys_.emplace(key); //store in cache
563  }
564 
565  __GEN_COUTT__ << "Group Keys end runTimeSeconds()=" << runTimeSeconds()
566  << __E__;
567 
568  // for each group get member map & comment, author, time, and type for latest key
569  if(getGroupInfo)
570  {
571  const int numOfThreads = StringMacros::getConcurrencyCount() / 2;
572  __GEN_COUT__ << " getConcurrencyCount "
573  << StringMacros::getConcurrencyCount() << " ==> "
574  << numOfThreads << " threads." << __E__;
575  if(numOfThreads < 2) // no multi-threading
576  for(auto& groupInfo : allGroupInfo_)
577  {
578  try
579  {
580  groupInfo.second.latestKey_ = groupInfo.second.getLastKey();
581  loadTableGroup(
582  groupInfo.first /*groupName*/,
583  groupInfo.second.latestKey_,
584  false /*doActivate*/,
585  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
586  0 /*progressBar*/,
587  0 /*accumulateErrors*/,
588  &groupInfo.second.latestKeyGroupComment_,
589  &groupInfo.second.latestKeyGroupAuthor_,
590  &groupInfo.second.latestKeyGroupCreationTime_,
591  true /*doNotLoadMember*/,
592  &groupInfo.second.latestKeyGroupTypeString_);
593  }
594  catch(...)
595  {
596  __GEN_COUT_WARN__ << "Error occurred loading latest group "
597  "info into cache for '"
598  << groupInfo.first << "("
599  << groupInfo.second.latestKey_ << ")'..."
600  << __E__;
601  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
602  groupInfo.second.latestKeyGroupComment_ =
603  ConfigurationManager::UNKNOWN_INFO;
604  groupInfo.second.latestKeyGroupAuthor_ =
605  ConfigurationManager::UNKNOWN_INFO;
606  groupInfo.second.latestKeyGroupCreationTime_ =
607  ConfigurationManager::UNKNOWN_TIME;
608  groupInfo.second.latestKeyGroupTypeString_ =
609  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
610  groupInfo.second.latestKeyMemberMap_ = {};
611  }
612  } // end group info loop
613  else //multi-threading
614  {
615  int threadsLaunched = 0;
616  int foundThreadIndex = 0;
617 
618  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
619  for(int i = 0; i < numOfThreads; ++i)
620  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
621 
622  std::vector<std::shared_ptr<ots::GroupInfo>> sharedGroupInfoPtrs;
623 
624  for(auto& groupInfo : allGroupInfo_)
625  {
626  //make temporary group info for thread
627  sharedGroupInfoPtrs.push_back(std::make_shared<ots::GroupInfo>());
628 
629  if(threadsLaunched >= numOfThreads)
630  {
631  //find availableThreadIndex
632  foundThreadIndex = -1;
633  while(foundThreadIndex == -1)
634  {
635  for(int i = 0; i < numOfThreads; ++i)
636  if(*(threadDone[i]))
637  {
638  foundThreadIndex = i;
639  break;
640  }
641  if(foundThreadIndex == -1)
642  {
643  __GEN_COUTT__ << "Waiting for available thread..."
644  << __E__;
645  usleep(10000);
646  }
647  } //end thread search loop
648  threadsLaunched = numOfThreads - 1;
649  }
650  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex
651  << " for " << groupInfo.first << "("
652  << groupInfo.second.getLastKey() << ")" << __E__;
653 
654  *(threadDone[foundThreadIndex]) = false;
655 
656  std::thread(
657  [](ConfigurationManagerRW* theCfgMgr,
658  std::string theGroupName,
659  ots::TableGroupKey theGroupKey,
660  std::shared_ptr<ots::GroupInfo> theGroupInfo,
661  std::shared_ptr<std::atomic<bool>> theThreadDone) {
662  ConfigurationManagerRW::loadTableGroupThread(
663  theCfgMgr,
664  theGroupName,
665  theGroupKey,
666  theGroupInfo,
667  theThreadDone);
668  },
669  this,
670  groupInfo.first,
671  groupInfo.second.getLastKey(),
672  sharedGroupInfoPtrs.back(),
673  threadDone[foundThreadIndex])
674  .detach();
675 
676  ++threadsLaunched;
677  ++foundThreadIndex;
678  } //end groupInfo thread loop
679 
680  //check for all threads done
681  do
682  {
683  foundThreadIndex = -1;
684  for(int i = 0; i < numOfThreads; ++i)
685  if(!*(threadDone[i]))
686  {
687  foundThreadIndex = i;
688  break;
689  }
690  if(foundThreadIndex != -1)
691  {
692  __GEN_COUTT__ << "Waiting for thread to finish... "
693  << foundThreadIndex << __E__;
694  usleep(10000);
695  }
696  } while(foundThreadIndex != -1); //end thread done search loop
697 
698  //threads done now, so copy group info
699  size_t i = 0;
700  for(auto& groupInfo : allGroupInfo_)
701  {
702  groupInfo.second.latestKey_ = sharedGroupInfoPtrs[i]->latestKey_;
703  groupInfo.second.latestKeyGroupComment_ =
704  sharedGroupInfoPtrs[i]->latestKeyGroupComment_;
705  groupInfo.second.latestKeyGroupAuthor_ =
706  sharedGroupInfoPtrs[i]->latestKeyGroupAuthor_;
707  groupInfo.second.latestKeyGroupCreationTime_ =
708  sharedGroupInfoPtrs[i]->latestKeyGroupCreationTime_;
709  groupInfo.second.latestKeyGroupTypeString_ =
710  sharedGroupInfoPtrs[i]->latestKeyGroupTypeString_;
711  groupInfo.second.latestKeyMemberMap_ =
712  sharedGroupInfoPtrs[i]->latestKeyMemberMap_;
713  ++i;
714  } //end copy group info loop
715 
716  } //end multi-thread handling
717  }
718  } // end get group info
719  catch(const std::runtime_error& e)
720  {
721  __SS__
722  << "A fatal error occurred reading the info for all table groups. Error: "
723  << e.what() << __E__;
724  __GEN_COUT_ERR__ << "Error at time: " << time(0) << "\n" << ss.str();
725  if(accumulatedWarnings)
726  *accumulatedWarnings += ss.str();
727  else
728  throw;
729  }
730  catch(...)
731  {
732  __SS__ << "An unknown fatal error occurred reading the info for all table "
733  "groups."
734  << __E__;
735  try
736  {
737  throw;
738  } //one more try to printout extra info
739  catch(const std::exception& e)
740  {
741  ss << "Exception message: " << e.what();
742  }
743  catch(...)
744  {
745  }
746  __GEN_COUT_ERR__ << "\n" << ss.str();
747  if(accumulatedWarnings)
748  *accumulatedWarnings += ss.str();
749  else
750  throw;
751  }
752  __GEN_COUTT__ << "Group Info end runTimeSeconds()=" << runTimeSeconds() << __E__;
753  } //end getGroupInfo
754  else
755  __GEN_COUTT__ << "Table Info end runTimeSeconds()=" << runTimeSeconds() << __E__;
756 
757  return allTableInfo_;
758 } // end getAllTableInfo()
759 
760 //==============================================================================
762 void ConfigurationManagerRW::loadTableInfoThread(
763  ConfigurationManagerRW* cfgMgr,
764  std::string tableName,
765  TableBase* existingTable,
766  std::shared_ptr<ots::TableInfo> tableInfo,
767  std::shared_ptr<std::atomic<bool>> threadDone)
768 try
769 {
770  __COUTT__ << "Thread started... table " << tableName << __E__;
771 
772  // 0 will force the creation of new instance (and reload from Info)
773  tableInfo->tablePtr_ = 0;
774 
775  try // only add valid table instances to maps
776  {
777  cfgMgr->theInterface_->get(tableInfo->tablePtr_,
778  tableName,
779  0,
780  0,
781  true); // dont fill
782  }
783  catch(cet::exception const&)
784  {
785  if(tableInfo->tablePtr_)
786  delete tableInfo->tablePtr_;
787  tableInfo->tablePtr_ = 0;
788 
789  __COUT__ << "Skipping! No valid class found for... " << tableName << "\n";
790  *(threadDone) = true;
791  return;
792  }
793  catch(std::runtime_error& e)
794  {
795  if(tableInfo->tablePtr_)
796  delete tableInfo->tablePtr_;
797  tableInfo->tablePtr_ = 0;
798 
799  __COUT__ << "Skipping! No valid class found for... " << tableName << "\n";
800  __COUTT__ << "Error: " << e.what() << __E__;
801 
802  // for a runtime_error, it is likely that columns are the problem
803  // the Table Editor needs to still fix these.. so attempt to
804  // proceed.
805  if(tableInfo->accumulatedWarnings_ == "ALLOW")
806  {
807  tableInfo->accumulatedWarnings_ = "";
808  if(1) //errorFilterName == "" || errorFilterName == tableName)
809  {
810  tableInfo->accumulatedWarnings_ += std::string("\nIn table '") +
811  tableName + "'..." +
812  e.what(); // global accumulate
813 
814  __SS__ << "Attempting to allow illegal columns!" << __E__;
815  tableInfo->accumulatedWarnings_ += ss.str();
816  }
817 
818  // attempt to recover and build a mock-up
819  __COUT__ << "Attempting to allow illegal columns!" << __E__;
820 
821  std::string returnedAccumulatedErrors;
822  try
823  {
824  tableInfo->tablePtr_ =
825  new TableBase(tableName, &returnedAccumulatedErrors);
826  }
827  catch(...)
828  {
829  __COUT__ << "Skipping! Allowing illegal columns didn't work either... "
830  << tableName << "\n";
831  *(threadDone) = true;
832  return;
833  }
834  __COUT_WARN__ << "Error (but allowed): " << returnedAccumulatedErrors
835  << __E__;
836 
837  if(1) //errorFilterName == "" || errorFilterName == entry->d_name)
838  tableInfo->accumulatedWarnings_ +=
839  std::string("\nIn table '") + tableName + "'..." +
840  returnedAccumulatedErrors; // global accumulate
841  }
842  else
843  {
844  tableInfo->accumulatedWarnings_ = "";
845  *(threadDone) = true;
846  return;
847  }
848  }
849 
850  if(existingTable) // handle if instance existed
851  {
852  __COUTT__ << "Copying temporary version from existing table object for "
853  << tableName << __E__;
854  // copy the existing temporary versions! (or else all is lost)
855  std::set<TableVersion> versions = existingTable->getStoredVersions();
856  for(auto& version : versions)
857  if(version.isTemporaryVersion())
858  {
859  try // do NOT let TableView::init() throw here
860  {
861  existingTable->setActiveView(version);
862  tableInfo->tablePtr_->copyView( // this calls TableView::init()
863  existingTable->getView(),
864  version,
865  cfgMgr->username_);
866  }
867  catch(...) // do NOT let invalid temporary version throw at this
868  // point
869  {
870  } // just trust configurationBase throws out the failed version
871  }
872 
873  delete existingTable;
874  existingTable = 0;
875  }
876 
877  tableInfo->versions_ = cfgMgr->theInterface_->getVersions(tableInfo->tablePtr_);
878 
879  // also add any existing temporary versions to all table info
880  // because the interface wont find those versions
881  std::set<TableVersion> versions = tableInfo->tablePtr_->getStoredVersions();
882  for(auto& version : versions)
883  if(version.isTemporaryVersion())
884  {
885  tableInfo->versions_.emplace(version);
886  }
887 
888  __COUTT__ << "Thread done... table " << tableName << __E__;
889  *(threadDone) = true;
890 } // end loadTableInfoThread
891 catch(...)
892 {
893  __COUT_ERR__ << "Error occurred loading latest table info into cache for '"
894  << tableName << "'..." << __E__;
895  *(threadDone) = true;
896 } // end loadTableInfoThread catch
897 
898 //==============================================================================
901 void ConfigurationManagerRW::loadTableGroup(
902  const std::string& groupName,
903  const TableGroupKey& groupKey,
904  bool doActivate /*=false*/,
905  std::map<std::string /*table name*/, TableVersion>*
906  groupMembers /*=0 , note: db time intensive! */,
907  ProgressBar* progressBar /*=0*/,
908  std::string* accumulatedWarnings /*=0*/,
909  std::string* groupComment /*=0 , note: in metadata */,
910  std::string* groupAuthor /*=0 , note: in metadata */,
911  std::string* groupCreateTime /*=0 , note: in metadata */,
912  bool doNotLoadMembers /*=false*/,
913  std::string* groupTypeString /*=0 , note: db time intensive! */,
914  std::map<std::string /*name*/, std::string /*alias*/>*
915  groupAliases /*=0 , note: in metadata */,
916  ConfigurationManager::LoadGroupType
917  groupTypeToLoad /*=ConfigurationManager::LoadGroupType::ALL_TYPES*/,
918  bool ignoreVersionTracking /*=false*/)
919 {
920  ConfigurationManager::loadTableGroup(groupName,
921  groupKey,
922  doActivate,
923  groupMembers,
924  progressBar,
925  accumulatedWarnings,
926  groupComment,
927  groupAuthor,
928  groupCreateTime,
929  doNotLoadMembers,
930  groupTypeString,
931  groupAliases,
932  groupTypeToLoad,
933  ignoreVersionTracking);
934 
935  if(!groupMembers || !groupMembers->size() || !groupComment || groupKey.isInvalid() ||
936  !groupAuthor || !groupCreateTime)
937  return;
938 
939  //treat successfull load as latest group key
940  auto groupInfo = allGroupInfo_.find(groupName);
941  if(groupInfo == allGroupInfo_.end())
942  return; //ignore if no group info cache
943 
944  groupInfo->second.latestKey_ = groupKey;
945  groupInfo->second.latestKeyGroupComment_ = *groupComment;
946  groupInfo->second.latestKeyGroupAuthor_ = *groupAuthor;
947  groupInfo->second.latestKeyGroupCreationTime_ = *groupCreateTime;
948  if(groupTypeString) //assume unlikely to change types
949  groupInfo->second.latestKeyGroupTypeString_ = *groupTypeString;
950  groupInfo->second.latestKeyMemberMap_ = *groupMembers;
951 
952 } //end loadTableGroup() RW version
953 
954 //==============================================================================
956 void ConfigurationManagerRW::loadTableGroupThread(
957  ConfigurationManagerRW* cfgMgr,
958  std::string groupName,
959  ots::TableGroupKey groupKey,
960  std::shared_ptr<ots::GroupInfo> groupInfo,
961  std::shared_ptr<std::atomic<bool>> threadDone)
962 try
963 {
964  __COUTT__ << "Thread started... " << groupName << "(" << groupKey << ")" << __E__;
965 
966  groupInfo->latestKey_ = groupKey;
967  cfgMgr->loadTableGroup(groupName,
968  groupKey,
969  false /*doActivate*/,
970  &(groupInfo->latestKeyMemberMap_) /*groupMembers*/,
971  0 /*progressBar*/,
972  0 /*accumulateErrors*/,
973  &(groupInfo->latestKeyGroupComment_),
974  &(groupInfo->latestKeyGroupAuthor_),
975  &(groupInfo->latestKeyGroupCreationTime_),
976  true /*doNotLoadMember*/,
977  &(groupInfo->latestKeyGroupTypeString_));
978 
979  *(threadDone) = true;
980 } // end loadTableGroupThread
981 catch(...)
982 {
983  __COUT_WARN__ << "Error occurred loading latest group info into cache for '"
984  << groupName << "(" << groupInfo->latestKey_ << ")'..." << __E__;
985  groupInfo->latestKey_ = TableGroupKey::INVALID;
986  groupInfo->latestKeyGroupComment_ = ConfigurationManager::UNKNOWN_INFO;
987  groupInfo->latestKeyGroupAuthor_ = ConfigurationManager::UNKNOWN_INFO;
988  groupInfo->latestKeyGroupCreationTime_ = ConfigurationManager::UNKNOWN_TIME;
989  groupInfo->latestKeyGroupTypeString_ = ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
990  groupInfo->latestKeyMemberMap_ = {};
991  *(threadDone) = true;
992 } // end loadTableGroupThread catch
993 
994 //==============================================================================
998 void ConfigurationManagerRW::compareTableGroupThread(
999  ConfigurationManagerRW* cfgMgr,
1000  std::string groupName,
1001  ots::TableGroupKey groupKeyToCompare,
1002  const std::map<std::string, TableVersion>& groupMemberMap,
1003  const std::map<std::string /*name*/, std::string /*alias*/>& memberTableAliases,
1004  std::atomic<bool>* foundIdentical,
1005  ots::TableGroupKey* identicalKey,
1006  std::mutex* threadMutex,
1007  std::shared_ptr<std::atomic<bool>> threadDone)
1008 try
1009 {
1010  std::map<std::string /*name*/, TableVersion /*version*/> compareToMemberMap;
1011  std::map<std::string /*name*/, std::string /*alias*/> compareToMemberTableAliases;
1012  std::map<std::string /*name*/, std::string /*alias*/>*
1013  compareToMemberTableAliasesPtr = nullptr;
1014  if(memberTableAliases
1015  .size()) //only give pointer if necessary, without will load group faster
1016  compareToMemberTableAliasesPtr = &compareToMemberTableAliases;
1017 
1018  cfgMgr->loadTableGroup(groupName,
1019  groupKeyToCompare,
1020  false /*doActivate*/,
1021  &compareToMemberMap /*memberMap*/,
1022  0, /*progressBar*/
1023  0, /*accumulatedWarnings*/
1024  0, /*groupComment*/
1025  0,
1026  0, /*null pointers*/
1027  true /*doNotLoadMember*/,
1028  0 /*groupTypeString*/,
1029  compareToMemberTableAliasesPtr);
1030 
1031  bool debug = false;
1032  if(TTEST(9)) // = DEBUG+9
1033  {
1034  debug = true;
1035  for(auto& memberPair : groupMemberMap)
1036  __COUTS__(9) << "member " << memberPair.first << " (" << memberPair.second
1037  << ")" << __E__;
1038  for(auto& memberPair : compareToMemberMap)
1039  __COUTS__(9) << "compare " << groupName << " (" << groupKeyToCompare
1040  << ") member:" << memberPair.first << " (" << memberPair.second
1041  << ")" << __E__;
1042  }
1043 
1044  bool isDifferent = false;
1045  for(auto& memberPair : groupMemberMap)
1046  {
1047  //groups can be different if Alias names are different
1048  // or if resolved Alias versions are different (for the current active backbone)
1049  if(memberTableAliases.find(memberPair.first) != memberTableAliases.end())
1050  {
1051  // handle this table as alias, not version
1052  if(compareToMemberTableAliases.find(memberPair.first) ==
1053  compareToMemberTableAliases.end() || // alias is missing
1054  memberTableAliases.at(memberPair.first) !=
1055  compareToMemberTableAliases.at(memberPair.first))
1056  { // then different
1057  isDifferent = true;
1058  if(debug)
1059  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1060  << ") on alias " << memberPair.first << __E__;
1061  break;
1062  }
1063  } // else check if compareTo group is using an alias for table
1064  else if(compareToMemberTableAliases.find(memberPair.first) !=
1065  compareToMemberTableAliases.end())
1066  {
1067  // then different
1068  isDifferent = true;
1069  if(debug)
1070  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1071  << ") on reverse alias " << memberPair.first << __E__;
1072  break;
1073 
1074  } // else handle as table version comparison
1075 
1076  //normal table version check
1077  if(compareToMemberMap.find(memberPair.first) ==
1078  compareToMemberMap.end() || // name is missing
1079  memberPair.second !=
1080  compareToMemberMap.at(memberPair.first)) // or version mismatch
1081  {
1082  // then different
1083  isDifferent = true;
1084  if(debug)
1085  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1086  << ") on mismatch " << memberPair.first << __E__;
1087  break;
1088  }
1089  } //end table version mismatch checking loop
1090 
1091  // check member size for exact match
1092  if(!isDifferent &&
1093  groupMemberMap.size() !=
1094  compareToMemberMap
1095  .size()) // different size, so not same (groupMemberMap is a subset of memberPairs)
1096  {
1097  isDifferent = true;
1098 
1099  if(debug)
1100  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare << ") on size "
1101  << __E__;
1102  }
1103 
1104  if(!isDifferent) //found an exact match!
1105  {
1106  *foundIdentical = true;
1107  __COUT__ << "=====> Found exact match with key: " << groupKeyToCompare << __E__;
1108 
1109  std::lock_guard<std::mutex> lock(*threadMutex);
1110  *identicalKey = groupKeyToCompare;
1111  }
1112 
1113  *(threadDone) = true;
1114 } // end compareTableGroupThread
1115 catch(...)
1116 {
1117  __COUT_WARN__ << "Error occurred comparing group '" << groupName << "("
1118  << groupKeyToCompare << ")'..." << __E__;
1119 
1120  *(threadDone) = true;
1121 } // end compareTableGroupThread catch
1122 
1123 //==============================================================================
1127 std::map<std::string /*table name*/,
1128  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
1129 ConfigurationManagerRW::getVersionAliases(void) const
1130 {
1131  std::map<std::string /*table name*/,
1132  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
1133  retMap = ConfigurationManager::getVersionAliases();
1134 
1135  // always have scratch alias for each table that has a scratch version
1136  // overwrite map entry if necessary
1137  if(!ConfigurationInterface::isVersionTrackingEnabled())
1138  for(const auto& tableInfo : allTableInfo_)
1139  for(const auto& version : tableInfo.second.versions_)
1140  if(version.isScratchVersion())
1141  retMap[tableInfo.first][ConfigurationManager::SCRATCH_VERSION_ALIAS] =
1142  TableVersion(TableVersion::SCRATCH);
1143 
1144  return retMap;
1145 } // end getVersionAliases()
1146 
1147 //==============================================================================
1151 void ConfigurationManagerRW::activateTableGroup(const std::string& tableGroupName,
1152  TableGroupKey tableGroupKey,
1153  std::string* accumulatedTreeErrors,
1154  std::string* groupTypeString)
1155 {
1156  try
1157  {
1158  loadTableGroup(tableGroupName,
1159  tableGroupKey,
1160  true, // loads and activates
1161  0, // no members needed
1162  0, // no progress bar
1163  accumulatedTreeErrors, // accumulate warnings or not
1164  0 /* groupComment */,
1165  0 /* groupAuthor */,
1166  0 /* groupCreateTime */,
1167  false /* doNotLoadMember */,
1168  groupTypeString);
1169  }
1170  catch(...)
1171  {
1172  __GEN_COUT_ERR__ << "There were errors, so de-activating group: "
1173  << tableGroupName << " (" << tableGroupKey << ")" << __E__;
1174  try // just in case any lingering pieces, let's deactivate
1175  {
1176  destroyTableGroup(tableGroupName, true);
1177  }
1178  catch(...)
1179  {
1180  }
1181  throw; // re-throw original exception
1182  }
1183 
1184  __GEN_COUT_INFO__ << "Updating persistent active groups to "
1185  << ConfigurationManager::ACTIVE_GROUPS_FILENAME << " ..." << __E__;
1186 
1187  __COUT_INFO__ << "Active Context table group: " << theContextTableGroup_ << "("
1188  << (theContextTableGroupKey_
1189  ? theContextTableGroupKey_->toString().c_str()
1190  : "-1")
1191  << ")" << __E__;
1192  __COUT_INFO__ << "Active Backbone table group: " << theBackboneTableGroup_ << "("
1193  << (theBackboneTableGroupKey_
1194  ? theBackboneTableGroupKey_->toString().c_str()
1195  : "-1")
1196  << ")" << __E__;
1197  __COUT_INFO__ << "Active Iterate table group: " << theIterateTableGroup_ << "("
1198  << (theIterateTableGroupKey_
1199  ? theIterateTableGroupKey_->toString().c_str()
1200  : "-1")
1201  << ")" << __E__;
1202  __COUT_INFO__ << "Active Configuration table group: " << theConfigurationTableGroup_
1203  << "("
1204  << (theConfigurationTableGroupKey_
1205  ? theConfigurationTableGroupKey_->toString().c_str()
1206  : "-1")
1207  << ")" << __E__;
1208 
1209  std::string fn = ConfigurationManager::ACTIVE_GROUPS_FILENAME;
1210  FILE* fp = fopen(fn.c_str(), "w");
1211  if(!fp)
1212  {
1213  __SS__ << "Fatal Error! Unable to open the file "
1214  << ConfigurationManager::ACTIVE_GROUPS_FILENAME
1215  << " for editing! Is there a permissions problem?" << __E__;
1216  __GEN_COUT_ERR__ << ss.str();
1217  __SS_THROW__;
1218  return;
1219  }
1220  fprintf(fp, "%s\n", theContextTableGroup_.c_str());
1221  fprintf(
1222  fp,
1223  "%s\n",
1224  theContextTableGroupKey_ ? theContextTableGroupKey_->toString().c_str() : "-1");
1225  fprintf(fp, "%s\n", theBackboneTableGroup_.c_str());
1226  fprintf(
1227  fp,
1228  "%s\n",
1229  theBackboneTableGroupKey_ ? theBackboneTableGroupKey_->toString().c_str() : "-1");
1230  fprintf(fp, "%s\n", theIterateTableGroup_.c_str());
1231  fprintf(
1232  fp,
1233  "%s\n",
1234  theIterateTableGroupKey_ ? theIterateTableGroupKey_->toString().c_str() : "-1");
1235  fprintf(fp, "%s\n", theConfigurationTableGroup_.c_str());
1236  fprintf(fp,
1237  "%s\n",
1238  theConfigurationTableGroupKey_
1239  ? theConfigurationTableGroupKey_->toString().c_str()
1240  : "-1");
1241  fclose(fp);
1242 
1243  // save last activated group
1244  {
1245  std::pair<std::string /*group name*/, TableGroupKey> activatedGroup(
1246  std::string(tableGroupName), tableGroupKey);
1247  if(theConfigurationTableGroupKey_ &&
1248  theConfigurationTableGroup_ == tableGroupName &&
1249  *theConfigurationTableGroupKey_ == tableGroupKey)
1250  {
1251  ConfigurationManager::saveGroupNameAndKey(
1252  activatedGroup,
1253  ConfigurationManager::LAST_ACTIVATED_CONFIG_GROUP_FILE,
1254  false /* appendMode */,
1255  username_);
1256  ConfigurationManager::saveGroupNameAndKey(
1257  activatedGroup,
1258  ConfigurationManager::ACTIVATED_CONFIGS_FILE,
1259  true /* appendMode */,
1260  username_);
1261  }
1262  else if(theContextTableGroupKey_ && theContextTableGroup_ == tableGroupName &&
1263  *theContextTableGroupKey_ == tableGroupKey)
1264  {
1265  ConfigurationManager::saveGroupNameAndKey(
1266  activatedGroup,
1267  ConfigurationManager::LAST_ACTIVATED_CONTEXT_GROUP_FILE,
1268  false /* appendMode */,
1269  username_);
1270  ConfigurationManager::saveGroupNameAndKey(
1271  activatedGroup,
1272  ConfigurationManager::ACTIVATED_CONTEXTS_FILE,
1273  true /* appendMode */,
1274  username_);
1275  }
1276  else if(theBackboneTableGroupKey_ && theBackboneTableGroup_ == tableGroupName &&
1277  *theBackboneTableGroupKey_ == tableGroupKey)
1278  {
1279  ConfigurationManager::saveGroupNameAndKey(
1280  activatedGroup,
1281  ConfigurationManager::LAST_ACTIVATED_BACKBONE_GROUP_FILE,
1282  false /* appendMode */,
1283  username_);
1284  ConfigurationManager::saveGroupNameAndKey(
1285  activatedGroup,
1286  ConfigurationManager::ACTIVATED_BACKBONES_FILE,
1287  true /* appendMode */,
1288  username_);
1289  }
1290  else if(theIterateTableGroupKey_ && theIterateTableGroup_ == tableGroupName &&
1291  *theIterateTableGroupKey_ == tableGroupKey)
1292  {
1293  ConfigurationManager::saveGroupNameAndKey(
1294  activatedGroup,
1295  ConfigurationManager::LAST_ACTIVATED_ITERATE_GROUP_FILE,
1296  false /* appendMode */,
1297  username_);
1298  ConfigurationManager::saveGroupNameAndKey(
1299  activatedGroup,
1300  ConfigurationManager::ACTIVATED_ITERATES_FILE,
1301  true /* appendMode */,
1302  username_);
1303  }
1304  } // end save last activated group
1305 
1306 } // end activateTableGroup()
1307 
1308 //==============================================================================
1312 TableVersion ConfigurationManagerRW::createTemporaryBackboneView(
1313  TableVersion sourceViewVersion)
1314 {
1315  __GEN_COUT_INFO__ << "Creating temporary backbone view from version "
1316  << sourceViewVersion << __E__;
1317 
1318  // find common available temporary version among backbone members
1319  TableVersion tmpVersion =
1320  TableVersion::getNextTemporaryVersion(); // get the default temporary version
1321  TableVersion retTmpVersion;
1322  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
1323  for(auto& name : backboneMemberNames)
1324  {
1325  retTmpVersion =
1326  ConfigurationManager::getTableByName(name)->getNextTemporaryVersion();
1327  if(retTmpVersion < tmpVersion)
1328  tmpVersion = retTmpVersion;
1329  }
1330 
1331  __GEN_COUT__ << "Common temporary backbone version found as " << tmpVersion << __E__;
1332 
1333  // create temporary views from source version to destination temporary version
1334  for(auto& name : backboneMemberNames)
1335  {
1336  retTmpVersion =
1337  getTableByName(name)->createTemporaryView(sourceViewVersion, tmpVersion);
1338  if(retTmpVersion != tmpVersion)
1339  {
1340  __SS__ << "Failure! Temporary view requested was " << tmpVersion
1341  << ". Mismatched temporary view created: " << retTmpVersion << __E__;
1342  __GEN_COUT_ERR__ << ss.str();
1343  __SS_THROW__;
1344  }
1345  }
1346 
1347  return tmpVersion;
1348 } // end createTemporaryBackboneView()
1349 
1350 //==============================================================================
1351 TableBase* ConfigurationManagerRW::getTableByName(const std::string& tableName)
1352 {
1353  if(nameToTableMap_.find(tableName) == nameToTableMap_.end())
1354  {
1355  if(tableName == ConfigurationManager::ARTDAQ_TOP_TABLE_NAME)
1356  {
1357  __GEN_COUT_WARN__
1358  << "Since target table was the artdaq top configuration level, "
1359  "attempting to help user by appending to core tables file: "
1360  << CORE_TABLE_INFO_FILENAME << __E__;
1361  FILE* fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "a");
1362  if(fp)
1363  {
1364  fprintf(fp, "\nARTDAQ/*");
1365  fclose(fp);
1366  }
1367  }
1368 
1369  __SS__ << "Table not found with name: " << tableName << __E__;
1370  size_t f;
1371  if((f = tableName.find(' ')) != std::string::npos)
1372  ss << "There was a space character found in the table name needle at "
1373  "position "
1374  << f << " in the string (was this intended?). " << __E__;
1375 
1376  ss << "\nIf you think this table should exist in the core set of tables, try "
1377  "running 'UpdateOTS.sh --tables' to update your tables, then relaunch ots."
1378  << __E__;
1379  ss << "\nTables must be defined at path $USER_DATA/TableInfo/ to exist in ots. "
1380  "Please verify your table definitions, and then restart ots."
1381  << __E__;
1382  __GEN_COUT_ERR__ << "\n" << ss.str();
1383  __SS_THROW__;
1384  }
1385  return nameToTableMap_[tableName];
1386 } // end getTableByName()
1387 
1388 //==============================================================================
1393 time_t ConfigurationManagerRW::getVersionCreationTime(const std::string& tableName,
1394  TableVersion version)
1395 {
1396  { // check process-wide cache first (only immutable persistent versions are cached)
1397  std::lock_guard<std::mutex> lock(versionCreationTimeCacheMutex_);
1398 
1399  auto tableIt = versionCreationTimeCache_.find(tableName);
1400  if(tableIt != versionCreationTimeCache_.end())
1401  {
1402  auto versionIt = tableIt->second.find(version);
1403  if(versionIt != tableIt->second.end())
1404  return versionIt->second;
1405  }
1406  }
1407 
1408  std::string localAccumulatedErrors;
1409  const auto loadStartTime = std::chrono::steady_clock::now();
1410  time_t creationTime = getVersionedTableByName(tableName,
1411  version,
1412  true /* looseColumnMatching */,
1413  &localAccumulatedErrors,
1414  false /* getRawData */,
1415  false /* touchLastAccessTime */)
1416  ->getView(version)
1417  .getCreationTime();
1418 
1419  double loadSec =
1420  std::chrono::duration<double>(std::chrono::steady_clock::now() - loadStartTime)
1421  .count();
1422  if(loadSec > 1.0)
1423  __GEN_COUT_WARN__ << "Slow creation time lookup: table '" << tableName
1424  << "' version v" << version << " load took " << loadSec << " s"
1425  << __E__;
1426 
1427  if(!version.isTemporaryVersion() && !version.isScratchVersion())
1428  {
1429  std::lock_guard<std::mutex> lock(versionCreationTimeCacheMutex_);
1430  versionCreationTimeCache_[tableName][version] = creationTime;
1431  }
1432 
1433  return creationTime;
1434 } // end getVersionCreationTime()
1435 
1436 //==============================================================================
1443 void ConfigurationManagerRW::preloadVersionCreationTimes(void)
1444 {
1445  const auto preloadStartTime = std::chrono::steady_clock::now();
1446 
1447  // Identify uncached versions grouped by table
1448  std::vector<std::pair<std::string, std::vector<TableVersion>>> groupedWork;
1449  size_t missingCount = 0;
1450  {
1451  std::lock_guard<std::mutex> lock(versionCreationTimeCacheMutex_);
1452 
1453  for(const auto& tableInfoPair : allTableInfo_)
1454  {
1455  auto tableIt = versionCreationTimeCache_.find(tableInfoPair.first);
1456 
1457  std::vector<TableVersion> missingVersions;
1458  for(const auto& version : tableInfoPair.second.versions_)
1459  {
1460  if(version.isTemporaryVersion() || version.isScratchVersion())
1461  continue;
1462  if(tableIt != versionCreationTimeCache_.end() &&
1463  tableIt->second.find(version) != tableIt->second.end())
1464  continue;
1465  missingVersions.push_back(version);
1466  }
1467  if(missingVersions.size())
1468  {
1469  missingCount += missingVersions.size();
1470  groupedWork.emplace_back(tableInfoPair.first, std::move(missingVersions));
1471  }
1472  }
1473  }
1474  if(groupedWork.empty())
1475  return;
1476 
1477  // Round-robin interleave so large tables are spread across all threads
1478  std::vector<std::pair<std::string, TableVersion>> flatWork;
1479  flatWork.reserve(missingCount);
1480  {
1481  size_t maxVersions = 0;
1482  for(const auto& gw : groupedWork)
1483  if(gw.second.size() > maxVersions)
1484  maxVersions = gw.second.size();
1485  for(size_t vi = 0; vi < maxVersions; ++vi)
1486  for(const auto& gw : groupedWork)
1487  if(vi < gw.second.size())
1488  flatWork.emplace_back(gw.first, gw.second[vi]);
1489  }
1490 
1491  __GEN_COUT__ << "preloadVersionCreationTimes() loading " << missingCount
1492  << " version creation times for " << groupedWork.size() << " tables..."
1493  << __E__;
1494 
1495  int numOfThreads = StringMacros::getConcurrencyCount() / 2;
1496  if(numOfThreads > (int)flatWork.size())
1497  numOfThreads = flatWork.size();
1498 
1499  if(numOfThreads < 2)
1500  {
1501  for(const auto& work : flatWork)
1502  {
1503  try
1504  {
1505  getVersionCreationTime(work.first, work.second);
1506  }
1507  catch(...)
1508  {
1509  __GEN_COUT__ << "Failed to get creation time for table '" << work.first
1510  << "' version v" << work.second << ", skipping." << __E__;
1511  }
1512  }
1513  }
1514  else
1515  {
1516  std::atomic<size_t> workIndex(0);
1517  std::vector<std::thread> threads;
1518  auto* iface = theInterface_;
1519 
1520  for(int i = 0; i < numOfThreads; ++i)
1521  threads.emplace_back([iface, &workIndex, &flatWork, this]() {
1522  std::map<std::string, TableBase*> localTables;
1523 
1524  try
1525  {
1526  size_t w;
1527  while((w = workIndex++) < flatWork.size())
1528  {
1529  const auto& tableName = flatWork[w].first;
1530  const auto& version = flatWork[w].second;
1531 
1532  try
1533  {
1534  TableBase*& table = localTables[tableName];
1535  if(!table)
1536  {
1537  std::string localAccumulatedErrors;
1538  table = new TableBase(tableName, &localAccumulatedErrors);
1539  }
1540 
1541  std::string localAccumulatedErrors;
1542  iface->get(table,
1543  tableName,
1544  0 /* groupKey */,
1545  0 /* groupName */,
1546  false /* dontFill */,
1547  version,
1548  true /* resetConfiguration */,
1549  true /* looseColumnMatching */,
1550  false /* rawDataOnly */,
1551  &localAccumulatedErrors,
1552  false /* touchLastAccessTime */);
1553 
1554  time_t creationTime =
1555  table->getView(version).getCreationTime();
1556 
1557  {
1558  std::lock_guard<std::mutex> lock(
1559  versionCreationTimeCacheMutex_);
1560  versionCreationTimeCache_[tableName][version] =
1561  creationTime;
1562  }
1563  }
1564  catch(...)
1565  {
1566  __GEN_COUT__ << "Failed to get creation time for table '"
1567  << tableName << "' version v" << version
1568  << ", skipping." << __E__;
1569  }
1570  }
1571  }
1572  catch(...)
1573  {
1574  __GEN_COUT_ERR__ << "Unexpected error in preload thread." << __E__;
1575  }
1576 
1577  for(auto& pair : localTables)
1578  if(pair.second)
1579  delete pair.second;
1580  });
1581 
1582  for(auto& thread : threads)
1583  thread.join();
1584  }
1585 
1586  __GEN_COUT__ << "preloadVersionCreationTimes() loaded " << missingCount
1587  << " version creation times with " << numOfThreads << " thread(s) in "
1588  << std::chrono::duration<double>(std::chrono::steady_clock::now() -
1589  preloadStartTime)
1590  .count()
1591  << " s" << __E__;
1592 } // end preloadVersionCreationTimes()
1593 
1594 //==============================================================================
1601 time_t ConfigurationManagerRW::getVersionLastAccessTime(const std::string& tableName,
1602  TableVersion version)
1603 {
1604  auto it = nameToTableMap_.find(tableName);
1605  if(it == nameToTableMap_.end() || !it->second->isStored(version))
1606  return 0; // never loaded (or evicted from cache) by this process
1607 
1608  return it->second->getView(version).getLastAccessTime();
1609 } // end getVersionLastAccessTime()
1610 
1611 //==============================================================================
1614 TableVersion ConfigurationManagerRW::saveNewTable(const std::string& tableName,
1615  TableVersion temporaryVersion,
1616  bool makeTemporary) //,
1618 {
1619  TableVersion newVersion(temporaryVersion);
1620 
1621  // set author of version
1622  TableBase* table = getTableByName(tableName);
1623  table->getTemporaryView(temporaryVersion)->setAuthor(username_);
1624  // NOTE: author is assigned to permanent versions when saved to DBI
1625 
1626  if(!makeTemporary) // saveNewVersion makes the new version the active version
1627  newVersion = theInterface_->saveNewVersion(table, temporaryVersion);
1628  else // make the temporary version active
1629  table->setActiveView(newVersion);
1630 
1631  // if there is a problem, try to recover
1632  while(!makeTemporary && !newVersion.isScratchVersion() &&
1633  allTableInfo_[tableName].versions_.find(newVersion) !=
1634  allTableInfo_[tableName].versions_.end())
1635  {
1636  __GEN_COUT_ERR__
1637  << "What happenened!?? ERROR::: new persistent version v" << newVersion
1638  << " already exists!? How is it possible? Retrace your steps and "
1639  "tell an admin."
1640  << __E__;
1641 
1642  // create a new temporary version of the target view
1643  temporaryVersion = table->createTemporaryView(newVersion);
1644 
1645  if(newVersion.isTemporaryVersion())
1646  newVersion = temporaryVersion;
1647  else
1648  newVersion = TableVersion::getNextVersion(newVersion);
1649 
1650  __GEN_COUT_WARN__ << "Attempting to recover and use v" << newVersion << __E__;
1651 
1652  if(!makeTemporary) // saveNewVersion makes the new version the active version
1653  newVersion =
1654  theInterface_->saveNewVersion(table, temporaryVersion, newVersion);
1655  else // make the temporary version active
1656  table->setActiveView(newVersion);
1657  }
1658 
1659  if(newVersion.isInvalid())
1660  {
1661  __SS__ << "Something went wrong saving the new version v" << newVersion
1662  << ". What happened?! (duplicates? database error?)" << __E__;
1663  __GEN_COUT_ERR__ << "\n" << ss.str();
1664  __SS_THROW__;
1665  }
1666 
1667  // update allTableInfo_ with the new version
1668  allTableInfo_[tableName].versions_.insert(newVersion);
1669 
1670  // table->getView().print();
1671  return newVersion;
1672 } // end saveNewTable()
1673 
1674 //==============================================================================
1679 void ConfigurationManagerRW::eraseTemporaryVersion(const std::string& tableName,
1680  TableVersion targetVersion)
1681 {
1682  TableBase* table = getTableByName(tableName);
1683 
1684  table->trimTemporary(targetVersion);
1685 
1686  // if allTableInfo_ is not setup, then done
1687  if(allTableInfo_.find(tableName) == allTableInfo_.end())
1688  return;
1689  // else cleanup table info
1690 
1691  if(targetVersion.isInvalid())
1692  {
1693  // erase all temporary versions!
1694  for(auto it = allTableInfo_[tableName].versions_.begin();
1695  it != allTableInfo_[tableName].versions_.end();
1696  /*no increment*/)
1697  {
1698  if(it->isTemporaryVersion())
1699  {
1700  __GEN_COUT__ << "Removing '" << tableName << "' version info: " << *it
1701  << __E__;
1702  allTableInfo_[tableName].versions_.erase(it++);
1703  }
1704  else
1705  ++it;
1706  }
1707  }
1708  else // erase target version only
1709  {
1710  //__GEN_COUT__ << "Removing '" << tableName << "' version info: " << targetVersion << __E__;
1711  auto it = allTableInfo_[tableName].versions_.find(targetVersion);
1712  if(it == allTableInfo_[tableName].versions_.end())
1713  {
1714  __GEN_COUT__ << "Target '" << tableName << "' version v" << targetVersion
1715  << " was not found in info versions..." << __E__;
1716  return;
1717  }
1718  allTableInfo_[tableName].versions_.erase(
1719  allTableInfo_[tableName].versions_.find(targetVersion));
1720  }
1721 } // end eraseTemporaryVersion()
1722 
1723 //==============================================================================
1728 void ConfigurationManagerRW::clearCachedVersions(const std::string& tableName)
1729 {
1730  TableBase* table = getTableByName(tableName);
1731 
1732  table->trimCache(0);
1733 } // end clearCachedVersions()
1734 
1735 //==============================================================================
1740 void ConfigurationManagerRW::clearAllCachedVersions()
1741 {
1742  for(auto configInfo : allTableInfo_)
1743  configInfo.second.tablePtr_->trimCache(0);
1744 } // end clearAllCachedVersions()
1745 
1746 //==============================================================================
1748 TableVersion ConfigurationManagerRW::copyViewToCurrentColumns(
1749  const std::string& tableName, TableVersion sourceVersion)
1750 {
1751  getTableByName(tableName)->reset();
1752 
1753  // make sure source version is loaded
1754  // need to load with loose column rules!
1755  TableBase* table =
1756  getVersionedTableByName(tableName, TableVersion(sourceVersion), true);
1757 
1758  // copy from source version to a new temporary version
1759  TableVersion newTemporaryVersion =
1760  table->copyView(table->getView(), TableVersion(), username_);
1761 
1762  // update allTableInfo_ with the new version
1763  allTableInfo_[tableName].versions_.insert(newTemporaryVersion);
1764 
1765  return newTemporaryVersion;
1766 } // end copyViewToCurrentColumns()
1767 
1768 //==============================================================================
1772 const GroupInfo& ConfigurationManagerRW::getGroupInfo(
1773  const std::string& groupName, bool attemptToReloadKeys /* = false */)
1774 {
1775  // //NOTE: seems like this filter is taking the long amount of time
1776  // std::set<std::string /*name*/> fullGroupNames =
1777  // theInterface_->getAllTableGroupNames(groupName); //db filter by group name
1778 
1779  // so instead caching ourselves...
1780  auto it = allGroupInfo_.find(groupName);
1781  if(it == allGroupInfo_.end())
1782  {
1783  __SS__ << "Group name '" << groupName
1784  << "' not found in group info! (creating empty info)" << __E__;
1785  __GEN_COUT_WARN__ << ss.str();
1786  //__SS_THROW__;
1787  return allGroupInfo_[groupName];
1788  }
1789 
1790  if(attemptToReloadKeys) //load keys from Interface group cache
1791  {
1792  __GEN_COUT__ << "Reloading keys from special db group cache if it exists..."
1793  << __E__;
1794 
1795  std::set<TableGroupKey> keys;
1796  { //load keys from special db group cache (this avoids pre-cache filling and avoids long db lookup, unless speed table cache missing for this group)
1797  //attempt to use cache first! (potentially way faster .04 s vs 4 s)
1798  bool cacheFailed = false;
1799  try
1800  {
1801  TableBase localGroupMemberCacheLoader(
1802  true /*special table*/
1803  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),,
1804  TableBase::GROUP_CACHE_PREPEND + groupName);
1805  auto versions = theInterface_->getVersions(&localGroupMemberCacheLoader);
1806  for(const auto& version : versions)
1807  keys.emplace(TableGroupKey(version.version()));
1808  }
1809  catch(...)
1810  {
1811  __GEN_COUT__ << "Ignoring cache loading error. Doing full load of keys..."
1812  << __E__;
1813  cacheFailed = true;
1814  }
1815 
1816  if(cacheFailed && 0) //could consider full load if cache failed
1817  keys = theInterface_->getKeys(groupName);
1818 
1819  if(!cacheFailed) //take keys
1820  {
1821  __GEN_COUT__ << "Key from special db group cache were loaded." << __E__;
1822  it->second.keys_ = keys; //update ConfigManager cache!
1823  }
1824  }
1825  }
1826 
1827  return it->second;
1828 } // end getGroupInfo()
1829 
1830 //==============================================================================
1840 TableGroupKey ConfigurationManagerRW::findTableGroup(
1841  const std::string& groupName,
1842  const std::map<std::string, TableVersion>& groupMemberMap,
1843  const std::map<std::string /*name*/, std::string /*alias*/>& memberTableAliases)
1844 {
1845  if(!groupMemberMap.size() || groupName.empty())
1846  {
1847  __SS__ << "Illegal name/members for requested group of name '" << groupName
1848  << "' and member count = " << groupMemberMap.size() << __E__;
1849  __SS_THROW__;
1850  }
1851 
1852  // //NOTE: seems like this filter is taking the long amount of time
1853  // std::set<std::string /*name*/> fullGroupNames =
1854  // theInterface_->getAllTableGroupNames(groupName); //db filter bygroup name
1855  // const GroupInfo& groupInfo = getGroupInfo(groupName); // Note this also seems to take too long because requires a pre-cache load!
1856  std::set<TableGroupKey> keys;
1857  { //so instead load keys from special db group cache (this avoids pre-cache filling and avoids long db lookup, unless speed table cache missing for this group)
1858  //attempt to use cache first! (potentially way faster .04 s vs 4 s)
1859  bool cacheFailed = false;
1860  try
1861  {
1862  TableBase localGroupMemberCacheLoader(
1863  true /*special table*/
1864  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),,
1865  TableBase::GROUP_CACHE_PREPEND + groupName);
1866  auto versions = theInterface_->getVersions(&localGroupMemberCacheLoader);
1867  for(const auto& version : versions)
1868  keys.emplace(TableGroupKey(version.version()));
1869  }
1870  catch(...)
1871  {
1872  __COUT__ << "Ignoring cache loading error. Doing full load of keys..."
1873  << __E__;
1874  cacheFailed = true;
1875  }
1876 
1877  if(cacheFailed) //since cache failed, do full load
1878  keys = theInterface_->getKeys(groupName);
1879  }
1880 
1881  __COUTTV__(StringMacros::setToString(keys));
1882 
1883  const unsigned int MAX_DEPTH_TO_CHECK = 20;
1884  unsigned int keyMinToCheck = 0;
1885 
1886  if(keys.size())
1887  keyMinToCheck = keys.rbegin()->key();
1888  if(keyMinToCheck > MAX_DEPTH_TO_CHECK)
1889  {
1890  keyMinToCheck -= MAX_DEPTH_TO_CHECK;
1891  __GEN_COUT__ << "Checking groups back to key... " << keyMinToCheck << __E__;
1892  }
1893  else
1894  {
1895  keyMinToCheck = 0;
1896  __GEN_COUT__ << "Checking all groups." << __E__;
1897  }
1898 
1899  __GEN_COUTTV__(StringMacros::mapToString(groupMemberMap));
1900 
1901  // have min key to check, now loop through and check groups
1902 
1903  const int numOfThreads = StringMacros::getConcurrencyCount() / 2;
1904  __GEN_COUT__ << " getConcurrencyCount " << StringMacros::getConcurrencyCount()
1905  << " ==> " << numOfThreads << " threads." << __E__;
1906  if(numOfThreads < 2) // no multi-threading
1907  {
1908  std::map<std::string /*name*/, TableVersion /*version*/> compareToMemberMap;
1909  std::map<std::string /*name*/, std::string /*alias*/> compareToMemberTableAliases;
1910  std::map<std::string /*name*/, std::string /*alias*/>*
1911  compareToMemberTableAliasesPtr = nullptr;
1912  if(memberTableAliases.size())
1913  compareToMemberTableAliasesPtr = &compareToMemberTableAliases;
1914 
1915  bool isDifferent;
1916  for(const auto& key : keys)
1917  {
1918  if(key.key() < keyMinToCheck)
1919  continue; // skip keys that are too old
1920 
1921  loadTableGroup(groupName,
1922  key,
1923  false /*doActivate*/,
1924  &compareToMemberMap /*memberMap*/,
1925  0, /*progressBar*/
1926  0, /*accumulatedWarnings*/
1927  0, /*groupComment*/
1928  0, /*groupAuthor*/
1929  0, /*groupCreateTime*/
1930  true /*doNotLoadMember*/,
1931  0 /*groupTypeString*/,
1932  compareToMemberTableAliasesPtr);
1933 
1934  isDifferent = false;
1935  for(auto& memberPair : groupMemberMap)
1936  {
1937  if(memberTableAliases.find(memberPair.first) != memberTableAliases.end())
1938  {
1939  // handle this table as alias, not version
1940  if(compareToMemberTableAliases.find(memberPair.first) ==
1941  compareToMemberTableAliases.end() || // alias is missing
1942  memberTableAliases.at(memberPair.first) !=
1943  compareToMemberTableAliases.at(memberPair.first))
1944  { // then different
1945  isDifferent = true;
1946  break;
1947  }
1948  // FIXED alias matches, but still need to check version
1949  } // else check if compareTo group is using an alias for table
1950  else if(compareToMemberTableAliases.find(memberPair.first) !=
1951  compareToMemberTableAliases.end())
1952  {
1953  // then different
1954  isDifferent = true;
1955  break;
1956  }
1957 
1958  // alias check complete
1959  // handle table version comparison
1960 
1961  if(compareToMemberMap.find(memberPair.first) ==
1962  compareToMemberMap.end() || // name is missing
1963  memberPair.second !=
1964  compareToMemberMap.at(memberPair.first)) // or version mismatch
1965  {
1966  // then different
1967  isDifferent = true;
1968  break;
1969  }
1970  }
1971  if(isDifferent)
1972  continue;
1973 
1974  // check member size for exact match
1975  if(groupMemberMap.size() != compareToMemberMap.size())
1976  continue; // different size, so not same (groupMemberMap is a subset of
1977  // memberPairs)
1978 
1979  __GEN_COUT__ << "Found exact match with key: " << key << __E__;
1980  // else found an exact match!
1981  return key;
1982  }
1983  __GEN_COUT__ << "No match found - this group is new!" << __E__;
1984  // if here, then no match found
1985  return TableGroupKey(); // return invalid key
1986  }
1987  else //multi-threading
1988  {
1989  int threadsLaunched = 0;
1990  int foundThreadIndex = 0;
1991  std::atomic<bool> foundIdentical = false;
1992  ots::TableGroupKey identicalKey;
1993  std::mutex threadMutex;
1994 
1995  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
1996  for(int i = 0; i < numOfThreads; ++i)
1997  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
1998 
1999  for(const auto& key : keys)
2000  {
2001  if(foundIdentical)
2002  break;
2003  if(key.key() < keyMinToCheck)
2004  continue; // skip keys that are too old
2005 
2006  if(threadsLaunched >= numOfThreads)
2007  {
2008  //find availableThreadIndex
2009  foundThreadIndex = -1;
2010  while(foundThreadIndex == -1)
2011  {
2012  if(foundIdentical)
2013  break;
2014 
2015  for(int i = 0; i < numOfThreads; ++i)
2016  if(*(threadDone[i]))
2017  {
2018  foundThreadIndex = i;
2019  break;
2020  }
2021  if(foundThreadIndex == -1)
2022  {
2023  __GEN_COUTT__ << "Waiting for available thread..." << __E__;
2024  usleep(10000);
2025  }
2026  } //end thread search loop
2027  threadsLaunched = numOfThreads - 1;
2028  }
2029  if(foundIdentical)
2030  break;
2031 
2032  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex << __E__;
2033  *(threadDone[foundThreadIndex]) = false;
2034 
2035  std::thread(
2036  [](ConfigurationManagerRW* cfgMgr,
2037  std::string theGroupName,
2038  ots::TableGroupKey groupKeyToCompare,
2039  const std::map<std::string, TableVersion>& groupMemberMap,
2040  const std::map<std::string /*name*/, std::string /*alias*/>&
2041  memberTableAliases,
2042  std::atomic<bool>* theFoundIdentical,
2043  ots::TableGroupKey* theIdenticalKey,
2044  std::mutex* theThreadMutex,
2045  std::shared_ptr<std::atomic<bool>> theThreadDone) {
2046  ConfigurationManagerRW::compareTableGroupThread(cfgMgr,
2047  theGroupName,
2048  groupKeyToCompare,
2049  groupMemberMap,
2050  memberTableAliases,
2051  theFoundIdentical,
2052  theIdenticalKey,
2053  theThreadMutex,
2054  theThreadDone);
2055  },
2056  this,
2057  groupName,
2058  key,
2059  groupMemberMap,
2060  memberTableAliases,
2061  &foundIdentical,
2062  &identicalKey,
2063  &threadMutex,
2064  threadDone[foundThreadIndex])
2065  .detach();
2066 
2067  ++threadsLaunched;
2068  ++foundThreadIndex;
2069  } //end group key check thread loop
2070 
2071  //check for all threads done
2072  do
2073  {
2074  foundThreadIndex = -1;
2075  for(int i = 0; i < numOfThreads; ++i)
2076  if(!*(threadDone[i]))
2077  {
2078  foundThreadIndex = i;
2079  break;
2080  }
2081  if(foundThreadIndex != -1)
2082  {
2083  __GEN_COUTT__ << "Waiting for thread to finish... " << foundThreadIndex
2084  << __E__;
2085  usleep(10000);
2086  }
2087  } while(foundThreadIndex != -1); //end thread done search loop
2088 
2089  if(foundIdentical)
2090  {
2091  __GEN_COUT__ << "Found exact match with key: " << identicalKey << __E__;
2092  return identicalKey;
2093  }
2094 
2095  // if here, then no match found
2096  return TableGroupKey(); // return invalid key
2097  } //end multi-thread handling
2098 } // end findTableGroup()
2099 
2100 //==============================================================================
2105 TableBase* ConfigurationManagerRW::getMetadataTable(
2106  TableVersion fillVersion /* = TableVersion()*/)
2107 {
2108  if(fillVersion.isInvalid())
2109  return &groupMetadataTable_;
2110  //else load specified fill version
2111 
2112  //only lock metadata table since it is shared by all group accesses
2113  std::lock_guard<std::mutex> lock(metaDataTableMutex_);
2114 
2115  // clear table
2116  while(groupMetadataTable_.getView().getNumberOfRows())
2117  groupMetadataTable_.getViewP()->deleteRow(0);
2118 
2119  // retrieve metadata from database
2120  try
2121  {
2122  theInterface_->fill(&groupMetadataTable_, fillVersion);
2123  }
2124  catch(const std::runtime_error& e)
2125  {
2126  __GEN_COUT_WARN__ << "Failed to load " << groupMetadataTable_.getTableName()
2127  << "-v" << fillVersion << ". Metadata error: " << e.what()
2128  << __E__;
2129  }
2130  catch(...)
2131  {
2132  __GEN_COUT_WARN__ << "Failed to load " << groupMetadataTable_.getTableName()
2133  << "-v" << fillVersion << ". Ignoring unknown metadata error. "
2134  << __E__;
2135  }
2136 
2137  // check that there is only 1 row
2138  if(groupMetadataTable_.getView().getNumberOfRows() != 1)
2139  {
2140  groupMetadataTable_.print();
2141  __GEN_COUT_ERR__ << "Ignoring that groupMetadataTable_ v" << fillVersion
2142  << " has wrong "
2143  "number of rows!' Must "
2144  "be 1. Going with anonymous defaults."
2145  << __E__;
2146 
2147  // fix metadata table
2148  while(groupMetadataTable_.getViewP()->getNumberOfRows() > 1)
2149  groupMetadataTable_.getViewP()->deleteRow(0);
2150  if(groupMetadataTable_.getViewP()->getNumberOfRows() == 0)
2151  groupMetadataTable_.getViewP()->addRow();
2152  }
2153 
2154  return &groupMetadataTable_;
2155 } // end getMetadataTable()
2156 
2157 //==============================================================================
2164 TableGroupKey ConfigurationManagerRW::saveNewTableGroup(
2165  const std::string& groupName,
2166  std::map<std::string, TableVersion>& groupMembers,
2167  const std::string& groupComment,
2168  std::map<std::string /*table*/, std::string /*alias*/>* groupAliases)
2169 {
2170  // steps:
2171  // determine new group key
2172  // verify group members
2173  // verify groupNameWithKey
2174  // verify store
2175 
2176  if(groupMembers.size() == 0) // do not allow empty groups
2177  {
2178  __SS__ << "Empty group member list. Can not create a group without members!"
2179  << __E__;
2180  __SS_THROW__;
2181  }
2182 
2183  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2184 
2185  // verify group members
2186  // - use all table info
2187  std::map<std::string, TableInfo> allCfgInfo = getAllTableInfo();
2188  for(auto& memberPair : groupMembers)
2189  {
2190  // check member name
2191  if(allCfgInfo.find(memberPair.first) == allCfgInfo.end())
2192  {
2193  __GEN_COUT_ERR__ << "Group member \"" << memberPair.first
2194  << "\" not found in database!";
2195 
2196  if(groupMetadataTable_.getTableName() == memberPair.first)
2197  {
2198  __GEN_COUT_WARN__
2199  << "Looks like this is the groupMetadataTable_ '"
2200  << TableBase::GROUP_METADATA_TABLE_NAME
2201  << ".' Note that this table is added to the member map when groups "
2202  "are saved."
2203  << "It should not be part of member map when calling this function."
2204  << __E__;
2205  __GEN_COUT__ << "Attempting to recover." << __E__;
2206  groupMembers.erase(groupMembers.find(memberPair.first));
2207  }
2208  else
2209  {
2210  __SS__ << ("Group member not found!") << __E__;
2211  __SS_THROW__;
2212  }
2213  }
2214  // check member version
2215  if(allCfgInfo[memberPair.first].versions_.find(memberPair.second) ==
2216  allCfgInfo[memberPair.first].versions_.end())
2217  {
2218  __SS__ << "Group member \"" << memberPair.first << "\" version \""
2219  << memberPair.second << "\" not found in database!";
2220  __SS_THROW__;
2221  }
2222  } // end verify members
2223 
2224  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2225 
2226  // verify group aliases
2227  if(groupAliases)
2228  {
2229  for(auto& aliasPair : *groupAliases)
2230  {
2231  // check for alias table in member names
2232  if(groupMembers.find(aliasPair.first) == groupMembers.end())
2233  {
2234  __GEN_COUT_ERR__ << "Group member \"" << aliasPair.first
2235  << "\" not found in group member map!";
2236 
2237  __SS__ << ("Alias table not found in member list!") << __E__;
2238  __SS_THROW__;
2239  }
2240  }
2241  } // end verify group aliases
2242 
2243  TableGroupKey newKey =
2244  TableGroupKey::getNextKey(theInterface_->findLatestGroupKey(groupName));
2245  __GEN_COUT__ << "New Key for group: " << groupName << " found as " << newKey << __E__;
2246  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2247 
2248  time_t groupCreationTime = time(0);
2249  // capture group type before adding metadata table!
2250  std::string groupType = getTypeNameOfGroup(groupMembers);
2251  std::map<std::string /*name*/, TableVersion /*version*/> groupMembersWithoutMeta =
2252  groupMembers;
2253 
2254  // verify groupNameWithKey and attempt to store
2255  try
2256  {
2257  // save meta data for group; reuse groupMetadataTable_
2258  std::string groupAliasesString = "";
2259  if(groupAliases)
2260  groupAliasesString = StringMacros::mapToString(
2261  *groupAliases, "," /*primary delimeter*/, ":" /*secondary delimeter*/);
2262  __GEN_COUT__ << "Metadata: " << username_ << " " << groupCreationTime << " "
2263  << groupComment << " " << groupAliasesString << " " << groupType
2264  << __E__;
2265 
2266  // to compensate for unusual errors upstream, make sure the metadata table has one
2267  // row
2268  while(groupMetadataTable_.getViewP()->getNumberOfRows() > 1)
2269  groupMetadataTable_.getViewP()->deleteRow(0);
2270  if(groupMetadataTable_.getViewP()->getNumberOfRows() == 0)
2271  groupMetadataTable_.getViewP()->addRow();
2272 
2273  // columns are uid,comment,author,time
2274  groupMetadataTable_.getViewP()->setValue(
2275  groupAliasesString, 0, ConfigurationManager::METADATA_COL_ALIASES);
2276  groupMetadataTable_.getViewP()->setValue(
2277  groupComment, 0, ConfigurationManager::METADATA_COL_COMMENT);
2278  groupMetadataTable_.getViewP()->setValue(
2279  username_, 0, ConfigurationManager::METADATA_COL_AUTHOR);
2280  groupMetadataTable_.getViewP()->setValue(
2281  groupCreationTime, 0, ConfigurationManager::METADATA_COL_TIMESTAMP);
2282 
2283  if(TTEST(2))
2284  {
2285  std::stringstream ss;
2286  groupMetadataTable_.print(ss);
2287  __COUT_MULTI__(2, ss.str());
2288  }
2289 
2290  // save table, and retry on save collision
2291  {
2292  // set version to first available persistent version
2293  TableVersion newVersion = TableVersion::getNextVersion(
2294  theInterface_->findLatestVersion(&groupMetadataTable_));
2295  groupMetadataTable_.getViewP()->setVersion(newVersion);
2296 
2297  uint16_t retries = 0;
2298  while(1)
2299  {
2300  try
2301  {
2302  theInterface_->saveActiveVersion(&groupMetadataTable_);
2303  }
2304  catch(const std::runtime_error& e)
2305  {
2306  __GEN_COUT__ << "Caught runtime_error exception during table save."
2307  << __E__;
2308  if(std::string(e.what()).find("there was a collision") !=
2309  std::string::npos)
2310  {
2311  __GEN_COUT_WARN__
2312  << "There was a collision saving the new table "
2313  << groupMetadataTable_ << "(" << newVersion
2314  << "), trying incremented table version... retries="
2315  << retries << __E__;
2316  if(++retries > 3) //give up
2317  throw;
2318  newVersion = TableVersion::getNextVersion(
2319  newVersion); //increment table version
2320  groupMetadataTable_.getViewP()->setVersion(newVersion);
2321  __GEN_COUT__ << "New version for table: " << groupMetadataTable_
2322  << " found as " << newVersion << __E__;
2323  continue;
2324  }
2325  else
2326  throw;
2327  }
2328 
2329  __GEN_COUT__ << "Created table: " << groupMetadataTable_ << "-v"
2330  << newVersion << __E__;
2331  break;
2332  } //end collission retry loop
2333  }
2334 
2335  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2336  << __E__;
2337 
2338  // force groupMetadataTable_ to be a member for the group
2339  groupMembers[groupMetadataTable_.getTableName()] =
2340  groupMetadataTable_.getViewVersion();
2341 
2342  // save group, and retry on save collision
2343  {
2344  uint16_t retries = 0;
2345  while(1)
2346  {
2347  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2348  << __E__;
2349 
2350  try
2351  {
2352  theInterface_->saveTableGroup(
2353  groupMembers,
2354  TableGroupKey::getFullGroupString(groupName, newKey));
2355  }
2356  catch(const std::runtime_error& e)
2357  {
2358  __GEN_COUT__ << "Caught runtime_error exception during group save."
2359  << __E__;
2360  if(std::string(e.what()).find("there was a collision") !=
2361  std::string::npos)
2362  {
2363  __GEN_COUT_WARN__
2364  << "There was a collision saving the new group " << groupName
2365  << "(" << newKey
2366  << "), trying incremented group key... retries=" << retries
2367  << __E__;
2368  if(++retries > 3) //give up
2369  throw;
2370  newKey = TableGroupKey::getNextKey(newKey); //increment group key
2371  __GEN_COUT__ << "New Key for group: " << groupName << " found as "
2372  << newKey << __E__;
2373  continue;
2374  }
2375  else
2376  throw;
2377  }
2378 
2379  __GEN_COUT__ << "Created table group: " << groupName << "(" << newKey
2380  << ")" << __E__;
2381  break;
2382  } //end collission retry loop
2383  }
2384 
2385  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2386  << __E__;
2387  }
2388  catch(std::runtime_error& e)
2389  {
2390  __GEN_COUT_ERR__ << "Failed to create table group: " << groupName << "(" << newKey
2391  << ")" << __E__;
2392  __GEN_COUT_ERR__ << "\n\n" << e.what() << __E__;
2393  throw;
2394  }
2395  catch(...)
2396  {
2397  __GEN_COUT_ERR__ << "Failed to create table group: " << groupName << ":" << newKey
2398  << __E__;
2399  throw;
2400  }
2401 
2402  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2403 
2404  // store cache of recent groups
2405  allGroupInfo_[groupName].keys_.emplace(newKey);
2406  //update latest group info with this group's info
2407  allGroupInfo_.at(groupName).latestKey_ = newKey;
2408  allGroupInfo_.at(groupName).latestKeyGroupAuthor_ = username_;
2409  allGroupInfo_.at(groupName).latestKeyGroupComment_ = groupComment;
2410  allGroupInfo_.at(groupName).latestKeyGroupCreationTime_ = groupCreationTime;
2411  allGroupInfo_.at(groupName).latestKeyGroupTypeString_ = groupType;
2412  allGroupInfo_.at(groupName).latestKeyMemberMap_ = groupMembersWithoutMeta;
2413 
2414  __GEN_COUT__ << "Saved " << groupName << "(" << newKey << ") of type " << groupType
2415  << __E__;
2416 
2417  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2418 
2419  // at this point succeeded!
2420  return newKey;
2421 } // end saveNewTableGroup()
2422 
2423 //==============================================================================
2427 TableVersion ConfigurationManagerRW::saveNewBackbone(TableVersion temporaryVersion)
2428 {
2429  __GEN_COUT_INFO__ << "Creating new backbone from temporary version "
2430  << temporaryVersion << __E__;
2431 
2432  // find common available temporary version among backbone members
2433  TableVersion newVersion(TableVersion::DEFAULT);
2434  TableVersion retNewVersion;
2435  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
2436  for(auto& name : backboneMemberNames)
2437  {
2438  retNewVersion = ConfigurationManager::getTableByName(name)->getNextVersion();
2439  __GEN_COUT__ << "New version for backbone member (" << name
2440  << "): " << retNewVersion << __E__;
2441  if(retNewVersion > newVersion)
2442  newVersion = retNewVersion;
2443  }
2444 
2445  __GEN_COUT__ << "Common new backbone version found as " << newVersion << __E__;
2446 
2447  // create new views from source temporary version
2448  for(auto& name : backboneMemberNames)
2449  {
2450  // saveNewVersion makes the new version the active version
2451  retNewVersion = getConfigurationInterface()->saveNewVersion(
2452  getTableByName(name), temporaryVersion, newVersion);
2453  if(retNewVersion != newVersion)
2454  {
2455  __SS__ << "Failure! New view requested was " << newVersion
2456  << ". Mismatched new view created: " << retNewVersion << __E__;
2457  __GEN_COUT_ERR__ << ss.str();
2458  __SS_THROW__;
2459  }
2460  }
2461 
2462  return newVersion;
2463 } // end saveNewBackbone()
2464 
2465 //==============================================================================
2470 TableVersion ConfigurationManagerRW::saveModifiedVersion(
2471  const std::string& tableName,
2472  TableVersion originalVersion,
2473  bool makeTemporary,
2474  TableBase* table,
2475  TableVersion temporaryModifiedVersion,
2476  bool ignoreDuplicates /*= false*/,
2477  bool lookForEquivalent /*= false*/,
2478  bool* foundEquivalent /*= nullptr*/)
2479 {
2480  bool needToEraseTemporarySource =
2481  (originalVersion.isTemporaryVersion() && !makeTemporary);
2482 
2483  if(foundEquivalent)
2484  *foundEquivalent = false; // initialize
2485 
2486  // check for duplicate tables already in cache, plus highest version numbers not in cache
2487  if(!ignoreDuplicates)
2488  {
2489  __GEN_COUT__ << "Checking for duplicate '" << tableName << "' tables..." << __E__;
2490 
2491  TableVersion duplicateVersion;
2492 
2493  {
2494  //"DEEP" checking
2495  // load into cache 'recent' versions for this table
2496  // 'recent' := those already in cache, plus highest version numbers not in cache
2497  const std::map<std::string, TableInfo>& allTableInfo =
2498  getAllTableInfo(); // do not refresh
2499 
2500  auto versionReverseIterator =
2501  allTableInfo.at(tableName).versions_.rbegin(); // get reverse iterator
2502  __GEN_COUT__ << "Filling up '" << tableName << "' cache from "
2503  << table->getNumberOfStoredViews() << " to max count of "
2504  << table->MAX_VIEWS_IN_CACHE << __E__;
2505  for(; table->getNumberOfStoredViews() < table->MAX_VIEWS_IN_CACHE &&
2506  versionReverseIterator != allTableInfo.at(tableName).versions_.rend();
2507  ++versionReverseIterator)
2508  {
2509  __GEN_COUTT__ << "'" << tableName << "' versions in reverse order "
2510  << *versionReverseIterator << __E__;
2511  try
2512  {
2513  getVersionedTableByName(tableName,
2514  *versionReverseIterator); // load to cache
2515  }
2516  catch(const std::runtime_error& e)
2517  {
2518  // ignore error
2519  __COUTT__ << "'" << tableName
2520  << "' version failed to load: " << *versionReverseIterator
2521  << __E__;
2522  }
2523  }
2524  }
2525 
2526  __GEN_COUT__ << "Checking '" << tableName << "' for duplicate..." << __E__;
2527 
2528  duplicateVersion = table->checkForDuplicate(
2529  temporaryModifiedVersion,
2530  (!originalVersion.isTemporaryVersion() && !makeTemporary)
2531  ? TableVersion()
2532  : // if from persistent to persistent, then include original version in search
2533  originalVersion);
2534 
2535  if(lookForEquivalent && !duplicateVersion.isInvalid())
2536  {
2537  // found an equivalent!
2538  __GEN_COUT__ << "Equivalent '" << tableName << "' table found in version v"
2539  << duplicateVersion << __E__;
2540 
2541  // if duplicate version was temporary, do not use
2542  if(duplicateVersion.isTemporaryVersion() && !makeTemporary)
2543  {
2544  __GEN_COUT__ << "Need persistent. Duplicate '" << tableName
2545  << "' version was temporary. "
2546  "Abandoning duplicate."
2547  << __E__;
2548  duplicateVersion = TableVersion(); // set invalid
2549  }
2550  else
2551  {
2552  // erase and return equivalent version
2553 
2554  // erase modified equivalent version
2555  eraseTemporaryVersion(tableName, temporaryModifiedVersion);
2556 
2557  // erase original if needed
2558  if(needToEraseTemporarySource)
2559  eraseTemporaryVersion(tableName, originalVersion);
2560 
2561  if(foundEquivalent)
2562  *foundEquivalent = true;
2563 
2564  __GEN_COUT__ << "\t\t Equivalent '" << tableName
2565  << "' assigned version: " << duplicateVersion << __E__;
2566 
2567  return duplicateVersion;
2568  }
2569  }
2570 
2571  if(!duplicateVersion.isInvalid())
2572  {
2573  __SS__ << "This version of table '" << tableName
2574  << "' is identical to another version currently cached v"
2575  << duplicateVersion << ". No reason to save a duplicate." << __E__;
2576  __GEN_COUT_ERR__ << "\n" << ss.str();
2577 
2578  // delete temporaryModifiedVersion
2579  table->eraseView(temporaryModifiedVersion);
2580  __SS_THROW__;
2581  }
2582 
2583  __GEN_COUT__ << "Check for duplicate '" << tableName << "' tables complete."
2584  << __E__;
2585  }
2586 
2587  if(makeTemporary)
2588  __GEN_COUT__ << "\t\t**************************** Save as temporary '"
2589  << tableName << "' table version" << __E__;
2590  else
2591  __GEN_COUT__ << "\t\t**************************** Save as new '" << tableName
2592  << "' table version" << __E__;
2593 
2594  TableVersion newAssignedVersion =
2595  saveNewTable(tableName, temporaryModifiedVersion, makeTemporary);
2596 
2597  __GEN_COUTTV__(table->getView().getComment());
2598 
2599  if(needToEraseTemporarySource)
2600  eraseTemporaryVersion(tableName, originalVersion);
2601 
2602  __GEN_COUT__ << "\t\t '" << tableName
2603  << "' new assigned version: " << newAssignedVersion << __E__;
2604  return newAssignedVersion;
2605 } // end saveModifiedVersion()
2606 
2607 //==============================================================================
2608 GroupEditStruct::GroupEditStruct(const ConfigurationManager::GroupType& groupType,
2609  ConfigurationManagerRW* cfgMgr)
2610  : groupType_(groupType)
2611  , originalGroupName_(cfgMgr->getActiveGroupName(groupType))
2612  , originalGroupKey_(cfgMgr->getActiveGroupKey(groupType))
2613  , cfgMgr_(cfgMgr)
2614  , mfSubject_(cfgMgr->getUsername())
2615 {
2616  if(originalGroupName_ == "" || originalGroupKey_.isInvalid())
2617  {
2618  __SS__ << "Error! No active group found for type '"
2619  << ConfigurationManager::convertGroupTypeToName(groupType)
2620  << ".' There must be an active group to edit the group." << __E__ << __E__
2621  << StringMacros::stackTrace() << __E__;
2622  __SS_THROW__;
2623  }
2624 
2625  __GEN_COUT__ << "Extracting Group-Edit Struct for type "
2626  << ConfigurationManager::convertGroupTypeToName(groupType) << __E__;
2627 
2628  std::map<std::string, TableVersion> activeTables = cfgMgr->getActiveVersions();
2629 
2630  const std::set<std::string>& memberNames =
2631  groupType == ConfigurationManager::GroupType::CONTEXT_TYPE
2632  ? cfgMgr->getActiveContextMemberNames()
2633  : (groupType == ConfigurationManager::GroupType::BACKBONE_TYPE
2634  ? ConfigurationManager::getBackboneMemberNames()
2635  : (groupType == ConfigurationManager::GroupType::ITERATE_TYPE
2636  ? ConfigurationManager::getIterateMemberNames()
2637  : cfgMgr->getConfigurationMemberNames()));
2638 
2639  for(auto& memberName : memberNames)
2640  try
2641  {
2642  groupMembers_.emplace(
2643  std::make_pair(memberName, activeTables.at(memberName)));
2644  groupTables_.emplace(std::make_pair(
2645  memberName,
2646  TableEditStruct(memberName, cfgMgr))); // Table ready for editing!
2647  }
2648  catch(...)
2649  {
2650  __GEN_COUTV__(StringMacros::mapToString(activeTables));
2651  __SS__ << "Error! Could not find group member table '" << memberName
2652  << "' for group type '"
2653  << ConfigurationManager::convertGroupTypeToName(groupType)
2654  << ".' All group members must be present to create the group editing "
2655  "structure."
2656  << __E__ << __E__ << StringMacros::stackTrace() << __E__;
2657  __SS_THROW__;
2658  }
2659 
2660 } // end GroupEditStruct constructor()
2661 
2662 //==============================================================================
2663 GroupEditStruct::~GroupEditStruct()
2664 {
2665  __GEN_COUT__ << "GroupEditStruct from editing '" << originalGroupName_ << "("
2666  << originalGroupKey_ << ")' Destructing..." << __E__;
2667  dropChanges();
2668  __GEN_COUT__ << "GroupEditStruct from editing '" << originalGroupName_ << "("
2669  << originalGroupKey_ << ")' Desctructed." << __E__;
2670 } // end GroupEditStruct destructor()
2671 
2672 //==============================================================================
2674 TableEditStruct& GroupEditStruct::getTableEditStruct(const std::string& tableName,
2675  bool markModified /*= false*/)
2676 {
2677  auto it = groupTables_.find(tableName);
2678  if(it == groupTables_.end())
2679  {
2680  if(groupType_ == ConfigurationManager::GroupType::CONFIGURATION_TYPE &&
2681  markModified)
2682  {
2683  __GEN_COUT__ << "Table '" << tableName
2684  << "' not found in configuration table members from editing '"
2685  << originalGroupName_ << "(" << originalGroupKey_ << ")..."
2686  << " Attempting to add it!" << __E__;
2687 
2688  // emplace returns pair<object,bool wasAdded>
2689  auto newIt = groupTables_.emplace(std::make_pair(
2690  tableName,
2691  TableEditStruct(tableName, cfgMgr_))); // Table ready for editing!
2692  if(newIt.second)
2693  {
2694  newIt.first->second.modified_ =
2695  markModified; // could indicate 'dirty' immediately in user code, which will cause a save of table
2696  groupMembers_.emplace(
2697  std::make_pair(tableName, newIt.first->second.temporaryVersion_));
2698  return newIt.first->second;
2699  }
2700  __GEN_COUT_ERR__ << "Failed to emplace new table..." << __E__;
2701  }
2702 
2703  __SS__ << "Table '" << tableName << "' not found in table members from editing '"
2704  << originalGroupName_ << "(" << originalGroupKey_ << ")!'" << __E__;
2705  __SS_THROW__;
2706  }
2707  it->second.modified_ =
2708  markModified; // could indicate 'dirty' immediately in user code, which will cause a save of table
2709  return it->second;
2710 } // end getTableEditStruct()
2711 
2712 //==============================================================================
2713 void GroupEditStruct::dropChanges()
2714 {
2715  __GEN_COUT__ << "Dropping unsaved changes from editing '" << originalGroupName_ << "("
2716  << originalGroupKey_ << ")'..." << __E__;
2717 
2718  ConfigurationManagerRW* cfgMgr = cfgMgr_;
2719 
2720  // drop all temporary versions
2721  for(auto& groupTable : groupTables_)
2722  if(groupTable.second
2723  .createdTemporaryVersion_) // if temporary version created here
2724  {
2725  // erase with proper version management
2726  cfgMgr->eraseTemporaryVersion(groupTable.second.tableName_,
2727  groupTable.second.temporaryVersion_);
2728  groupTable.second.createdTemporaryVersion_ = false;
2729  groupTable.second.modified_ = false;
2730  }
2731 
2732  __GEN_COUT__ << "Unsaved changes dropped from editing '" << originalGroupName_ << "("
2733  << originalGroupKey_ << ").'" << __E__;
2734 } // end GroupEditStruct::dropChanges()
2735 
2736 //==============================================================================
2737 void GroupEditStruct::saveChanges(const std::string& groupNameToSave,
2738  TableGroupKey& newGroupKey,
2739  bool* foundEquivalentGroupKey /*= nullptr*/,
2740  bool activateNewGroup /*= false*/,
2741  bool updateGroupAliases /*= false*/,
2742  bool updateTableAliases /*= false*/,
2743  TableGroupKey* newBackboneKey /*= nullptr*/,
2744  bool* foundEquivalentBackboneKey /*= nullptr*/,
2745  std::string* accumulatedWarnings /*= nullptr*/)
2746 {
2747  __GEN_COUT__ << "Saving changes..." << __E__;
2748 
2749  newGroupKey = TableGroupKey(); // invalidate reference parameter
2750  if(newBackboneKey)
2751  *newBackboneKey = TableGroupKey(); // invalidate reference parameter
2752  if(foundEquivalentBackboneKey)
2753  *foundEquivalentBackboneKey = false; // clear to start
2754  ConfigurationManagerRW* cfgMgr = cfgMgr_;
2755 
2756  // save all temporary modified versions
2757  bool anyTableNew = false;
2758  for(auto& groupTable : groupTables_)
2759  {
2760  if(!groupTable.second.modified_)
2761  continue; // skip if not modified
2762 
2763  __GEN_COUT__ << "Original version is " << groupTable.second.tableName_ << "-v"
2764  << groupTable.second.originalVersion_ << __E__;
2765 
2766  groupMembers_.at(groupTable.first) = cfgMgr->saveModifiedVersion(
2767  groupTable.second.tableName_,
2768  groupTable.second.originalVersion_,
2769  true /*make temporary*/,
2770  groupTable.second.table_,
2771  groupTable.second.temporaryVersion_,
2772  true /*ignoreDuplicates*/); // make temporary version to save persistent version properly
2773 
2774  __GEN_COUT__ << "Temporary target version is " << groupTable.second.tableName_
2775  << "-v" << groupMembers_.at(groupTable.first) << "-v"
2776  << groupTable.second.temporaryVersion_ << __E__;
2777 
2778  groupMembers_.at(groupTable.first) = cfgMgr->saveModifiedVersion(
2779  groupTable.second.tableName_,
2780  groupTable.second.originalVersion_,
2781  false /*make temporary*/,
2782  groupTable.second.table_,
2783  groupTable.second.temporaryVersion_,
2784  false /*ignoreDuplicates*/,
2785  true /*lookForEquivalent*/); // save persistent version properly
2786 
2787  if(groupTable.second.originalVersion_ != groupMembers_.at(groupTable.first))
2788  {
2789  anyTableNew = true;
2790  __GEN_COUT__ << "Final NEW target version is " << groupTable.second.tableName_
2791  << "-v" << groupMembers_.at(groupTable.first) << __E__;
2792  }
2793  else
2794  __GEN_COUT__ << "Final target version is " << groupTable.second.tableName_
2795  << "-v" << groupMembers_.at(groupTable.first) << __E__;
2796 
2797  groupTable.second.modified_ = false; // clear modified flag
2798  groupTable.second.createdTemporaryVersion_ = false; // modified version is gone
2799  } // loop through table edit structs
2800 
2801  for(auto& table : groupMembers_)
2802  {
2803  __GEN_COUT__ << table.first << " v" << table.second << __E__;
2804  }
2805 
2806  if(!anyTableNew) //then could be duplicate group
2807  {
2808  __GEN_COUT__ << "Checking for duplicate groups..." << __E__;
2809  newGroupKey = cfgMgr->findTableGroup(groupNameToSave, groupMembers_);
2810  }
2811  else
2812  __GEN_COUT__ << "New table found, so no need to check duplicate groups." << __E__;
2813 
2814  if(!newGroupKey.isInvalid())
2815  {
2816  __GEN_COUT__ << "Found equivalent group key (" << newGroupKey << ") for "
2817  << groupNameToSave << "." << __E__;
2818  if(foundEquivalentGroupKey)
2819  *foundEquivalentGroupKey = true;
2820  }
2821  else
2822  {
2823  newGroupKey = cfgMgr->saveNewTableGroup(groupNameToSave, groupMembers_);
2824  __GEN_COUT__ << "Saved new Context group key (" << newGroupKey << ") for "
2825  << groupNameToSave << "." << __E__;
2826  }
2827 
2828  bool groupAliasChange = false;
2829  bool tableAliasChange = false;
2830 
2831  if(groupType_ !=
2832  ConfigurationManager::GroupType::
2833  BACKBONE_TYPE) //if not backbone group save, consider changing aliases (not if it is backbone group, tables not necessarily active yet, which causes error in GroupEditStruct backboneGroupEdit)
2834  {
2835  GroupEditStruct backboneGroupEdit(ConfigurationManager::GroupType::BACKBONE_TYPE,
2836  cfgMgr);
2837 
2838  if(groupType_ != ConfigurationManager::GroupType::BACKBONE_TYPE &&
2839  updateGroupAliases)
2840  {
2841  // check group aliases ... a la
2842  // ConfigurationGUISupervisor::handleSetGroupAliasInBackboneXML
2843 
2844  TableEditStruct& groupAliasTable = backboneGroupEdit.getTableEditStruct(
2845  ConfigurationManager::GROUP_ALIASES_TABLE_NAME, true /*markModified*/);
2846  TableView* tableView = groupAliasTable.tableView_;
2847 
2848  // unsigned int col;
2849  unsigned int row = 0;
2850 
2851  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
2852  cfgMgr->getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
2853  .getChildren();
2854  std::string groupName, groupKey;
2855  for(auto& aliasNodePair : aliasNodePairs)
2856  {
2857  groupName = aliasNodePair.second.getNode("GroupName").getValueAsString();
2858  groupKey = aliasNodePair.second.getNode("GroupKey").getValueAsString();
2859 
2860  __GEN_COUT__ << "Group Alias: " << aliasNodePair.first << " => "
2861  << groupName << "(" << groupKey << "); row=" << row << __E__;
2862 
2863  if(groupName == originalGroupName_ &&
2864  TableGroupKey(groupKey) == originalGroupKey_)
2865  {
2866  __GEN_COUT__ << "Found alias! Changing group key from ("
2867  << originalGroupKey_ << ") to (" << newGroupKey << ")"
2868  << __E__;
2869 
2870  groupAliasChange = true;
2871 
2872  tableView->setValueAsString(
2873  newGroupKey.toString(), row, tableView->findCol("GroupKey"));
2874  }
2875 
2876  ++row;
2877  }
2878 
2879  if(groupAliasChange)
2880  {
2881  std::stringstream ss;
2882  tableView->print(ss);
2883  __GEN_COUT__ << ss.str();
2884  }
2885  } // end updateGroupAliases handling
2886 
2887  if(groupType_ != ConfigurationManager::GroupType::BACKBONE_TYPE &&
2888  updateTableAliases)
2889  {
2890  // update all table version aliases
2891  TableView* tableView =
2892  backboneGroupEdit
2893  .getTableEditStruct(ConfigurationManager::VERSION_ALIASES_TABLE_NAME,
2894  true /*markModified*/)
2895  .tableView_;
2896 
2897  for(auto& groupTable : groupTables_)
2898  {
2899  if(groupTable.second.originalVersion_ ==
2900  groupMembers_.at(groupTable.second.tableName_))
2901  continue; // skip if no change
2902 
2903  __GEN_COUT__ << "Checking alias... original version is "
2904  << groupTable.second.tableName_ << "-v"
2905  << groupTable.second.originalVersion_
2906  << " and new version is v"
2907  << groupMembers_.at(groupTable.second.tableName_) << __E__;
2908 
2909  // unsigned int col;
2910  unsigned int row = 0;
2911 
2912  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
2913  cfgMgr->getNode(ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
2914  .getChildren();
2915  std::string tableName, tableVersion;
2916  for(auto& aliasNodePair : aliasNodePairs)
2917  {
2918  tableName =
2919  aliasNodePair.second.getNode("TableName").getValueAsString();
2920  tableVersion =
2921  aliasNodePair.second.getNode("Version").getValueAsString();
2922 
2923  __GEN_COUT__ << "Table Alias: " << aliasNodePair.first << " => "
2924  << tableName << "-v" << tableVersion << "" << __E__;
2925 
2926  if(tableName == groupTable.second.tableName_ &&
2927  TableVersion(tableVersion) == groupTable.second.originalVersion_)
2928  {
2929  __GEN_COUT__ << "Found alias! Changing icon table version alias."
2930  << __E__;
2931 
2932  tableAliasChange = true;
2933 
2934  tableView->setValueAsString(
2935  groupMembers_.at(groupTable.second.tableName_).toString(),
2936  row,
2937  tableView->findCol("Version"));
2938  }
2939 
2940  ++row;
2941  }
2942  }
2943 
2944  if(tableAliasChange)
2945  {
2946  std::stringstream ss;
2947  tableView->print(ss);
2948  __GEN_COUT__ << ss.str();
2949  }
2950  } // end updateTableAliases handling
2951 
2952  TableGroupKey localNewBackboneKey;
2953  // if backbone modified, save group and activate it
2954  if(groupAliasChange || tableAliasChange)
2955  {
2956  for(auto& table : backboneGroupEdit.groupMembers_)
2957  {
2958  __GEN_COUT__ << table.first << " v" << table.second << __E__;
2959  }
2960  backboneGroupEdit.saveChanges(
2961  backboneGroupEdit.originalGroupName_,
2962  localNewBackboneKey,
2963  foundEquivalentBackboneKey ? foundEquivalentBackboneKey : nullptr);
2964 
2965  if(newBackboneKey)
2966  *newBackboneKey = localNewBackboneKey;
2967  }
2968 
2969  // acquire all active groups and ignore errors, so that activateTableGroup does not
2970  // erase other active groups
2971  {
2972  __GEN_COUT__
2973  << "Restoring active table groups, before activating new groups..."
2974  << __E__;
2975 
2976  std::string localAccumulatedWarnings;
2977  cfgMgr->restoreActiveTableGroups(
2978  false /*throwErrors*/,
2979  "" /*pathToActiveGroupsFile*/,
2980  ConfigurationManager::LoadGroupType::
2981  ALL_TYPES /*onlyLoadIfBackboneOrContext*/,
2982  &localAccumulatedWarnings);
2983  }
2984 
2985  // activate new groups
2986  if(!localNewBackboneKey.isInvalid())
2987  cfgMgr->activateTableGroup(
2988  backboneGroupEdit.originalGroupName_,
2989  localNewBackboneKey,
2990  accumulatedWarnings ? accumulatedWarnings : nullptr);
2991 
2992  } //end non-backbone save type handling
2993  else //is backbone save type
2994  {
2995  // acquire all active groups and ignore errors, so that activateTableGroup does not
2996  // erase other active groups
2997  {
2998  __GEN_COUT__
2999  << "Restoring active table groups, before activating new groups..."
3000  << __E__;
3001 
3002  std::string localAccumulatedWarnings;
3003  cfgMgr->restoreActiveTableGroups(
3004  false /*throwErrors*/,
3005  "" /*pathToActiveGroupsFile*/,
3006  ConfigurationManager::LoadGroupType::
3007  ALL_TYPES /*onlyLoadIfBackboneOrContext*/,
3008  &localAccumulatedWarnings);
3009  }
3010  } //end backbone save type handling
3011 
3012  if(activateNewGroup)
3013  cfgMgr->activateTableGroup(groupNameToSave,
3014  newGroupKey,
3015  accumulatedWarnings ? accumulatedWarnings : nullptr);
3016 
3017  __GEN_COUT__ << "Changes saved." << __E__;
3018 } // end GroupEditStruct::saveChanges()
3019 
3020 //==============================================================================
3022 void ConfigurationManagerRW::testXDAQContext()
3023 {
3024  if(1)
3025  return; //if 0 to debug
3026  __GEN_COUTV__(runTimeSeconds());
3027 
3028  std::string accumulatedWarningsStr;
3029  std::string* accumulatedWarnings = &accumulatedWarningsStr;
3030 
3031  // get Group Info too!
3032  try
3033  {
3034  //test lookup of which groups a table is in
3035  {
3036  std::string documentNameToLoad = "XDAQApplicationTable";
3037  TableVersion documentVersionToLoad(
3038  (int)134); //1 is easy, 134 is hard on daq13 mongodb
3039 
3040  std::set<std::string> groupsContainingTable =
3041  theInterface_->findGroupsWithTable(documentNameToLoad,
3042  documentVersionToLoad);
3043  __GEN_COUT__ << "Groups containing " << documentNameToLoad << "-v"
3044  << documentVersionToLoad
3045  << " count: " << groupsContainingTable.size() << __E__;
3046  for(const auto& group : groupsContainingTable)
3047  {
3048  __GEN_COUT__ << "\t" << group << __E__;
3049  }
3050  }
3051 
3052  std::string debugGroupName = "Mu2eHWEmulatorContext";
3053 
3054  //final solution demo of getting latest group key:
3055  {
3056  TableGroupKey latestGroupKey =
3057  theInterface_->findLatestGroupKey(debugGroupName);
3058  __GEN_COUTV__(latestGroupKey);
3059 
3060  __GEN_COUTV__(runTimeSeconds());
3061  }
3062 
3063  //steps to do time comparison for getting last group key and table key:
3064 
3065  // build allGroupInfo_ for the ConfigurationManagerRW
3066 
3067  std::set<std::string /*name*/> tableGroups =
3068  theInterface_->getAllTableGroupNames();
3069  __GEN_COUT__ << "Number of Groups: " << tableGroups.size() << __E__;
3070 
3071  __GEN_COUTV__(runTimeSeconds());
3072  // return;
3073 
3074  TableGroupKey key;
3075  std::string name;
3076  for(const auto& fullName : tableGroups)
3077  {
3078  TableGroupKey::getGroupNameAndKey(fullName, name, key);
3079  allGroupInfo_[name].keys_.emplace(key);
3080 
3081  if(name == debugGroupName)
3082  {
3083  __GEN_COUTV__(key);
3084  }
3085  }
3086  __GEN_COUTV__(runTimeSeconds());
3087 
3088  std::set<std::string /*name*/> tableNames = theInterface_->getAllTableNames();
3089  __GEN_COUT__ << "Number of Tables: " << tableNames.size() << __E__;
3090 
3091  __GEN_COUTV__(runTimeSeconds());
3092 
3093  for(const auto& fullName : tableNames)
3094  {
3095  if(fullName.find(debugGroupName) != std::string::npos)
3096  {
3097  __GEN_COUTV__(fullName);
3098  }
3099  }
3100  __GEN_COUTV__(runTimeSeconds());
3101 
3102  TableGroupKey latestGroupKey = theInterface_->findLatestGroupKey(debugGroupName);
3103  __GEN_COUTV__(latestGroupKey);
3104 
3105  __GEN_COUTV__(runTimeSeconds());
3106 
3107  TableBase localGroupMemberCacheSaver(
3108  true /*special table*/
3109  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),
3110  TableBase::GROUP_CACHE_PREPEND + debugGroupName);
3111  TableVersion lastestGroupCacheKey =
3112  theInterface_->findLatestVersion(&localGroupMemberCacheSaver);
3113  __GEN_COUTV__(lastestGroupCacheKey);
3114 
3115  __GEN_COUTV__(runTimeSeconds());
3116 
3117  //test a group save that already exists
3118  try
3119  {
3120  TableGroupKey groupKey(int(0));
3121  __GEN_COUT__ << "Testing group save of pre-existing " << debugGroupName << "("
3122  << groupKey << ")" << __E__;
3123  std::map<std::string, TableVersion> groupMembers;
3124  groupMembers["DesktopIconTable"] = TableVersion(123);
3125  theInterface_->saveTableGroup(
3126  groupMembers,
3127  TableGroupKey::getFullGroupString(debugGroupName, groupKey));
3128  }
3129  catch(...)
3130  {
3131  __GEN_COUT__ << "Exception during group save." << __E__;
3132  }
3133  __GEN_COUTV__(runTimeSeconds());
3134 
3135  //test a group save that does not already exists
3136  try
3137  {
3138  std::string debugGroupName = "testGroupSave";
3139  TableGroupKey groupKey(int(2));
3140  __GEN_COUT__ << "Testing group save of non-existing " << debugGroupName << "("
3141  << groupKey << ")" << __E__;
3142  std::map<std::string, TableVersion> groupMembers;
3143  groupMembers["DesktopIconTable"] = TableVersion(123);
3144  groupMembers["MessageFacilityTable"] = TableVersion(7);
3145  theInterface_->saveTableGroup(
3146  groupMembers,
3147  TableGroupKey::getFullGroupString(debugGroupName, groupKey));
3148  }
3149  catch(...)
3150  {
3151  __GEN_COUT__ << "Exception during new group save." << __E__;
3152  }
3153  __GEN_COUTV__(runTimeSeconds());
3154 
3155  //test a table save that already exists
3156  {
3157  std::string documentNameToLoad = "XDAQApplicationTable";
3158  TableVersion documentVersionToLoad(134);
3159 
3160  __GEN_COUT__ << "Testing table save of pre-existing " << documentNameToLoad
3161  << __E__;
3162 
3163  { //load to prove it exists
3164  TableBase localDocLoader(
3165  documentNameToLoad); //can not use special table when filling
3166  localDocLoader.changeVersionAndActivateView(
3167  localDocLoader.createTemporaryView(), documentVersionToLoad);
3168  theInterface_->fill(&localDocLoader, documentVersionToLoad);
3169  __SS__;
3170  localDocLoader.print(ss);
3171  __GEN_COUTV__(ss.str());
3172  }
3173  __GEN_COUTV__(runTimeSeconds());
3174 
3175  try
3176  { //attempt to save over existing version
3177  std::string documentNameToSave = documentNameToLoad;
3178  TableBase
3179  localDocSaver( //true /*special table*/, //special table only allows 1 view in cache and does not load schema (which is perfect for this check),
3180  documentNameToSave); //can not use special table when filling
3181  localDocSaver.changeVersionAndActivateView(
3182  localDocSaver.createTemporaryView(), documentVersionToLoad);
3183 
3184  std::string json = "{ }";
3185  localDocSaver.getViewP()->setCustomStorageData(json);
3186 
3187  __COUTT__ << "Saving JSON string: "
3188  << localDocSaver.getViewP()->getCustomStorageData() << __E__;
3189 
3190  __COUTT__ << "Saving JSON doc as "
3191  << localDocSaver.getView().getTableName() << "("
3192  << localDocSaver.getView().getVersion().toString() << ")"
3193  << __E__;
3194 
3195  // save to db, and do not allow overwrite
3196  theInterface_->saveActiveVersion(&localDocSaver, false /* overwrite */);
3197  }
3198  catch(...)
3199  {
3200  __GEN_COUT__ << "Exception during table save." << __E__;
3201  }
3202  __GEN_COUTV__(runTimeSeconds());
3203 
3204  { //load to prove it exists
3205  TableBase localDocLoader(
3206  documentNameToLoad); //can not use special table when filling
3207  localDocLoader.changeVersionAndActivateView(
3208  localDocLoader.createTemporaryView(), documentVersionToLoad);
3209  theInterface_->fill(&localDocLoader, documentVersionToLoad);
3210  __SS__;
3211  localDocLoader.print(ss);
3212  __GEN_COUTV__(ss.str());
3213  }
3214  __GEN_COUTV__(runTimeSeconds());
3215  }
3216  __GEN_COUTV__(runTimeSeconds());
3217 
3218  //test a table save that does not already exist
3219  {
3220  std::string documentNameToLoad = "MessageFacilityTable";
3221  TableVersion documentVersionToLoad(7);
3222  TableBase localDocLoader(
3223  documentNameToLoad); //can not use special table when filling
3224 
3225  __GEN_COUT__ << "Testing table save of non-existing " << documentNameToLoad
3226  << __E__;
3227 
3228  { //load to prove it exists
3229  localDocLoader.changeVersionAndActivateView(
3230  localDocLoader.createTemporaryView(), documentVersionToLoad);
3231  theInterface_->fill(&localDocLoader, documentVersionToLoad);
3232  __SS__;
3233  localDocLoader.print(ss);
3234  __GEN_COUTV__(ss.str());
3235  __GEN_COUTV__(runTimeSeconds());
3236  }
3237  __GEN_COUTV__(runTimeSeconds());
3238 
3239  try
3240  { //attempt to save new version
3241 
3242  // modify it
3243  TableVersion newVersion = TableVersion::getNextVersion(
3244  theInterface_->findLatestVersion(&localDocLoader));
3245  localDocLoader.getViewP()->setVersion(newVersion);
3246 
3247  __GEN_COUTT__ << "Saving new table as "
3248  << localDocLoader.getView().getTableName() << "("
3249  << localDocLoader.getView().getVersion().toString() << ")"
3250  << __E__;
3251 
3252  localDocLoader.getViewP()->setValueAsString(
3253  "10.226.9.17", 0, 4); //modify value that is 10.226.9.16
3254 
3255  __SS__;
3256  localDocLoader.print(ss);
3257  __GEN_COUTV__(ss.str());
3258 
3259  // save to db, and do not allow overwrite
3260  theInterface_->saveActiveVersion(&localDocLoader, false /* overwrite */);
3261  }
3262  catch(...)
3263  {
3264  __GEN_COUT__ << "Exception during new table save." << __E__;
3265  }
3266  __GEN_COUTV__(runTimeSeconds());
3267  }
3268  __GEN_COUTV__(runTimeSeconds());
3269  return;
3270 
3271  // for each group get member map & comment, author, time, and type for latest key
3272  for(auto& groupInfo : allGroupInfo_)
3273  {
3274  try
3275  {
3276  groupInfo.second.latestKey_ = groupInfo.second.getLastKey();
3277  loadTableGroup(groupInfo.first /*groupName*/,
3278  groupInfo.second.latestKey_,
3279  false /*doActivate*/,
3280  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
3281  0 /*progressBar*/,
3282  0 /*accumulateErrors*/,
3283  &groupInfo.second.latestKeyGroupComment_,
3284  &groupInfo.second.latestKeyGroupAuthor_,
3285  &groupInfo.second.latestKeyGroupCreationTime_,
3286  true /*doNotLoadMember*/,
3287  &groupInfo.second.latestKeyGroupTypeString_);
3288  }
3289  catch(const std::runtime_error& e)
3290  {
3291  __GEN_COUT_WARN__
3292  << "Error occurred loading latest group info into cache for '"
3293  << groupInfo.first << "(" << groupInfo.second.latestKey_ << ")': \n"
3294  << e.what() << __E__;
3295 
3296  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
3297  groupInfo.second.latestKeyGroupComment_ =
3298  ConfigurationManager::UNKNOWN_INFO;
3299  groupInfo.second.latestKeyGroupAuthor_ =
3300  ConfigurationManager::UNKNOWN_INFO;
3301  groupInfo.second.latestKeyGroupCreationTime_ =
3302  ConfigurationManager::UNKNOWN_TIME;
3303  groupInfo.second.latestKeyGroupTypeString_ =
3304  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
3305  groupInfo.second.latestKeyMemberMap_ = {};
3306  }
3307  catch(...)
3308  {
3309  __GEN_COUT_WARN__
3310  << "Error occurred loading latest group info into cache for '"
3311  << groupInfo.first << "(" << groupInfo.second.latestKey_ << ")'..."
3312  << __E__;
3313 
3314  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
3315  groupInfo.second.latestKeyGroupComment_ =
3316  ConfigurationManager::UNKNOWN_INFO;
3317  groupInfo.second.latestKeyGroupAuthor_ =
3318  ConfigurationManager::UNKNOWN_INFO;
3319  groupInfo.second.latestKeyGroupCreationTime_ =
3320  ConfigurationManager::UNKNOWN_TIME;
3321  groupInfo.second.latestKeyGroupTypeString_ =
3322  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
3323  groupInfo.second.latestKeyMemberMap_ = {};
3324  }
3325  } // end group info loop
3326  __GEN_COUTV__(runTimeSeconds());
3327  } // end get group info
3328  catch(const std::runtime_error& e)
3329  {
3330  __SS__ << "A fatal error occurred reading the info for all table groups. Error: "
3331  << e.what() << __E__;
3332  __GEN_COUT_ERR__ << "\n" << ss.str();
3333  if(accumulatedWarnings)
3334  *accumulatedWarnings += ss.str();
3335  else
3336  throw;
3337  }
3338  catch(...)
3339  {
3340  __SS__ << "An unknown fatal error occurred reading the info for all table groups."
3341  << __E__;
3342  __GEN_COUT_ERR__ << "\n" << ss.str();
3343  if(accumulatedWarnings)
3344  *accumulatedWarnings += ss.str();
3345  else
3346  throw;
3347  } //end catch
3348 
3349  __GEN_COUT__ << "testXDAQContext() end runTimeSeconds()=" << runTimeSeconds()
3350  << __E__;
3351  return;
3352 
3353  try
3354  {
3355  __GEN_COUT__ << "Loading table..." << __E__;
3356  loadTableGroup("FETest", TableGroupKey(2)); // Context_1
3357  ConfigurationTree t = getNode("/FETable/DEFAULT/FrontEndType");
3358 
3359  std::string v;
3360 
3361  __GEN_COUT__ << __E__;
3362  t.getValue(v);
3363  __GEN_COUT__ << "Value: " << v << __E__;
3364  __GEN_COUT__ << "Value index: " << t.getValue<int>() << __E__;
3365 
3366  return;
3367  }
3368  catch(...)
3369  {
3370  __GEN_COUT__ << "Failed to load table..." << __E__;
3371  }
3372 } //end testXDAQContext()
ots::ConfigurationInterface::saveNewVersion
TableVersion saveNewVersion(TableBase *table, TableVersion temporaryVersion, TableVersion newVersion=TableVersion())
Definition: ConfigurationInterface.cc:79
ots::ConfigurationManagerRW::saveNewTable
TableVersion saveNewTable(const std::string &tableName, TableVersion temporaryVersion=TableVersion(), bool makeTemporary=false)
modifiers of generic TableBase
Definition: ConfigurationManagerRW.cc:1614
ots::ConfigurationManagerRW::findTableGroup
TableGroupKey findTableGroup(const std::string &groupName, const std::map< std::string, TableVersion > &groupMembers, const std::map< std::string, std::string > &groupAliases=std::map< std::string, std::string >())
Definition: ConfigurationManagerRW.cc:1840
ots::ConfigurationManagerRW::testXDAQContext
void testXDAQContext(void)
for debugging
Definition: ConfigurationManagerRW.cc:3022
ots::ConfigurationManagerRW::saveNewBackbone
TableVersion saveNewBackbone(TableVersion temporaryVersion=TableVersion())
Definition: ConfigurationManagerRW.cc:2427
ots::ConfigurationManagerRW::getGroupInfo
const GroupInfo & getGroupInfo(const std::string &groupName, bool attemptToReloadKeys=false)
public group cache handling
Definition: ConfigurationManagerRW.cc:1772
ots::ConfigurationManagerRW::getAllTableInfo
const std::map< std::string, TableInfo > & getAllTableInfo(bool refresh=false, std::string *accumulatedWarnings=0, const std::string &errorFilterName="", bool getGroupKeys=false, bool getGroupInfo=false, bool initializeActiveGroups=false)
Definition: ConfigurationManagerRW.cc:209
ots::ConfigurationManagerRW::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false)
Definition: ConfigurationManagerRW.cc:901
ots::ConfigurationManagerRW::copyViewToCurrentColumns
TableVersion copyViewToCurrentColumns(const std::string &tableName, TableVersion sourceVersion)
copyViewToCurrentColumns
Definition: ConfigurationManagerRW.cc:1748
ots::ConfigurationManagerRW::saveNewTableGroup
TableGroupKey saveNewTableGroup(const std::string &groupName, std::map< std::string, TableVersion > &groupMembers, const std::string &groupComment=TableViewColumnInfo::DATATYPE_COMMENT_DEFAULT, std::map< std::string, std::string > *groupAliases=0)
modifiers of a table group based on alias, e.g. "Physics"
Definition: ConfigurationManagerRW.cc:2164
ots::ConfigurationManagerRW::activateTableGroup
void activateTableGroup(const std::string &tableGroupName, TableGroupKey tableGroupKey, std::string *accumulatedTreeErrors=0, std::string *groupTypeString=0)
modifiers of table groups
Definition: ConfigurationManagerRW.cc:1151
ots::ConfigurationManagerRW::getMetadataTable
TableBase * getMetadataTable(TableVersion fillVersion=TableVersion())
created for use in otsdaq_flatten_system_aliases and otsdaq_export_system_aliases,...
Definition: ConfigurationManagerRW.cc:2105
ots::ConfigurationManagerRW::saveModifiedVersion
TableVersion saveModifiedVersion(const std::string &tableName, TableVersion originalVersion, bool makeTemporary, TableBase *config, TableVersion temporaryModifiedVersion, bool ignoreDuplicates=false, bool lookForEquivalent=false, bool *foundEquivalent=nullptr)
Definition: ConfigurationManagerRW.cc:2470
ots::ConfigurationManagerRW::createTemporaryBackboneView
TableVersion createTemporaryBackboneView(TableVersion sourceViewVersion=TableVersion())
-1, from MockUp, else from valid backbone view version
Definition: ConfigurationManagerRW.cc:1312
ots::ConfigurationManagerRW::clearCachedVersions
void clearCachedVersions(const std::string &tableName)
Definition: ConfigurationManagerRW.cc:1728
ots::ConfigurationManagerRW::getVersionAliases
std::map< std::string, std::map< std::string, TableVersion > > getVersionAliases(void) const
Definition: ConfigurationManagerRW.cc:1129
ots::ConfigurationManagerRW::eraseTemporaryVersion
void eraseTemporaryVersion(const std::string &tableName, TableVersion targetVersion=TableVersion())
Definition: ConfigurationManagerRW.cc:1679
ots::ConfigurationManagerRW::preloadVersionCreationTimes
void preloadVersionCreationTimes(void)
parallel load of all version creation times into the process-wide cache
Definition: ConfigurationManagerRW.cc:1443
ots::ConfigurationManagerRW::compareTableGroupThread
static void compareTableGroupThread(ConfigurationManagerRW *cfgMgr, std::string groupName, ots::TableGroupKey groupKeyToCompare, const std::map< std::string, TableVersion > &groupMemberMap, const std::map< std::string, std::string > &memberTableAliases, std::atomic< bool > *theFoundIdentical, ots::TableGroupKey *theIdenticalKey, std::mutex *theThreadMutex, std::shared_ptr< std::atomic< bool >> theThreadDone)
Definition: ConfigurationManagerRW.cc:998
ots::ConfigurationManagerRW::loadTableGroupThread
static void loadTableGroupThread(ConfigurationManagerRW *cfgMgr, std::string groupName, ots::TableGroupKey groupKey, std::shared_ptr< ots::GroupInfo > theGroupInfo, std::shared_ptr< std::atomic< bool >> theThreadDone)
loadTableGroupThread()
Definition: ConfigurationManagerRW.cc:956
ots::ConfigurationManagerRW::getVersionCreationTime
time_t getVersionCreationTime(const std::string &tableName, TableVersion version)
Definition: ConfigurationManagerRW.cc:1393
ots::ConfigurationManagerRW::loadTableInfoThread
static void loadTableInfoThread(ConfigurationManagerRW *cfgMgr, std::string tableName, TableBase *existingTable, std::shared_ptr< ots::TableInfo > tableInfo, std::shared_ptr< std::atomic< bool >> threadDone)
loadTableInfoThread()
Definition: ConfigurationManagerRW.cc:762
ots::ConfigurationManager::getVersionAliases
std::map< std::string, std::map< std::string, TableVersion > > getVersionAliases(void) const
Definition: ConfigurationManager.cc:4092
ots::ConfigurationManager::convertGroupTypeToName
static const std::string & convertGroupTypeToName(const ConfigurationManager::GroupType &groupTypeId)
Definition: ConfigurationManager.cc:651
ots::ConfigurationManager::restoreActiveTableGroups
void restoreActiveTableGroups(bool throwErrors=false, const std::string &pathToActiveGroupsFile="", ConfigurationManager::LoadGroupType onlyLoadIfBackboneOrContext=ConfigurationManager::LoadGroupType::ALL_TYPES, std::string *accumulatedWarnings=0)
Definition: ConfigurationManager.cc:328
ots::ConfigurationManager::getActiveVersions
std::map< std::string, TableVersion > getActiveVersions(void) const
getActiveVersions
Definition: ConfigurationManager.cc:4271
ots::ConfigurationManager::getNode
ConfigurationTree getNode(const std::string &nodeString, bool doNotThrowOnBrokenUIDLinks=false) const
"root/parent/parent/"
Definition: ConfigurationManager.cc:3616
ots::ConfigurationManager::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false, std::map< std::string, TableVersion > mergeInTables={}, std::map< std::string, TableVersion > overrideTables={})
Definition: ConfigurationManager.cc:2025
ots::ConfigurationManager::init
void init(std::string *accumulatedErrors=0, bool initForWriteAccess=false, std::string *accumulatedWarnings=0)
Definition: ConfigurationManager.cc:277
ots::ConfigurationManager::getTypeNameOfGroup
static const std::string & getTypeNameOfGroup(const std::map< std::string, TableVersion > &memberMap)
Definition: ConfigurationManager.cc:849
ots::ConfigurationManager::destroyTableGroup
void destroyTableGroup(const std::string &theGroup="", bool onlyDeactivate=false)
Definition: ConfigurationManager.cc:516
ots::ConfigurationManager::ACTIVE_GROUPS_FILENAME
static const std::string ACTIVE_GROUPS_FILENAME
added env check for otsdaq_flatten_active_to_version to function
Definition: ConfigurationManager.h:38
ots::ConfigurationManager::getTableByName
const TableBase * getTableByName(const std::string &configurationName) const
Definition: ConfigurationManager.cc:3904
ots::ConfigurationTree::getValue
void getValue(T &value) const
ots::ConfigurationTree::getChildren
std::vector< std::pair< std::string, ConfigurationTree > > getChildren(std::map< std::string, std::string > filterMap=std::map< std::string, std::string >(), bool byPriority=false, bool onlyStatusTrue=false) const
Definition: ConfigurationTree.cc:2699
ots::TableBase::createTemporaryView
TableVersion createTemporaryView(TableVersion sourceViewVersion=TableVersion(), TableVersion destTemporaryViewVersion=TableVersion::getNextTemporaryVersion())
source of -1, from MockUp, else from valid view version
Definition: TableBase.cc:1773
ots::TableBase::trimTemporary
void trimTemporary(TableVersion targetVersion=TableVersion())
Definition: TableBase.cc:362
ots::TableBase::getNumberOfStoredViews
unsigned int getNumberOfStoredViews(void) const
Definition: TableBase.cc:856
ots::TableBase::checkForDuplicate
TableVersion checkForDuplicate(TableVersion needleVersion, TableVersion ignoreVersion=TableVersion()) const
Definition: TableBase.cc:404
ots::TableBase::getTemporaryView
TableView * getTemporaryView(TableVersion temporaryVersion)
Definition: TableBase.cc:1884
ots::TableBase::MAX_VIEWS_IN_CACHE
const unsigned int MAX_VIEWS_IN_CACHE
Definition: TableBase.h:30
ots::TableBase::getNextVersion
TableVersion getNextVersion(void) const
Definition: TableBase.cc:1860
ots::TableBase::copyView
TableVersion copyView(const TableView &sourceView, TableVersion destinationVersion, const std::string &author, bool looseColumnMatching=false)
Definition: TableBase.cc:1714
ots::TableBase::print
void print(std::ostream &out=std::cout) const
always prints active view
Definition: TableBase.cc:277
ots::TableBase::getNextTemporaryVersion
TableVersion getNextTemporaryVersion(void) const
Definition: TableBase.cc:1837
ots::TableBase::trimCache
void trimCache(unsigned int trimSize=-1)
Definition: TableBase.cc:322
ots::TableGroupKey::toString
std::string toString(void) const
toString
Definition: TableGroupKey.cc:111
ots::TableGroupKey::getNextKey
static TableGroupKey getNextKey(const TableGroupKey &key=TableGroupKey())
Definition: TableGroupKey.cc:156
ots::TableGroupKey::getGroupNameAndKey
static void getGroupNameAndKey(const std::string &fullGroupString, std::string &groupName, TableGroupKey &key)
requires fullGroupString created as name + "_v" + key + ""
Definition: TableGroupKey.cc:218
ots::TableGroupKey::isInvalid
bool isInvalid(void) const
isInvalid
Definition: TableGroupKey.cc:148
ots::TableGroupKey::getFullGroupString
static std::string getFullGroupString(const std::string &groupName, const TableGroupKey &key, const std::string &preKey="_v", const std::string &postKey="")
Definition: TableGroupKey.cc:174
ots::TableVersion::toString
std::string toString(void) const
toString
Definition: TableVersion.cc:33
ots::TableVersion::isInvalid
bool isInvalid(void) const
isInvalid
Definition: TableVersion.cc:127
ots::TableVersion::getNextVersion
static TableVersion getNextVersion(const TableVersion &version=TableVersion())
Definition: TableVersion.cc:161
ots::TableVersion::isScratchVersion
bool isScratchVersion(void) const
Definition: TableVersion.cc:145
ots::TableVersion::isTemporaryVersion
bool isTemporaryVersion(void) const
Definition: TableVersion.cc:137
ots::TableVersion::getNextTemporaryVersion
static TableVersion getNextTemporaryVersion(const TableVersion &version=TableVersion())
Definition: TableVersion.cc:174
ots::TableView::setValueAsString
void setValueAsString(const std::string &value, unsigned int row, unsigned int col)
Definition: TableView.cc:1090
ots::TableView::setVersion
void setVersion(const T &version)
< in included .icc source
ots::TableView::getCustomStorageData
const std::string & getCustomStorageData(void) const
Getters.
Definition: TableView.h:71
ots::TableView::findCol
unsigned int findCol(const std::string &name) const
Definition: TableView.cc:1973
ots::TableView::setCustomStorageData
void setCustomStorageData(const std::string &storageData)
Definition: TableView.h:168
ots
defines used also by OtsConfigurationWizardSupervisor
Definition: ArtdaqOtsBuildInfo_module.cc:10
ots::GroupEditStruct::getTableEditStruct
TableEditStruct & getTableEditStruct(const std::string &tableName, bool markModified=false)
Note: if markModified, and table not found in group, this function will try to add it to group.
Definition: ConfigurationManagerRW.cc:2674
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
setToString ~
ots::StringMacros::mapToString
static std::string mapToString(const std::map< std::string, T > &mapToReturn, const std::string &primaryDelimeter=", ", const std::string &secondaryDelimeter=": ")
ots::StringMacros::stackTrace
static std::string stackTrace(void)
Definition: StringMacros.cc:1836