otsdaq  3.09.00
ConfigurationManagerRW.cc
1 #include "otsdaq/ConfigurationInterface/ConfigurationManagerRW.h"
2 
3 #include <dirent.h>
4 #include <chrono>
5 
6 using namespace ots;
7 
8 #undef __MF_SUBJECT__
9 #define __MF_SUBJECT__ "ConfigurationManagerRW"
10 
11 #define TABLE_INFO_PATH std::string(__ENV__("TABLE_INFO_PATH")) + "/"
12 #define TABLE_INFO_EXT "Info.xml"
13 
14 #define CORE_TABLE_INFO_FILENAME \
15  ((getenv("SERVICE_DATA_PATH") == NULL) \
16  ? (std::string(__ENV__("USER_DATA")) + "/ServiceData") \
17  : (std::string(__ENV__("SERVICE_DATA_PATH")))) + \
18  "/CoreTableInfoNames.dat"
19 
20 std::atomic<bool> ConfigurationManagerRW::firstTimeConstructed_ = true;
21 
22 std::mutex ConfigurationManagerRW::versionCreationTimeCacheMutex_;
23 std::map<std::string, std::map<TableVersion, time_t>>
24  ConfigurationManagerRW::versionCreationTimeCache_;
25 
26 //==============================================================================
28 ConfigurationManagerRW::ConfigurationManagerRW(const std::string& username)
29  : ConfigurationManager(username) // for use as author of new views
30 {
31  __GEN_COUT__ << "Instantiating Config Manager with Write Access! (for " << username
32  << ") time=" << time(0) << " runTimeSeconds()=" << runTimeSeconds()
33  << __E__;
34 
35  theInterface_ = ConfigurationInterface::getInstance(
36  ConfigurationInterface::CONFIGURATION_MODE::
37  ARTDAQ_DATABASE); // false to use artdaq DB
38 
39  //=========================
40  // dump names of core tables (so UpdateOTS.sh can copy core tables for user)
41  // only if table does not exist
42  if(firstTimeConstructed_)
43  {
44  firstTimeConstructed_ = false;
45 
46  // make table group history directory here and at Gateway Supervisor (just in case)
47  mkdir((ConfigurationManager::LAST_TABLE_GROUP_SAVE_PATH).c_str(), 0755);
48 
49  const std::set<std::string>& contextMemberNames = getFixedContextMemberNames();
50  const std::set<std::string>& backboneMemberNames = getBackboneMemberNames();
51  const std::set<std::string>& iterateMemberNames = getIterateMemberNames();
52 
53  FILE* fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "r");
54 
55  if(fp) // check for all core table names in file, and force their presence
56  {
57  std::vector<unsigned int> foundVector;
58  char line[100];
59  for(const auto& name : contextMemberNames)
60  {
61  foundVector.push_back(false);
62  rewind(fp);
63  while(fgets(line, 100, fp))
64  {
65  if(strlen(line) < 1)
66  continue;
67  line[strlen(line) - 1] = '\0'; // remove endline
68  if(strcmp(line, ("ContextGroup/" + name).c_str()) == 0) // is match?
69  {
70  foundVector.back() = true;
71  break;
72  }
73  }
74  }
75 
76  for(const auto& name : backboneMemberNames)
77  {
78  foundVector.push_back(false);
79  rewind(fp);
80  while(fgets(line, 100, fp))
81  {
82  if(strlen(line) < 1)
83  continue;
84  line[strlen(line) - 1] = '\0'; // remove endline
85  if(strcmp(line, ("BackboneGroup/" + name).c_str()) == 0) // is match?
86  {
87  foundVector.back() = true;
88  break;
89  }
90  }
91  }
92 
93  for(const auto& name : iterateMemberNames)
94  {
95  foundVector.push_back(false);
96  rewind(fp);
97  while(fgets(line, 100, fp))
98  {
99  if(strlen(line) < 1)
100  continue;
101  line[strlen(line) - 1] = '\0'; // remove endline
102  if(strcmp(line, ("IterateGroup/" + name).c_str()) == 0) // is match?
103  {
104  foundVector.back() = true;
105  break;
106  }
107  }
108  }
109 
110  //look for optional Context table ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE
111  {
112  foundVector.push_back(false);
113  rewind(fp);
114  while(fgets(line, 100, fp))
115  {
116  if(strlen(line) < 1)
117  continue;
118  line[strlen(line) - 1] = '\0'; // remove endline
119  if(strcmp(line,
120  ("ContextGroup/" +
121  ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE)
122  .c_str()) == 0) // is match?
123  {
124  foundVector.back() = true;
125  break;
126  }
127  }
128  }
129 
130  fclose(fp);
131 
132  // open file for appending the missing names
133  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "a");
134  if(fp)
135  {
136  unsigned int i = 0;
137  for(const auto& name : contextMemberNames)
138  {
139  if(!foundVector[i])
140  fprintf(fp, "\nContextGroup/%s", name.c_str());
141 
142  ++i;
143  }
144  for(const auto& name : backboneMemberNames)
145  {
146  if(!foundVector[i])
147  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
148 
149  ++i;
150  }
151  for(const auto& name : iterateMemberNames)
152  {
153  if(!foundVector[i])
154  fprintf(fp, "\nIterateGroup/%s", name.c_str());
155 
156  ++i;
157  }
158 
159  //last is optional Context table ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE
160  if(!foundVector[i])
161  fprintf(
162  fp,
163  "\nContextGroup/%s",
164  ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE.c_str());
165 
166  fclose(fp);
167  }
168  else
169  {
170  __SS__ << "Failed to open core table info file for appending: "
171  << CORE_TABLE_INFO_FILENAME << __E__;
172  __SS_THROW__;
173  }
174  }
175  else
176  {
177  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "w");
178  if(fp)
179  {
180  fprintf(fp, "ARTDAQ/*");
181  fprintf(fp, "\nConfigCore/*");
182  for(const auto& name : contextMemberNames)
183  fprintf(fp, "\nContextGroup/%s", name.c_str());
184  for(const auto& name : backboneMemberNames)
185  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
186  for(const auto& name : iterateMemberNames)
187  fprintf(fp, "\nIterateGroup/%s", name.c_str());
188  fclose(fp);
189  }
190  else
191  {
192  __SS__ << "Failed to open core table info file: "
193  << CORE_TABLE_INFO_FILENAME << __E__;
194  __SS_THROW__;
195  }
196  }
197  } // end dump names of core tables
198 
199  __GEN_COUTV__(runTimeSeconds());
200 } // end constructor
201 
202 //==============================================================================
209 const std::map<std::string, TableInfo>& ConfigurationManagerRW::getAllTableInfo(
210  bool refresh /* = false */,
211  std::string* accumulatedWarnings /* = 0 */,
212  const std::string& errorFilterName /* = "" */,
213  bool getGroupKeys /* = false */,
214  bool getGroupInfo /* = false */,
215  bool initializeActiveGroups /* = false */)
216 {
217  // allTableInfo_ is container to be returned
218 
219  if(!refresh)
220  return allTableInfo_;
221 
222  // else refresh!
223  allTableInfo_.clear();
224 
225  TableBase* table;
226 
227  // existing configurations are defined by which infos are in TABLE_INFO_PATH
228  // can test that the class exists based on this
229  // and then which versions
230  __GEN_COUT__ << "======================================================== "
231  "getAllTableInfo start runTimeSeconds()="
232  << runTimeSeconds() << __E__;
233  {
234  __GEN_COUT__ << "Refreshing all! Extracting list of tables..." << __E__;
235  DIR* pDIR;
236  struct dirent* entry;
237  std::string path = TABLE_INFO_PATH;
238  char fileExt[] = TABLE_INFO_EXT;
239  const unsigned char MIN_TABLE_NAME_SZ = 3;
240 
241  const int numOfThreads = PROCESSOR_COUNT / 2;
242  __GEN_COUT__ << " PROCESSOR_COUNT " << PROCESSOR_COUNT << " ==> " << numOfThreads
243  << " threads." << __E__;
244  if(numOfThreads < 2) // no multi-threading
245  {
246  if((pDIR = opendir(path.c_str())) != 0)
247  {
248  while((entry = readdir(pDIR)) != 0)
249  {
250  // enforce table name length
251  if(strlen(entry->d_name) < strlen(fileExt) + MIN_TABLE_NAME_SZ)
252  continue;
253 
254  // find file names with correct file extenstion
255  if(strcmp(&(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
256  fileExt) != 0)
257  continue; // skip different extentions
258 
259  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] =
260  '\0'; // remove file extension to get table name
261 
262  // 0 will force the creation of new instance (and reload from Info)
263  table = 0;
264 
265  try // only add valid table instances to maps
266  {
267  theInterface_->get(table,
268  entry->d_name,
269  0,
270  0,
271  true); // dont fill
272  }
273  catch(cet::exception const&)
274  {
275  if(table)
276  delete table;
277  table = 0;
278 
279  __GEN_COUT__ << "Skipping! No valid class found for... "
280  << entry->d_name << "\n";
281  continue;
282  }
283  catch(std::runtime_error& e)
284  {
285  if(table)
286  delete table;
287  table = 0;
288 
289  __GEN_COUT__ << "Skipping! No valid class found for... "
290  << entry->d_name << "\n";
291  __GEN_COUT__ << "Error: " << e.what() << __E__;
292 
293  // for a runtime_error, it is likely that columns are the problem
294  // the Table Editor needs to still fix these.. so attempt to
295  // proceed.
296  if(accumulatedWarnings)
297  {
298  if(errorFilterName == "" || errorFilterName == entry->d_name)
299  {
300  *accumulatedWarnings += std::string("\nIn table '") +
301  entry->d_name + "'..." +
302  e.what(); // global accumulate
303 
304  __SS__ << "Attempting to allow illegal columns!" << __E__;
305  *accumulatedWarnings += ss.str();
306  }
307 
308  // attempt to recover and build a mock-up
309  __GEN_COUT__ << "Attempting to allow illegal columns!"
310  << __E__;
311 
312  std::string returnedAccumulatedErrors;
313  try
314  {
315  table = new TableBase(entry->d_name,
316  &returnedAccumulatedErrors);
317  }
318  catch(...)
319  {
320  __GEN_COUT__ << "Skipping! Allowing illegal columns "
321  "didn't work either... "
322  << entry->d_name << "\n";
323  continue;
324  }
325  __GEN_COUT__
326  << "Error (but allowed): " << returnedAccumulatedErrors
327  << __E__;
328 
329  if(errorFilterName == "" || errorFilterName == entry->d_name)
330  *accumulatedWarnings +=
331  std::string("\nIn table '") + entry->d_name + "'..." +
332  returnedAccumulatedErrors; // global accumulate
333  }
334  else
335  continue;
336  }
337 
338  if(nameToTableMap_[entry->d_name]) // handle if instance existed
339  {
340  // copy the existing temporary versions! (or else all is lost)
341  std::set<TableVersion> versions =
342  nameToTableMap_[entry->d_name]->getStoredVersions();
343  for(auto& version : versions)
344  if(version.isTemporaryVersion())
345  {
346  try // do NOT let TableView::init() throw here
347  {
348  nameToTableMap_[entry->d_name]->setActiveView(
349  version);
350  table->copyView( // this calls TableView::init()
351  nameToTableMap_[entry->d_name]->getView(),
352  version,
353  username_);
354  }
355  catch(
356  ...) // do NOT let invalid temporary version throw at this
357  // point
358  {
359  } // just trust configurationBase throws out the failed version
360  }
361 
362  delete nameToTableMap_[entry->d_name];
363  nameToTableMap_[entry->d_name] = 0;
364  }
365 
366  nameToTableMap_[entry->d_name] = table;
367 
368  allTableInfo_[entry->d_name].tablePtr_ = table;
369  allTableInfo_[entry->d_name].versions_ =
370  theInterface_->getVersions(table);
371 
372  // also add any existing temporary versions to all table info
373  // because the interface wont find those versions
374  std::set<TableVersion> versions =
375  nameToTableMap_[entry->d_name]->getStoredVersions();
376  for(auto& version : versions)
377  if(version.isTemporaryVersion())
378  {
379  allTableInfo_[entry->d_name].versions_.emplace(version);
380  }
381  } //end table name handling from directory
382  closedir(pDIR);
383  }
384  }
385  else //multi-threading
386  {
387  int threadsLaunched = 0;
388  int foundThreadIndex = 0;
389  std::string tableName;
390 
391  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
392  for(int i = 0; i < numOfThreads; ++i)
393  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
394 
395  std::vector<std::shared_ptr<ots::TableInfo>> sharedTableInfoPtrs;
396 
397  if((pDIR = opendir(path.c_str())) != 0)
398  {
399  while((entry = readdir(pDIR)) != 0)
400  {
401  // enforce table name length
402  if(strlen(entry->d_name) < strlen(fileExt) + MIN_TABLE_NAME_SZ)
403  continue;
404 
405  // find file names with correct file extenstion
406  if(strcmp(&(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
407  fileExt) != 0)
408  continue; // skip different extentions
409 
410  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] =
411  '\0'; // remove file extension to get table name
412  tableName =
413  entry
414  ->d_name; //copy immediate, thread does not like using pointer to char*, corrupts immediately on wraparound
415 
416  //make temporary table info for thread
417  sharedTableInfoPtrs.push_back(std::make_shared<ots::TableInfo>());
418  sharedTableInfoPtrs.back()->accumulatedWarnings_ =
419  accumulatedWarnings ? "ALLOW"
420  : ""; //mark to allow accumulated warnings
421 
422  if(threadsLaunched >= numOfThreads)
423  {
424  //find availableThreadIndex
425  foundThreadIndex = -1;
426  size_t ii = 0;
427  while(foundThreadIndex == -1)
428  {
429  for(int i = 0; i < numOfThreads; ++i)
430  if(*(threadDone[i]))
431  {
432  foundThreadIndex = i;
433  break;
434  }
435  if(foundThreadIndex == -1)
436  {
437  __GEN_COUT_TYPE__(TLVL_DEBUG + 2)
438  << __COUT_HDR__
439  << "Waiting for available thread... iteration # "
440  << ii << __E__;
441  if(++ii > 100 /* 1s */ * 10)
442  {
443  __GEN_SS__ << "Threads seem to be stuck getting "
444  "table info! Timeout while waiting..."
445  << __E__;
446  __GEN_SS_THROW__;
447  }
448  usleep(10000);
449  }
450  } //end thread search loop
451  threadsLaunched = numOfThreads - 1;
452  }
453  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex
454  << " for table " << tableName << __E__;
455 
456  *(threadDone[foundThreadIndex]) = false;
457  std::thread(
458  [](ConfigurationManagerRW* theCfgMgr,
459  std::string theTableName,
460  TableBase* existingTable,
461  std::shared_ptr<ots::TableInfo> theTableInfo,
462  std::shared_ptr<std::atomic<bool>> theThreadDone) {
463  ConfigurationManagerRW::loadTableInfoThread(theCfgMgr,
464  theTableName,
465  existingTable,
466  theTableInfo,
467  theThreadDone);
468  },
469  this,
470  tableName,
471  nameToTableMap_[tableName],
472  sharedTableInfoPtrs.back(),
473  threadDone[foundThreadIndex])
474  .detach();
475 
476  ++threadsLaunched;
477  ++foundThreadIndex;
478  } //end table name handling from directory
479  closedir(pDIR);
480  } //end tableInfo thread loop
481 
482  //check for all threads done
483  do
484  {
485  foundThreadIndex = -1;
486  for(int i = 0; i < numOfThreads; ++i)
487  if(!*(threadDone[i]))
488  {
489  foundThreadIndex = i;
490  break;
491  }
492  if(foundThreadIndex != -1)
493  {
494  __GEN_COUTT__ << "Waiting for thread to finish... "
495  << foundThreadIndex << __E__;
496  usleep(10000);
497  }
498  } while(foundThreadIndex != -1); //end thread done search loop
499 
500  //threads done now, so copy table info
501  for(auto& tableInfo : sharedTableInfoPtrs)
502  {
503  if(tableInfo->tablePtr_ == nullptr)
504  {
505  __SS__ << "Fatal error occurred loading table info into cache! "
506  "Perhaps there is an illegal Table schema definition? "
507  "Check logs to resolve."
508  << __E__;
509  __SS_THROW__;
510  }
511  __GEN_COUT_TYPE__(TLVL_DEBUG + 3)
512  << __COUT_HDR__ << "Copying table info for "
513  << tableInfo->tablePtr_->getTableName() << __E__;
514  nameToTableMap_[tableInfo->tablePtr_->getTableName()] =
515  tableInfo->tablePtr_;
516  allTableInfo_[tableInfo->tablePtr_->getTableName()].tablePtr_ =
517  tableInfo->tablePtr_;
518  allTableInfo_[tableInfo->tablePtr_->getTableName()].versions_ =
519  tableInfo->versions_;
520  } //end copy group info loop
521  }
522  __GEN_COUT__ << "Extracting list of tables complete." << __E__;
523  } //end Extracting list of tables
524 
525  // call init to load active versions by default, activate with warnings allowed (assuming development going on)
526  if(initializeActiveGroups)
527  {
528  __GEN_COUT__ << "Now initializing..." << __E__;
529  // if there is a filter name, do not include init warnings (it just scares people in the table editor)
530  std::string tmpAccumulateWarnings;
531  init(0 /*accumulatedErrors*/,
532  false /*initForWriteAccess*/,
533  accumulatedWarnings ? &tmpAccumulateWarnings : nullptr);
534 
535  if(accumulatedWarnings && errorFilterName == "")
536  *accumulatedWarnings += tmpAccumulateWarnings;
537  }
538  __GEN_COUT__ << "======================================================== "
539  "getAllTableInfo end runTimeSeconds()="
540  << runTimeSeconds() << __E__;
541 
542  // get Group Info too!
543  if(getGroupKeys || getGroupInfo)
544  {
545  allGroupInfo_.clear();
546  try
547  {
548  // build allGroupInfo_ for the ConfigurationManagerRW
549 
550  std::set<std::string /*name*/> tableGroups =
551  theInterface_->getAllTableGroupNames();
552  __GEN_COUT__ << "Number of Groups: " << tableGroups.size() << __E__;
553 
554  __GEN_COUTT__ << "Group Info start runTimeSeconds()=" << runTimeSeconds()
555  << __E__;
556 
557  TableGroupKey key;
558  std::string name;
559  for(const auto& fullName : tableGroups)
560  {
561  TableGroupKey::getGroupNameAndKey(fullName, name, key);
562  allGroupInfo_[name].keys_.emplace(key); //store in cache
563  }
564 
565  __GEN_COUTT__ << "Group Keys end runTimeSeconds()=" << runTimeSeconds()
566  << __E__;
567 
568  // for each group get member map & comment, author, time, and type for latest key
569  if(getGroupInfo)
570  {
571  const int numOfThreads = PROCESSOR_COUNT / 2;
572  __GEN_COUT__ << " PROCESSOR_COUNT " << PROCESSOR_COUNT << " ==> "
573  << numOfThreads << " threads." << __E__;
574  if(numOfThreads < 2) // no multi-threading
575  for(auto& groupInfo : allGroupInfo_)
576  {
577  try
578  {
579  groupInfo.second.latestKey_ = groupInfo.second.getLastKey();
580  loadTableGroup(
581  groupInfo.first /*groupName*/,
582  groupInfo.second.latestKey_,
583  false /*doActivate*/,
584  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
585  0 /*progressBar*/,
586  0 /*accumulateErrors*/,
587  &groupInfo.second.latestKeyGroupComment_,
588  &groupInfo.second.latestKeyGroupAuthor_,
589  &groupInfo.second.latestKeyGroupCreationTime_,
590  true /*doNotLoadMember*/,
591  &groupInfo.second.latestKeyGroupTypeString_);
592  }
593  catch(...)
594  {
595  __GEN_COUT_WARN__ << "Error occurred loading latest group "
596  "info into cache for '"
597  << groupInfo.first << "("
598  << groupInfo.second.latestKey_ << ")'..."
599  << __E__;
600  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
601  groupInfo.second.latestKeyGroupComment_ =
602  ConfigurationManager::UNKNOWN_INFO;
603  groupInfo.second.latestKeyGroupAuthor_ =
604  ConfigurationManager::UNKNOWN_INFO;
605  groupInfo.second.latestKeyGroupCreationTime_ =
606  ConfigurationManager::UNKNOWN_TIME;
607  groupInfo.second.latestKeyGroupTypeString_ =
608  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
609  groupInfo.second.latestKeyMemberMap_ = {};
610  }
611  } // end group info loop
612  else //multi-threading
613  {
614  int threadsLaunched = 0;
615  int foundThreadIndex = 0;
616 
617  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
618  for(int i = 0; i < numOfThreads; ++i)
619  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
620 
621  std::vector<std::shared_ptr<ots::GroupInfo>> sharedGroupInfoPtrs;
622 
623  for(auto& groupInfo : allGroupInfo_)
624  {
625  //make temporary group info for thread
626  sharedGroupInfoPtrs.push_back(std::make_shared<ots::GroupInfo>());
627 
628  if(threadsLaunched >= numOfThreads)
629  {
630  //find availableThreadIndex
631  foundThreadIndex = -1;
632  while(foundThreadIndex == -1)
633  {
634  for(int i = 0; i < numOfThreads; ++i)
635  if(*(threadDone[i]))
636  {
637  foundThreadIndex = i;
638  break;
639  }
640  if(foundThreadIndex == -1)
641  {
642  __GEN_COUTT__ << "Waiting for available thread..."
643  << __E__;
644  usleep(10000);
645  }
646  } //end thread search loop
647  threadsLaunched = numOfThreads - 1;
648  }
649  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex
650  << " for " << groupInfo.first << "("
651  << groupInfo.second.getLastKey() << ")" << __E__;
652 
653  *(threadDone[foundThreadIndex]) = false;
654 
655  std::thread(
656  [](ConfigurationManagerRW* theCfgMgr,
657  std::string theGroupName,
658  ots::TableGroupKey theGroupKey,
659  std::shared_ptr<ots::GroupInfo> theGroupInfo,
660  std::shared_ptr<std::atomic<bool>> theThreadDone) {
661  ConfigurationManagerRW::loadTableGroupThread(
662  theCfgMgr,
663  theGroupName,
664  theGroupKey,
665  theGroupInfo,
666  theThreadDone);
667  },
668  this,
669  groupInfo.first,
670  groupInfo.second.getLastKey(),
671  sharedGroupInfoPtrs.back(),
672  threadDone[foundThreadIndex])
673  .detach();
674 
675  ++threadsLaunched;
676  ++foundThreadIndex;
677  } //end groupInfo thread loop
678 
679  //check for all threads done
680  do
681  {
682  foundThreadIndex = -1;
683  for(int i = 0; i < numOfThreads; ++i)
684  if(!*(threadDone[i]))
685  {
686  foundThreadIndex = i;
687  break;
688  }
689  if(foundThreadIndex != -1)
690  {
691  __GEN_COUTT__ << "Waiting for thread to finish... "
692  << foundThreadIndex << __E__;
693  usleep(10000);
694  }
695  } while(foundThreadIndex != -1); //end thread done search loop
696 
697  //threads done now, so copy group info
698  size_t i = 0;
699  for(auto& groupInfo : allGroupInfo_)
700  {
701  groupInfo.second.latestKey_ = sharedGroupInfoPtrs[i]->latestKey_;
702  groupInfo.second.latestKeyGroupComment_ =
703  sharedGroupInfoPtrs[i]->latestKeyGroupComment_;
704  groupInfo.second.latestKeyGroupAuthor_ =
705  sharedGroupInfoPtrs[i]->latestKeyGroupAuthor_;
706  groupInfo.second.latestKeyGroupCreationTime_ =
707  sharedGroupInfoPtrs[i]->latestKeyGroupCreationTime_;
708  groupInfo.second.latestKeyGroupTypeString_ =
709  sharedGroupInfoPtrs[i]->latestKeyGroupTypeString_;
710  groupInfo.second.latestKeyMemberMap_ =
711  sharedGroupInfoPtrs[i]->latestKeyMemberMap_;
712  ++i;
713  } //end copy group info loop
714 
715  } //end multi-thread handling
716  }
717  } // end get group info
718  catch(const std::runtime_error& e)
719  {
720  __SS__
721  << "A fatal error occurred reading the info for all table groups. Error: "
722  << e.what() << __E__;
723  __GEN_COUT_ERR__ << "Error at time: " << time(0) << "\n" << ss.str();
724  if(accumulatedWarnings)
725  *accumulatedWarnings += ss.str();
726  else
727  throw;
728  }
729  catch(...)
730  {
731  __SS__ << "An unknown fatal error occurred reading the info for all table "
732  "groups."
733  << __E__;
734  try
735  {
736  throw;
737  } //one more try to printout extra info
738  catch(const std::exception& e)
739  {
740  ss << "Exception message: " << e.what();
741  }
742  catch(...)
743  {
744  }
745  __GEN_COUT_ERR__ << "\n" << ss.str();
746  if(accumulatedWarnings)
747  *accumulatedWarnings += ss.str();
748  else
749  throw;
750  }
751  __GEN_COUTT__ << "Group Info end runTimeSeconds()=" << runTimeSeconds() << __E__;
752  } //end getGroupInfo
753  else
754  __GEN_COUTT__ << "Table Info end runTimeSeconds()=" << runTimeSeconds() << __E__;
755 
756  return allTableInfo_;
757 } // end getAllTableInfo()
758 
759 //==============================================================================
761 void ConfigurationManagerRW::loadTableInfoThread(
762  ConfigurationManagerRW* cfgMgr,
763  std::string tableName,
764  TableBase* existingTable,
765  std::shared_ptr<ots::TableInfo> tableInfo,
766  std::shared_ptr<std::atomic<bool>> threadDone)
767 try
768 {
769  __COUTT__ << "Thread started... table " << tableName << __E__;
770 
771  // 0 will force the creation of new instance (and reload from Info)
772  tableInfo->tablePtr_ = 0;
773 
774  try // only add valid table instances to maps
775  {
776  cfgMgr->theInterface_->get(tableInfo->tablePtr_,
777  tableName,
778  0,
779  0,
780  true); // dont fill
781  }
782  catch(cet::exception const&)
783  {
784  if(tableInfo->tablePtr_)
785  delete tableInfo->tablePtr_;
786  tableInfo->tablePtr_ = 0;
787 
788  __COUT__ << "Skipping! No valid class found for... " << tableName << "\n";
789  *(threadDone) = true;
790  return;
791  }
792  catch(std::runtime_error& e)
793  {
794  if(tableInfo->tablePtr_)
795  delete tableInfo->tablePtr_;
796  tableInfo->tablePtr_ = 0;
797 
798  __COUT__ << "Skipping! No valid class found for... " << tableName << "\n";
799  __COUTT__ << "Error: " << e.what() << __E__;
800 
801  // for a runtime_error, it is likely that columns are the problem
802  // the Table Editor needs to still fix these.. so attempt to
803  // proceed.
804  if(tableInfo->accumulatedWarnings_ == "ALLOW")
805  {
806  tableInfo->accumulatedWarnings_ = "";
807  if(1) //errorFilterName == "" || errorFilterName == tableName)
808  {
809  tableInfo->accumulatedWarnings_ += std::string("\nIn table '") +
810  tableName + "'..." +
811  e.what(); // global accumulate
812 
813  __SS__ << "Attempting to allow illegal columns!" << __E__;
814  tableInfo->accumulatedWarnings_ += ss.str();
815  }
816 
817  // attempt to recover and build a mock-up
818  __COUT__ << "Attempting to allow illegal columns!" << __E__;
819 
820  std::string returnedAccumulatedErrors;
821  try
822  {
823  tableInfo->tablePtr_ =
824  new TableBase(tableName, &returnedAccumulatedErrors);
825  }
826  catch(...)
827  {
828  __COUT__ << "Skipping! Allowing illegal columns didn't work either... "
829  << tableName << "\n";
830  *(threadDone) = true;
831  return;
832  }
833  __COUT_WARN__ << "Error (but allowed): " << returnedAccumulatedErrors
834  << __E__;
835 
836  if(1) //errorFilterName == "" || errorFilterName == entry->d_name)
837  tableInfo->accumulatedWarnings_ +=
838  std::string("\nIn table '") + tableName + "'..." +
839  returnedAccumulatedErrors; // global accumulate
840  }
841  else
842  {
843  tableInfo->accumulatedWarnings_ = "";
844  *(threadDone) = true;
845  return;
846  }
847  }
848 
849  if(existingTable) // handle if instance existed
850  {
851  __COUTT__ << "Copying temporary version from existing table object for "
852  << tableName << __E__;
853  // copy the existing temporary versions! (or else all is lost)
854  std::set<TableVersion> versions = existingTable->getStoredVersions();
855  for(auto& version : versions)
856  if(version.isTemporaryVersion())
857  {
858  try // do NOT let TableView::init() throw here
859  {
860  existingTable->setActiveView(version);
861  tableInfo->tablePtr_->copyView( // this calls TableView::init()
862  existingTable->getView(),
863  version,
864  cfgMgr->username_);
865  }
866  catch(...) // do NOT let invalid temporary version throw at this
867  // point
868  {
869  } // just trust configurationBase throws out the failed version
870  }
871 
872  delete existingTable;
873  existingTable = 0;
874  }
875 
876  tableInfo->versions_ = cfgMgr->theInterface_->getVersions(tableInfo->tablePtr_);
877 
878  // also add any existing temporary versions to all table info
879  // because the interface wont find those versions
880  std::set<TableVersion> versions = tableInfo->tablePtr_->getStoredVersions();
881  for(auto& version : versions)
882  if(version.isTemporaryVersion())
883  {
884  tableInfo->versions_.emplace(version);
885  }
886 
887  __COUTT__ << "Thread done... table " << tableName << __E__;
888  *(threadDone) = true;
889 } // end loadTableInfoThread
890 catch(...)
891 {
892  __COUT_ERR__ << "Error occurred loading latest table info into cache for '"
893  << tableName << "'..." << __E__;
894  *(threadDone) = true;
895 } // end loadTableInfoThread catch
896 
897 //==============================================================================
900 void ConfigurationManagerRW::loadTableGroup(
901  const std::string& groupName,
902  const TableGroupKey& groupKey,
903  bool doActivate /*=false*/,
904  std::map<std::string /*table name*/, TableVersion>*
905  groupMembers /*=0 , note: db time intensive! */,
906  ProgressBar* progressBar /*=0*/,
907  std::string* accumulatedWarnings /*=0*/,
908  std::string* groupComment /*=0 , note: in metadata */,
909  std::string* groupAuthor /*=0 , note: in metadata */,
910  std::string* groupCreateTime /*=0 , note: in metadata */,
911  bool doNotLoadMembers /*=false*/,
912  std::string* groupTypeString /*=0 , note: db time intensive! */,
913  std::map<std::string /*name*/, std::string /*alias*/>*
914  groupAliases /*=0 , note: in metadata */,
915  ConfigurationManager::LoadGroupType
916  groupTypeToLoad /*=ConfigurationManager::LoadGroupType::ALL_TYPES*/,
917  bool ignoreVersionTracking /*=false*/)
918 {
919  ConfigurationManager::loadTableGroup(groupName,
920  groupKey,
921  doActivate,
922  groupMembers,
923  progressBar,
924  accumulatedWarnings,
925  groupComment,
926  groupAuthor,
927  groupCreateTime,
928  doNotLoadMembers,
929  groupTypeString,
930  groupAliases,
931  groupTypeToLoad,
932  ignoreVersionTracking);
933 
934  if(!groupMembers || !groupMembers->size() || !groupComment || groupKey.isInvalid() ||
935  !groupAuthor || !groupCreateTime)
936  return;
937 
938  //treat successfull load as latest group key
939  auto groupInfo = allGroupInfo_.find(groupName);
940  if(groupInfo == allGroupInfo_.end())
941  return; //ignore if no group info cache
942 
943  groupInfo->second.latestKey_ = groupKey;
944  groupInfo->second.latestKeyGroupComment_ = *groupComment;
945  groupInfo->second.latestKeyGroupAuthor_ = *groupAuthor;
946  groupInfo->second.latestKeyGroupCreationTime_ = *groupCreateTime;
947  if(groupTypeString) //assume unlikely to change types
948  groupInfo->second.latestKeyGroupTypeString_ = *groupTypeString;
949  groupInfo->second.latestKeyMemberMap_ = *groupMembers;
950 
951 } //end loadTableGroup() RW version
952 
953 //==============================================================================
955 void ConfigurationManagerRW::loadTableGroupThread(
956  ConfigurationManagerRW* cfgMgr,
957  std::string groupName,
958  ots::TableGroupKey groupKey,
959  std::shared_ptr<ots::GroupInfo> groupInfo,
960  std::shared_ptr<std::atomic<bool>> threadDone)
961 try
962 {
963  __COUTT__ << "Thread started... " << groupName << "(" << groupKey << ")" << __E__;
964 
965  groupInfo->latestKey_ = groupKey;
966  cfgMgr->loadTableGroup(groupName,
967  groupKey,
968  false /*doActivate*/,
969  &(groupInfo->latestKeyMemberMap_) /*groupMembers*/,
970  0 /*progressBar*/,
971  0 /*accumulateErrors*/,
972  &(groupInfo->latestKeyGroupComment_),
973  &(groupInfo->latestKeyGroupAuthor_),
974  &(groupInfo->latestKeyGroupCreationTime_),
975  true /*doNotLoadMember*/,
976  &(groupInfo->latestKeyGroupTypeString_));
977 
978  *(threadDone) = true;
979 } // end loadTableGroupThread
980 catch(...)
981 {
982  __COUT_WARN__ << "Error occurred loading latest group info into cache for '"
983  << groupName << "(" << groupInfo->latestKey_ << ")'..." << __E__;
984  groupInfo->latestKey_ = TableGroupKey::INVALID;
985  groupInfo->latestKeyGroupComment_ = ConfigurationManager::UNKNOWN_INFO;
986  groupInfo->latestKeyGroupAuthor_ = ConfigurationManager::UNKNOWN_INFO;
987  groupInfo->latestKeyGroupCreationTime_ = ConfigurationManager::UNKNOWN_TIME;
988  groupInfo->latestKeyGroupTypeString_ = ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
989  groupInfo->latestKeyMemberMap_ = {};
990  *(threadDone) = true;
991 } // end loadTableGroupThread catch
992 
993 //==============================================================================
997 void ConfigurationManagerRW::compareTableGroupThread(
998  ConfigurationManagerRW* cfgMgr,
999  std::string groupName,
1000  ots::TableGroupKey groupKeyToCompare,
1001  const std::map<std::string, TableVersion>& groupMemberMap,
1002  const std::map<std::string /*name*/, std::string /*alias*/>& memberTableAliases,
1003  std::atomic<bool>* foundIdentical,
1004  ots::TableGroupKey* identicalKey,
1005  std::mutex* threadMutex,
1006  std::shared_ptr<std::atomic<bool>> threadDone)
1007 try
1008 {
1009  std::map<std::string /*name*/, TableVersion /*version*/> compareToMemberMap;
1010  std::map<std::string /*name*/, std::string /*alias*/> compareToMemberTableAliases;
1011  std::map<std::string /*name*/, std::string /*alias*/>*
1012  compareToMemberTableAliasesPtr = nullptr;
1013  if(memberTableAliases
1014  .size()) //only give pointer if necessary, without will load group faster
1015  compareToMemberTableAliasesPtr = &compareToMemberTableAliases;
1016 
1017  cfgMgr->loadTableGroup(groupName,
1018  groupKeyToCompare,
1019  false /*doActivate*/,
1020  &compareToMemberMap /*memberMap*/,
1021  0, /*progressBar*/
1022  0, /*accumulatedWarnings*/
1023  0, /*groupComment*/
1024  0,
1025  0, /*null pointers*/
1026  true /*doNotLoadMember*/,
1027  0 /*groupTypeString*/,
1028  compareToMemberTableAliasesPtr);
1029 
1030  bool debug = false;
1031  if(TTEST(9)) // = DEBUG+9
1032  {
1033  debug = true;
1034  for(auto& memberPair : groupMemberMap)
1035  __COUTS__(9) << "member " << memberPair.first << " (" << memberPair.second
1036  << ")" << __E__;
1037  for(auto& memberPair : compareToMemberMap)
1038  __COUTS__(9) << "compare " << groupName << " (" << groupKeyToCompare
1039  << ") member:" << memberPair.first << " (" << memberPair.second
1040  << ")" << __E__;
1041  }
1042 
1043  bool isDifferent = false;
1044  for(auto& memberPair : groupMemberMap)
1045  {
1046  //groups can be different if Alias names are different
1047  // or if resolved Alias versions are different (for the current active backbone)
1048  if(memberTableAliases.find(memberPair.first) != memberTableAliases.end())
1049  {
1050  // handle this table as alias, not version
1051  if(compareToMemberTableAliases.find(memberPair.first) ==
1052  compareToMemberTableAliases.end() || // alias is missing
1053  memberTableAliases.at(memberPair.first) !=
1054  compareToMemberTableAliases.at(memberPair.first))
1055  { // then different
1056  isDifferent = true;
1057  if(debug)
1058  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1059  << ") on alias " << memberPair.first << __E__;
1060  break;
1061  }
1062  } // else check if compareTo group is using an alias for table
1063  else if(compareToMemberTableAliases.find(memberPair.first) !=
1064  compareToMemberTableAliases.end())
1065  {
1066  // then different
1067  isDifferent = true;
1068  if(debug)
1069  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1070  << ") on reverse alias " << memberPair.first << __E__;
1071  break;
1072 
1073  } // else handle as table version comparison
1074 
1075  //normal table version check
1076  if(compareToMemberMap.find(memberPair.first) ==
1077  compareToMemberMap.end() || // name is missing
1078  memberPair.second !=
1079  compareToMemberMap.at(memberPair.first)) // or version mismatch
1080  {
1081  // then different
1082  isDifferent = true;
1083  if(debug)
1084  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1085  << ") on mismatch " << memberPair.first << __E__;
1086  break;
1087  }
1088  } //end table version mismatch checking loop
1089 
1090  // check member size for exact match
1091  if(!isDifferent &&
1092  groupMemberMap.size() !=
1093  compareToMemberMap
1094  .size()) // different size, so not same (groupMemberMap is a subset of memberPairs)
1095  {
1096  isDifferent = true;
1097 
1098  if(debug)
1099  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare << ") on size "
1100  << __E__;
1101  }
1102 
1103  if(!isDifferent) //found an exact match!
1104  {
1105  *foundIdentical = true;
1106  __COUT__ << "=====> Found exact match with key: " << groupKeyToCompare << __E__;
1107 
1108  std::lock_guard<std::mutex> lock(*threadMutex);
1109  *identicalKey = groupKeyToCompare;
1110  }
1111 
1112  *(threadDone) = true;
1113 } // end compareTableGroupThread
1114 catch(...)
1115 {
1116  __COUT_WARN__ << "Error occurred comparing group '" << groupName << "("
1117  << groupKeyToCompare << ")'..." << __E__;
1118 
1119  *(threadDone) = true;
1120 } // end compareTableGroupThread catch
1121 
1122 //==============================================================================
1126 std::map<std::string /*table name*/,
1127  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
1128 ConfigurationManagerRW::getVersionAliases(void) const
1129 {
1130  std::map<std::string /*table name*/,
1131  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
1132  retMap = ConfigurationManager::getVersionAliases();
1133 
1134  // always have scratch alias for each table that has a scratch version
1135  // overwrite map entry if necessary
1136  if(!ConfigurationInterface::isVersionTrackingEnabled())
1137  for(const auto& tableInfo : allTableInfo_)
1138  for(const auto& version : tableInfo.second.versions_)
1139  if(version.isScratchVersion())
1140  retMap[tableInfo.first][ConfigurationManager::SCRATCH_VERSION_ALIAS] =
1141  TableVersion(TableVersion::SCRATCH);
1142 
1143  return retMap;
1144 } // end getVersionAliases()
1145 
1146 //==============================================================================
1150 void ConfigurationManagerRW::activateTableGroup(const std::string& tableGroupName,
1151  TableGroupKey tableGroupKey,
1152  std::string* accumulatedTreeErrors,
1153  std::string* groupTypeString)
1154 {
1155  try
1156  {
1157  loadTableGroup(tableGroupName,
1158  tableGroupKey,
1159  true, // loads and activates
1160  0, // no members needed
1161  0, // no progress bar
1162  accumulatedTreeErrors, // accumulate warnings or not
1163  0 /* groupComment */,
1164  0 /* groupAuthor */,
1165  0 /* groupCreateTime */,
1166  false /* doNotLoadMember */,
1167  groupTypeString);
1168  }
1169  catch(...)
1170  {
1171  __GEN_COUT_ERR__ << "There were errors, so de-activating group: "
1172  << tableGroupName << " (" << tableGroupKey << ")" << __E__;
1173  try // just in case any lingering pieces, let's deactivate
1174  {
1175  destroyTableGroup(tableGroupName, true);
1176  }
1177  catch(...)
1178  {
1179  }
1180  throw; // re-throw original exception
1181  }
1182 
1183  __GEN_COUT_INFO__ << "Updating persistent active groups to "
1184  << ConfigurationManager::ACTIVE_GROUPS_FILENAME << " ..." << __E__;
1185 
1186  __COUT_INFO__ << "Active Context table group: " << theContextTableGroup_ << "("
1187  << (theContextTableGroupKey_
1188  ? theContextTableGroupKey_->toString().c_str()
1189  : "-1")
1190  << ")" << __E__;
1191  __COUT_INFO__ << "Active Backbone table group: " << theBackboneTableGroup_ << "("
1192  << (theBackboneTableGroupKey_
1193  ? theBackboneTableGroupKey_->toString().c_str()
1194  : "-1")
1195  << ")" << __E__;
1196  __COUT_INFO__ << "Active Iterate table group: " << theIterateTableGroup_ << "("
1197  << (theIterateTableGroupKey_
1198  ? theIterateTableGroupKey_->toString().c_str()
1199  : "-1")
1200  << ")" << __E__;
1201  __COUT_INFO__ << "Active Configuration table group: " << theConfigurationTableGroup_
1202  << "("
1203  << (theConfigurationTableGroupKey_
1204  ? theConfigurationTableGroupKey_->toString().c_str()
1205  : "-1")
1206  << ")" << __E__;
1207 
1208  std::string fn = ConfigurationManager::ACTIVE_GROUPS_FILENAME;
1209  FILE* fp = fopen(fn.c_str(), "w");
1210  if(!fp)
1211  {
1212  __SS__ << "Fatal Error! Unable to open the file "
1213  << ConfigurationManager::ACTIVE_GROUPS_FILENAME
1214  << " for editing! Is there a permissions problem?" << __E__;
1215  __GEN_COUT_ERR__ << ss.str();
1216  __SS_THROW__;
1217  return;
1218  }
1219  fprintf(fp, "%s\n", theContextTableGroup_.c_str());
1220  fprintf(
1221  fp,
1222  "%s\n",
1223  theContextTableGroupKey_ ? theContextTableGroupKey_->toString().c_str() : "-1");
1224  fprintf(fp, "%s\n", theBackboneTableGroup_.c_str());
1225  fprintf(
1226  fp,
1227  "%s\n",
1228  theBackboneTableGroupKey_ ? theBackboneTableGroupKey_->toString().c_str() : "-1");
1229  fprintf(fp, "%s\n", theIterateTableGroup_.c_str());
1230  fprintf(
1231  fp,
1232  "%s\n",
1233  theIterateTableGroupKey_ ? theIterateTableGroupKey_->toString().c_str() : "-1");
1234  fprintf(fp, "%s\n", theConfigurationTableGroup_.c_str());
1235  fprintf(fp,
1236  "%s\n",
1237  theConfigurationTableGroupKey_
1238  ? theConfigurationTableGroupKey_->toString().c_str()
1239  : "-1");
1240  fclose(fp);
1241 
1242  // save last activated group
1243  {
1244  std::pair<std::string /*group name*/, TableGroupKey> activatedGroup(
1245  std::string(tableGroupName), tableGroupKey);
1246  if(theConfigurationTableGroupKey_ &&
1247  theConfigurationTableGroup_ == tableGroupName &&
1248  *theConfigurationTableGroupKey_ == tableGroupKey)
1249  {
1250  ConfigurationManager::saveGroupNameAndKey(
1251  activatedGroup,
1252  ConfigurationManager::LAST_ACTIVATED_CONFIG_GROUP_FILE,
1253  false /* appendMode */,
1254  username_);
1255  ConfigurationManager::saveGroupNameAndKey(
1256  activatedGroup,
1257  ConfigurationManager::ACTIVATED_CONFIGS_FILE,
1258  true /* appendMode */,
1259  username_);
1260  }
1261  else if(theContextTableGroupKey_ && theContextTableGroup_ == tableGroupName &&
1262  *theContextTableGroupKey_ == tableGroupKey)
1263  {
1264  ConfigurationManager::saveGroupNameAndKey(
1265  activatedGroup,
1266  ConfigurationManager::LAST_ACTIVATED_CONTEXT_GROUP_FILE,
1267  false /* appendMode */,
1268  username_);
1269  ConfigurationManager::saveGroupNameAndKey(
1270  activatedGroup,
1271  ConfigurationManager::ACTIVATED_CONTEXTS_FILE,
1272  true /* appendMode */,
1273  username_);
1274  }
1275  else if(theBackboneTableGroupKey_ && theBackboneTableGroup_ == tableGroupName &&
1276  *theBackboneTableGroupKey_ == tableGroupKey)
1277  {
1278  ConfigurationManager::saveGroupNameAndKey(
1279  activatedGroup,
1280  ConfigurationManager::LAST_ACTIVATED_BACKBONE_GROUP_FILE,
1281  false /* appendMode */,
1282  username_);
1283  ConfigurationManager::saveGroupNameAndKey(
1284  activatedGroup,
1285  ConfigurationManager::ACTIVATED_BACKBONES_FILE,
1286  true /* appendMode */,
1287  username_);
1288  }
1289  else if(theIterateTableGroupKey_ && theIterateTableGroup_ == tableGroupName &&
1290  *theIterateTableGroupKey_ == tableGroupKey)
1291  {
1292  ConfigurationManager::saveGroupNameAndKey(
1293  activatedGroup,
1294  ConfigurationManager::LAST_ACTIVATED_ITERATE_GROUP_FILE,
1295  false /* appendMode */,
1296  username_);
1297  ConfigurationManager::saveGroupNameAndKey(
1298  activatedGroup,
1299  ConfigurationManager::ACTIVATED_ITERATES_FILE,
1300  true /* appendMode */,
1301  username_);
1302  }
1303  } // end save last activated group
1304 
1305 } // end activateTableGroup()
1306 
1307 //==============================================================================
1311 TableVersion ConfigurationManagerRW::createTemporaryBackboneView(
1312  TableVersion sourceViewVersion)
1313 {
1314  __GEN_COUT_INFO__ << "Creating temporary backbone view from version "
1315  << sourceViewVersion << __E__;
1316 
1317  // find common available temporary version among backbone members
1318  TableVersion tmpVersion =
1319  TableVersion::getNextTemporaryVersion(); // get the default temporary version
1320  TableVersion retTmpVersion;
1321  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
1322  for(auto& name : backboneMemberNames)
1323  {
1324  retTmpVersion =
1325  ConfigurationManager::getTableByName(name)->getNextTemporaryVersion();
1326  if(retTmpVersion < tmpVersion)
1327  tmpVersion = retTmpVersion;
1328  }
1329 
1330  __GEN_COUT__ << "Common temporary backbone version found as " << tmpVersion << __E__;
1331 
1332  // create temporary views from source version to destination temporary version
1333  for(auto& name : backboneMemberNames)
1334  {
1335  retTmpVersion =
1336  getTableByName(name)->createTemporaryView(sourceViewVersion, tmpVersion);
1337  if(retTmpVersion != tmpVersion)
1338  {
1339  __SS__ << "Failure! Temporary view requested was " << tmpVersion
1340  << ". Mismatched temporary view created: " << retTmpVersion << __E__;
1341  __GEN_COUT_ERR__ << ss.str();
1342  __SS_THROW__;
1343  }
1344  }
1345 
1346  return tmpVersion;
1347 } // end createTemporaryBackboneView()
1348 
1349 //==============================================================================
1350 TableBase* ConfigurationManagerRW::getTableByName(const std::string& tableName)
1351 {
1352  if(nameToTableMap_.find(tableName) == nameToTableMap_.end())
1353  {
1354  if(tableName == ConfigurationManager::ARTDAQ_TOP_TABLE_NAME)
1355  {
1356  __GEN_COUT_WARN__
1357  << "Since target table was the artdaq top configuration level, "
1358  "attempting to help user by appending to core tables file: "
1359  << CORE_TABLE_INFO_FILENAME << __E__;
1360  FILE* fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "a");
1361  if(fp)
1362  {
1363  fprintf(fp, "\nARTDAQ/*");
1364  fclose(fp);
1365  }
1366  }
1367 
1368  __SS__ << "Table not found with name: " << tableName << __E__;
1369  size_t f;
1370  if((f = tableName.find(' ')) != std::string::npos)
1371  ss << "There was a space character found in the table name needle at "
1372  "position "
1373  << f << " in the string (was this intended?). " << __E__;
1374 
1375  ss << "\nIf you think this table should exist in the core set of tables, try "
1376  "running 'UpdateOTS.sh --tables' to update your tables, then relaunch ots."
1377  << __E__;
1378  ss << "\nTables must be defined at path $USER_DATA/TableInfo/ to exist in ots. "
1379  "Please verify your table definitions, and then restart ots."
1380  << __E__;
1381  __GEN_COUT_ERR__ << "\n" << ss.str();
1382  __SS_THROW__;
1383  }
1384  return nameToTableMap_[tableName];
1385 } // end getTableByName()
1386 
1387 //==============================================================================
1392 time_t ConfigurationManagerRW::getVersionCreationTime(const std::string& tableName,
1393  TableVersion version)
1394 {
1395  { // check process-wide cache first (only immutable persistent versions are cached)
1396  std::lock_guard<std::mutex> lock(versionCreationTimeCacheMutex_);
1397 
1398  auto tableIt = versionCreationTimeCache_.find(tableName);
1399  if(tableIt != versionCreationTimeCache_.end())
1400  {
1401  auto versionIt = tableIt->second.find(version);
1402  if(versionIt != tableIt->second.end())
1403  return versionIt->second;
1404  }
1405  }
1406 
1407  std::string localAccumulatedErrors;
1408  const auto loadStartTime = std::chrono::steady_clock::now();
1409  time_t creationTime = getVersionedTableByName(tableName,
1410  version,
1411  true /* looseColumnMatching */,
1412  &localAccumulatedErrors,
1413  false /* getRawData */,
1414  false /* touchLastAccessTime */)
1415  ->getView(version)
1416  .getCreationTime();
1417 
1418  double loadSec =
1419  std::chrono::duration<double>(std::chrono::steady_clock::now() - loadStartTime)
1420  .count();
1421  if(loadSec > 1.0)
1422  __GEN_COUT_WARN__ << "Slow creation time lookup: table '" << tableName
1423  << "' version v" << version << " load took " << loadSec << " s"
1424  << __E__;
1425 
1426  if(!version.isTemporaryVersion() && !version.isScratchVersion())
1427  {
1428  std::lock_guard<std::mutex> lock(versionCreationTimeCacheMutex_);
1429  versionCreationTimeCache_[tableName][version] = creationTime;
1430  }
1431 
1432  return creationTime;
1433 } // end getVersionCreationTime()
1434 
1435 //==============================================================================
1442 void ConfigurationManagerRW::preloadVersionCreationTimes(void)
1443 {
1444  const auto preloadStartTime = std::chrono::steady_clock::now();
1445 
1446  // Identify uncached versions grouped by table
1447  std::vector<std::pair<std::string, std::vector<TableVersion>>> groupedWork;
1448  size_t missingCount = 0;
1449  {
1450  std::lock_guard<std::mutex> lock(versionCreationTimeCacheMutex_);
1451 
1452  for(const auto& tableInfoPair : allTableInfo_)
1453  {
1454  auto tableIt = versionCreationTimeCache_.find(tableInfoPair.first);
1455 
1456  std::vector<TableVersion> missingVersions;
1457  for(const auto& version : tableInfoPair.second.versions_)
1458  {
1459  if(version.isTemporaryVersion() || version.isScratchVersion())
1460  continue;
1461  if(tableIt != versionCreationTimeCache_.end() &&
1462  tableIt->second.find(version) != tableIt->second.end())
1463  continue;
1464  missingVersions.push_back(version);
1465  }
1466  if(missingVersions.size())
1467  {
1468  missingCount += missingVersions.size();
1469  groupedWork.emplace_back(tableInfoPair.first, std::move(missingVersions));
1470  }
1471  }
1472  }
1473  if(groupedWork.empty())
1474  return;
1475 
1476  // Round-robin interleave so large tables are spread across all threads
1477  std::vector<std::pair<std::string, TableVersion>> flatWork;
1478  flatWork.reserve(missingCount);
1479  {
1480  size_t maxVersions = 0;
1481  for(const auto& gw : groupedWork)
1482  if(gw.second.size() > maxVersions)
1483  maxVersions = gw.second.size();
1484  for(size_t vi = 0; vi < maxVersions; ++vi)
1485  for(const auto& gw : groupedWork)
1486  if(vi < gw.second.size())
1487  flatWork.emplace_back(gw.first, gw.second[vi]);
1488  }
1489 
1490  __GEN_COUT__ << "preloadVersionCreationTimes() loading " << missingCount
1491  << " version creation times for " << groupedWork.size() << " tables..."
1492  << __E__;
1493 
1494  int numOfThreads = PROCESSOR_COUNT / 2;
1495  if(numOfThreads > (int)flatWork.size())
1496  numOfThreads = flatWork.size();
1497 
1498  if(numOfThreads < 2)
1499  {
1500  for(const auto& work : flatWork)
1501  {
1502  try
1503  {
1504  getVersionCreationTime(work.first, work.second);
1505  }
1506  catch(...)
1507  {
1508  __GEN_COUT__ << "Failed to get creation time for table '" << work.first
1509  << "' version v" << work.second << ", skipping." << __E__;
1510  }
1511  }
1512  }
1513  else
1514  {
1515  std::atomic<size_t> workIndex(0);
1516  std::vector<std::thread> threads;
1517  auto* iface = theInterface_;
1518 
1519  for(int i = 0; i < numOfThreads; ++i)
1520  threads.emplace_back([iface, &workIndex, &flatWork, this]() {
1521  std::map<std::string, TableBase*> localTables;
1522 
1523  try
1524  {
1525  size_t w;
1526  while((w = workIndex++) < flatWork.size())
1527  {
1528  const auto& tableName = flatWork[w].first;
1529  const auto& version = flatWork[w].second;
1530 
1531  try
1532  {
1533  TableBase*& table = localTables[tableName];
1534  if(!table)
1535  {
1536  std::string localAccumulatedErrors;
1537  table = new TableBase(tableName, &localAccumulatedErrors);
1538  }
1539 
1540  std::string localAccumulatedErrors;
1541  iface->get(table,
1542  tableName,
1543  0 /* groupKey */,
1544  0 /* groupName */,
1545  false /* dontFill */,
1546  version,
1547  true /* resetConfiguration */,
1548  true /* looseColumnMatching */,
1549  false /* rawDataOnly */,
1550  &localAccumulatedErrors,
1551  false /* touchLastAccessTime */);
1552 
1553  time_t creationTime =
1554  table->getView(version).getCreationTime();
1555 
1556  {
1557  std::lock_guard<std::mutex> lock(
1558  versionCreationTimeCacheMutex_);
1559  versionCreationTimeCache_[tableName][version] =
1560  creationTime;
1561  }
1562  }
1563  catch(...)
1564  {
1565  __GEN_COUT__ << "Failed to get creation time for table '"
1566  << tableName << "' version v" << version
1567  << ", skipping." << __E__;
1568  }
1569  }
1570  }
1571  catch(...)
1572  {
1573  __GEN_COUT_ERR__ << "Unexpected error in preload thread." << __E__;
1574  }
1575 
1576  for(auto& pair : localTables)
1577  if(pair.second)
1578  delete pair.second;
1579  });
1580 
1581  for(auto& thread : threads)
1582  thread.join();
1583  }
1584 
1585  __GEN_COUT__ << "preloadVersionCreationTimes() loaded " << missingCount
1586  << " version creation times with " << numOfThreads << " thread(s) in "
1587  << std::chrono::duration<double>(std::chrono::steady_clock::now() -
1588  preloadStartTime)
1589  .count()
1590  << " s" << __E__;
1591 } // end preloadVersionCreationTimes()
1592 
1593 //==============================================================================
1600 time_t ConfigurationManagerRW::getVersionLastAccessTime(const std::string& tableName,
1601  TableVersion version)
1602 {
1603  auto it = nameToTableMap_.find(tableName);
1604  if(it == nameToTableMap_.end() || !it->second->isStored(version))
1605  return 0; // never loaded (or evicted from cache) by this process
1606 
1607  return it->second->getView(version).getLastAccessTime();
1608 } // end getVersionLastAccessTime()
1609 
1610 //==============================================================================
1613 TableVersion ConfigurationManagerRW::saveNewTable(const std::string& tableName,
1614  TableVersion temporaryVersion,
1615  bool makeTemporary) //,
1617 {
1618  TableVersion newVersion(temporaryVersion);
1619 
1620  // set author of version
1621  TableBase* table = getTableByName(tableName);
1622  table->getTemporaryView(temporaryVersion)->setAuthor(username_);
1623  // NOTE: author is assigned to permanent versions when saved to DBI
1624 
1625  if(!makeTemporary) // saveNewVersion makes the new version the active version
1626  newVersion = theInterface_->saveNewVersion(table, temporaryVersion);
1627  else // make the temporary version active
1628  table->setActiveView(newVersion);
1629 
1630  // if there is a problem, try to recover
1631  while(!makeTemporary && !newVersion.isScratchVersion() &&
1632  allTableInfo_[tableName].versions_.find(newVersion) !=
1633  allTableInfo_[tableName].versions_.end())
1634  {
1635  __GEN_COUT_ERR__
1636  << "What happenened!?? ERROR::: new persistent version v" << newVersion
1637  << " already exists!? How is it possible? Retrace your steps and "
1638  "tell an admin."
1639  << __E__;
1640 
1641  // create a new temporary version of the target view
1642  temporaryVersion = table->createTemporaryView(newVersion);
1643 
1644  if(newVersion.isTemporaryVersion())
1645  newVersion = temporaryVersion;
1646  else
1647  newVersion = TableVersion::getNextVersion(newVersion);
1648 
1649  __GEN_COUT_WARN__ << "Attempting to recover and use v" << newVersion << __E__;
1650 
1651  if(!makeTemporary) // saveNewVersion makes the new version the active version
1652  newVersion =
1653  theInterface_->saveNewVersion(table, temporaryVersion, newVersion);
1654  else // make the temporary version active
1655  table->setActiveView(newVersion);
1656  }
1657 
1658  if(newVersion.isInvalid())
1659  {
1660  __SS__ << "Something went wrong saving the new version v" << newVersion
1661  << ". What happened?! (duplicates? database error?)" << __E__;
1662  __GEN_COUT_ERR__ << "\n" << ss.str();
1663  __SS_THROW__;
1664  }
1665 
1666  // update allTableInfo_ with the new version
1667  allTableInfo_[tableName].versions_.insert(newVersion);
1668 
1669  // table->getView().print();
1670  return newVersion;
1671 } // end saveNewTable()
1672 
1673 //==============================================================================
1678 void ConfigurationManagerRW::eraseTemporaryVersion(const std::string& tableName,
1679  TableVersion targetVersion)
1680 {
1681  TableBase* table = getTableByName(tableName);
1682 
1683  table->trimTemporary(targetVersion);
1684 
1685  // if allTableInfo_ is not setup, then done
1686  if(allTableInfo_.find(tableName) == allTableInfo_.end())
1687  return;
1688  // else cleanup table info
1689 
1690  if(targetVersion.isInvalid())
1691  {
1692  // erase all temporary versions!
1693  for(auto it = allTableInfo_[tableName].versions_.begin();
1694  it != allTableInfo_[tableName].versions_.end();
1695  /*no increment*/)
1696  {
1697  if(it->isTemporaryVersion())
1698  {
1699  __GEN_COUT__ << "Removing '" << tableName << "' version info: " << *it
1700  << __E__;
1701  allTableInfo_[tableName].versions_.erase(it++);
1702  }
1703  else
1704  ++it;
1705  }
1706  }
1707  else // erase target version only
1708  {
1709  //__GEN_COUT__ << "Removing '" << tableName << "' version info: " << targetVersion << __E__;
1710  auto it = allTableInfo_[tableName].versions_.find(targetVersion);
1711  if(it == allTableInfo_[tableName].versions_.end())
1712  {
1713  __GEN_COUT__ << "Target '" << tableName << "' version v" << targetVersion
1714  << " was not found in info versions..." << __E__;
1715  return;
1716  }
1717  allTableInfo_[tableName].versions_.erase(
1718  allTableInfo_[tableName].versions_.find(targetVersion));
1719  }
1720 } // end eraseTemporaryVersion()
1721 
1722 //==============================================================================
1727 void ConfigurationManagerRW::clearCachedVersions(const std::string& tableName)
1728 {
1729  TableBase* table = getTableByName(tableName);
1730 
1731  table->trimCache(0);
1732 } // end clearCachedVersions()
1733 
1734 //==============================================================================
1739 void ConfigurationManagerRW::clearAllCachedVersions()
1740 {
1741  for(auto configInfo : allTableInfo_)
1742  configInfo.second.tablePtr_->trimCache(0);
1743 } // end clearAllCachedVersions()
1744 
1745 //==============================================================================
1747 TableVersion ConfigurationManagerRW::copyViewToCurrentColumns(
1748  const std::string& tableName, TableVersion sourceVersion)
1749 {
1750  getTableByName(tableName)->reset();
1751 
1752  // make sure source version is loaded
1753  // need to load with loose column rules!
1754  TableBase* table =
1755  getVersionedTableByName(tableName, TableVersion(sourceVersion), true);
1756 
1757  // copy from source version to a new temporary version
1758  TableVersion newTemporaryVersion =
1759  table->copyView(table->getView(), TableVersion(), username_);
1760 
1761  // update allTableInfo_ with the new version
1762  allTableInfo_[tableName].versions_.insert(newTemporaryVersion);
1763 
1764  return newTemporaryVersion;
1765 } // end copyViewToCurrentColumns()
1766 
1767 //==============================================================================
1771 const GroupInfo& ConfigurationManagerRW::getGroupInfo(
1772  const std::string& groupName, bool attemptToReloadKeys /* = false */)
1773 {
1774  // //NOTE: seems like this filter is taking the long amount of time
1775  // std::set<std::string /*name*/> fullGroupNames =
1776  // theInterface_->getAllTableGroupNames(groupName); //db filter by group name
1777 
1778  // so instead caching ourselves...
1779  auto it = allGroupInfo_.find(groupName);
1780  if(it == allGroupInfo_.end())
1781  {
1782  __SS__ << "Group name '" << groupName
1783  << "' not found in group info! (creating empty info)" << __E__;
1784  __GEN_COUT_WARN__ << ss.str();
1785  //__SS_THROW__;
1786  return allGroupInfo_[groupName];
1787  }
1788 
1789  if(attemptToReloadKeys) //load keys from Interface group cache
1790  {
1791  __GEN_COUT__ << "Reloading keys from special db group cache if it exists..."
1792  << __E__;
1793 
1794  std::set<TableGroupKey> keys;
1795  { //load keys from special db group cache (this avoids pre-cache filling and avoids long db lookup, unless speed table cache missing for this group)
1796  //attempt to use cache first! (potentially way faster .04 s vs 4 s)
1797  bool cacheFailed = false;
1798  try
1799  {
1800  TableBase localGroupMemberCacheLoader(
1801  true /*special table*/
1802  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),,
1803  TableBase::GROUP_CACHE_PREPEND + groupName);
1804  auto versions = theInterface_->getVersions(&localGroupMemberCacheLoader);
1805  for(const auto& version : versions)
1806  keys.emplace(TableGroupKey(version.version()));
1807  }
1808  catch(...)
1809  {
1810  __GEN_COUT__ << "Ignoring cache loading error. Doing full load of keys..."
1811  << __E__;
1812  cacheFailed = true;
1813  }
1814 
1815  if(cacheFailed && 0) //could consider full load if cache failed
1816  keys = theInterface_->getKeys(groupName);
1817 
1818  if(!cacheFailed) //take keys
1819  {
1820  __GEN_COUT__ << "Key from special db group cache were loaded." << __E__;
1821  it->second.keys_ = keys; //update ConfigManager cache!
1822  }
1823  }
1824  }
1825 
1826  return it->second;
1827 } // end getGroupInfo()
1828 
1829 //==============================================================================
1839 TableGroupKey ConfigurationManagerRW::findTableGroup(
1840  const std::string& groupName,
1841  const std::map<std::string, TableVersion>& groupMemberMap,
1842  const std::map<std::string /*name*/, std::string /*alias*/>& memberTableAliases)
1843 {
1844  if(!groupMemberMap.size() || groupName.empty())
1845  {
1846  __SS__ << "Illegal name/members for requested group of name '" << groupName
1847  << "' and member count = " << groupMemberMap.size() << __E__;
1848  __SS_THROW__;
1849  }
1850 
1851  // //NOTE: seems like this filter is taking the long amount of time
1852  // std::set<std::string /*name*/> fullGroupNames =
1853  // theInterface_->getAllTableGroupNames(groupName); //db filter bygroup name
1854  // const GroupInfo& groupInfo = getGroupInfo(groupName); // Note this also seems to take too long because requires a pre-cache load!
1855  std::set<TableGroupKey> keys;
1856  { //so instead load keys from special db group cache (this avoids pre-cache filling and avoids long db lookup, unless speed table cache missing for this group)
1857  //attempt to use cache first! (potentially way faster .04 s vs 4 s)
1858  bool cacheFailed = false;
1859  try
1860  {
1861  TableBase localGroupMemberCacheLoader(
1862  true /*special table*/
1863  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),,
1864  TableBase::GROUP_CACHE_PREPEND + groupName);
1865  auto versions = theInterface_->getVersions(&localGroupMemberCacheLoader);
1866  for(const auto& version : versions)
1867  keys.emplace(TableGroupKey(version.version()));
1868  }
1869  catch(...)
1870  {
1871  __COUT__ << "Ignoring cache loading error. Doing full load of keys..."
1872  << __E__;
1873  cacheFailed = true;
1874  }
1875 
1876  if(cacheFailed) //since cache failed, do full load
1877  keys = theInterface_->getKeys(groupName);
1878  }
1879 
1880  __COUTTV__(StringMacros::setToString(keys));
1881 
1882  const unsigned int MAX_DEPTH_TO_CHECK = 20;
1883  unsigned int keyMinToCheck = 0;
1884 
1885  if(keys.size())
1886  keyMinToCheck = keys.rbegin()->key();
1887  if(keyMinToCheck > MAX_DEPTH_TO_CHECK)
1888  {
1889  keyMinToCheck -= MAX_DEPTH_TO_CHECK;
1890  __GEN_COUT__ << "Checking groups back to key... " << keyMinToCheck << __E__;
1891  }
1892  else
1893  {
1894  keyMinToCheck = 0;
1895  __GEN_COUT__ << "Checking all groups." << __E__;
1896  }
1897 
1898  __GEN_COUTTV__(StringMacros::mapToString(groupMemberMap));
1899 
1900  // have min key to check, now loop through and check groups
1901 
1902  const int numOfThreads = PROCESSOR_COUNT / 2;
1903  __GEN_COUT__ << " PROCESSOR_COUNT " << PROCESSOR_COUNT << " ==> " << numOfThreads
1904  << " threads." << __E__;
1905  if(numOfThreads < 2) // no multi-threading
1906  {
1907  std::map<std::string /*name*/, TableVersion /*version*/> compareToMemberMap;
1908  std::map<std::string /*name*/, std::string /*alias*/> compareToMemberTableAliases;
1909  std::map<std::string /*name*/, std::string /*alias*/>*
1910  compareToMemberTableAliasesPtr = nullptr;
1911  if(memberTableAliases.size())
1912  compareToMemberTableAliasesPtr = &compareToMemberTableAliases;
1913 
1914  bool isDifferent;
1915  for(const auto& key : keys)
1916  {
1917  if(key.key() < keyMinToCheck)
1918  continue; // skip keys that are too old
1919 
1920  loadTableGroup(groupName,
1921  key,
1922  false /*doActivate*/,
1923  &compareToMemberMap /*memberMap*/,
1924  0, /*progressBar*/
1925  0, /*accumulatedWarnings*/
1926  0, /*groupComment*/
1927  0, /*groupAuthor*/
1928  0, /*groupCreateTime*/
1929  true /*doNotLoadMember*/,
1930  0 /*groupTypeString*/,
1931  compareToMemberTableAliasesPtr);
1932 
1933  isDifferent = false;
1934  for(auto& memberPair : groupMemberMap)
1935  {
1936  if(memberTableAliases.find(memberPair.first) != memberTableAliases.end())
1937  {
1938  // handle this table as alias, not version
1939  if(compareToMemberTableAliases.find(memberPair.first) ==
1940  compareToMemberTableAliases.end() || // alias is missing
1941  memberTableAliases.at(memberPair.first) !=
1942  compareToMemberTableAliases.at(memberPair.first))
1943  { // then different
1944  isDifferent = true;
1945  break;
1946  }
1947  // FIXED alias matches, but still need to check version
1948  } // else check if compareTo group is using an alias for table
1949  else if(compareToMemberTableAliases.find(memberPair.first) !=
1950  compareToMemberTableAliases.end())
1951  {
1952  // then different
1953  isDifferent = true;
1954  break;
1955  }
1956 
1957  // alias check complete
1958  // handle table version comparison
1959 
1960  if(compareToMemberMap.find(memberPair.first) ==
1961  compareToMemberMap.end() || // name is missing
1962  memberPair.second !=
1963  compareToMemberMap.at(memberPair.first)) // or version mismatch
1964  {
1965  // then different
1966  isDifferent = true;
1967  break;
1968  }
1969  }
1970  if(isDifferent)
1971  continue;
1972 
1973  // check member size for exact match
1974  if(groupMemberMap.size() != compareToMemberMap.size())
1975  continue; // different size, so not same (groupMemberMap is a subset of
1976  // memberPairs)
1977 
1978  __GEN_COUT__ << "Found exact match with key: " << key << __E__;
1979  // else found an exact match!
1980  return key;
1981  }
1982  __GEN_COUT__ << "No match found - this group is new!" << __E__;
1983  // if here, then no match found
1984  return TableGroupKey(); // return invalid key
1985  }
1986  else //multi-threading
1987  {
1988  int threadsLaunched = 0;
1989  int foundThreadIndex = 0;
1990  std::atomic<bool> foundIdentical = false;
1991  ots::TableGroupKey identicalKey;
1992  std::mutex threadMutex;
1993 
1994  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
1995  for(int i = 0; i < numOfThreads; ++i)
1996  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
1997 
1998  for(const auto& key : keys)
1999  {
2000  if(foundIdentical)
2001  break;
2002  if(key.key() < keyMinToCheck)
2003  continue; // skip keys that are too old
2004 
2005  if(threadsLaunched >= numOfThreads)
2006  {
2007  //find availableThreadIndex
2008  foundThreadIndex = -1;
2009  while(foundThreadIndex == -1)
2010  {
2011  if(foundIdentical)
2012  break;
2013 
2014  for(int i = 0; i < numOfThreads; ++i)
2015  if(*(threadDone[i]))
2016  {
2017  foundThreadIndex = i;
2018  break;
2019  }
2020  if(foundThreadIndex == -1)
2021  {
2022  __GEN_COUTT__ << "Waiting for available thread..." << __E__;
2023  usleep(10000);
2024  }
2025  } //end thread search loop
2026  threadsLaunched = numOfThreads - 1;
2027  }
2028  if(foundIdentical)
2029  break;
2030 
2031  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex << __E__;
2032  *(threadDone[foundThreadIndex]) = false;
2033 
2034  std::thread(
2035  [](ConfigurationManagerRW* cfgMgr,
2036  std::string theGroupName,
2037  ots::TableGroupKey groupKeyToCompare,
2038  const std::map<std::string, TableVersion>& groupMemberMap,
2039  const std::map<std::string /*name*/, std::string /*alias*/>&
2040  memberTableAliases,
2041  std::atomic<bool>* theFoundIdentical,
2042  ots::TableGroupKey* theIdenticalKey,
2043  std::mutex* theThreadMutex,
2044  std::shared_ptr<std::atomic<bool>> theThreadDone) {
2045  ConfigurationManagerRW::compareTableGroupThread(cfgMgr,
2046  theGroupName,
2047  groupKeyToCompare,
2048  groupMemberMap,
2049  memberTableAliases,
2050  theFoundIdentical,
2051  theIdenticalKey,
2052  theThreadMutex,
2053  theThreadDone);
2054  },
2055  this,
2056  groupName,
2057  key,
2058  groupMemberMap,
2059  memberTableAliases,
2060  &foundIdentical,
2061  &identicalKey,
2062  &threadMutex,
2063  threadDone[foundThreadIndex])
2064  .detach();
2065 
2066  ++threadsLaunched;
2067  ++foundThreadIndex;
2068  } //end group key check thread loop
2069 
2070  //check for all threads done
2071  do
2072  {
2073  foundThreadIndex = -1;
2074  for(int i = 0; i < numOfThreads; ++i)
2075  if(!*(threadDone[i]))
2076  {
2077  foundThreadIndex = i;
2078  break;
2079  }
2080  if(foundThreadIndex != -1)
2081  {
2082  __GEN_COUTT__ << "Waiting for thread to finish... " << foundThreadIndex
2083  << __E__;
2084  usleep(10000);
2085  }
2086  } while(foundThreadIndex != -1); //end thread done search loop
2087 
2088  if(foundIdentical)
2089  {
2090  __GEN_COUT__ << "Found exact match with key: " << identicalKey << __E__;
2091  return identicalKey;
2092  }
2093 
2094  // if here, then no match found
2095  return TableGroupKey(); // return invalid key
2096  } //end multi-thread handling
2097 } // end findTableGroup()
2098 
2099 //==============================================================================
2104 TableBase* ConfigurationManagerRW::getMetadataTable(
2105  TableVersion fillVersion /* = TableVersion()*/)
2106 {
2107  if(fillVersion.isInvalid())
2108  return &groupMetadataTable_;
2109  //else load specified fill version
2110 
2111  //only lock metadata table since it is shared by all group accesses
2112  std::lock_guard<std::mutex> lock(metaDataTableMutex_);
2113 
2114  // clear table
2115  while(groupMetadataTable_.getView().getNumberOfRows())
2116  groupMetadataTable_.getViewP()->deleteRow(0);
2117 
2118  // retrieve metadata from database
2119  try
2120  {
2121  theInterface_->fill(&groupMetadataTable_, fillVersion);
2122  }
2123  catch(const std::runtime_error& e)
2124  {
2125  __GEN_COUT_WARN__ << "Failed to load " << groupMetadataTable_.getTableName()
2126  << "-v" << fillVersion << ". Metadata error: " << e.what()
2127  << __E__;
2128  }
2129  catch(...)
2130  {
2131  __GEN_COUT_WARN__ << "Failed to load " << groupMetadataTable_.getTableName()
2132  << "-v" << fillVersion << ". Ignoring unknown metadata error. "
2133  << __E__;
2134  }
2135 
2136  // check that there is only 1 row
2137  if(groupMetadataTable_.getView().getNumberOfRows() != 1)
2138  {
2139  groupMetadataTable_.print();
2140  __GEN_COUT_ERR__ << "Ignoring that groupMetadataTable_ v" << fillVersion
2141  << " has wrong "
2142  "number of rows!' Must "
2143  "be 1. Going with anonymous defaults."
2144  << __E__;
2145 
2146  // fix metadata table
2147  while(groupMetadataTable_.getViewP()->getNumberOfRows() > 1)
2148  groupMetadataTable_.getViewP()->deleteRow(0);
2149  if(groupMetadataTable_.getViewP()->getNumberOfRows() == 0)
2150  groupMetadataTable_.getViewP()->addRow();
2151  }
2152 
2153  return &groupMetadataTable_;
2154 } // end getMetadataTable()
2155 
2156 //==============================================================================
2163 TableGroupKey ConfigurationManagerRW::saveNewTableGroup(
2164  const std::string& groupName,
2165  std::map<std::string, TableVersion>& groupMembers,
2166  const std::string& groupComment,
2167  std::map<std::string /*table*/, std::string /*alias*/>* groupAliases)
2168 {
2169  // steps:
2170  // determine new group key
2171  // verify group members
2172  // verify groupNameWithKey
2173  // verify store
2174 
2175  if(groupMembers.size() == 0) // do not allow empty groups
2176  {
2177  __SS__ << "Empty group member list. Can not create a group without members!"
2178  << __E__;
2179  __SS_THROW__;
2180  }
2181 
2182  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2183 
2184  // verify group members
2185  // - use all table info
2186  std::map<std::string, TableInfo> allCfgInfo = getAllTableInfo();
2187  for(auto& memberPair : groupMembers)
2188  {
2189  // check member name
2190  if(allCfgInfo.find(memberPair.first) == allCfgInfo.end())
2191  {
2192  __GEN_COUT_ERR__ << "Group member \"" << memberPair.first
2193  << "\" not found in database!";
2194 
2195  if(groupMetadataTable_.getTableName() == memberPair.first)
2196  {
2197  __GEN_COUT_WARN__
2198  << "Looks like this is the groupMetadataTable_ '"
2199  << TableBase::GROUP_METADATA_TABLE_NAME
2200  << ".' Note that this table is added to the member map when groups "
2201  "are saved."
2202  << "It should not be part of member map when calling this function."
2203  << __E__;
2204  __GEN_COUT__ << "Attempting to recover." << __E__;
2205  groupMembers.erase(groupMembers.find(memberPair.first));
2206  }
2207  else
2208  {
2209  __SS__ << ("Group member not found!") << __E__;
2210  __SS_THROW__;
2211  }
2212  }
2213  // check member version
2214  if(allCfgInfo[memberPair.first].versions_.find(memberPair.second) ==
2215  allCfgInfo[memberPair.first].versions_.end())
2216  {
2217  __SS__ << "Group member \"" << memberPair.first << "\" version \""
2218  << memberPair.second << "\" not found in database!";
2219  __SS_THROW__;
2220  }
2221  } // end verify members
2222 
2223  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2224 
2225  // verify group aliases
2226  if(groupAliases)
2227  {
2228  for(auto& aliasPair : *groupAliases)
2229  {
2230  // check for alias table in member names
2231  if(groupMembers.find(aliasPair.first) == groupMembers.end())
2232  {
2233  __GEN_COUT_ERR__ << "Group member \"" << aliasPair.first
2234  << "\" not found in group member map!";
2235 
2236  __SS__ << ("Alias table not found in member list!") << __E__;
2237  __SS_THROW__;
2238  }
2239  }
2240  } // end verify group aliases
2241 
2242  TableGroupKey newKey =
2243  TableGroupKey::getNextKey(theInterface_->findLatestGroupKey(groupName));
2244  __GEN_COUT__ << "New Key for group: " << groupName << " found as " << newKey << __E__;
2245  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2246 
2247  time_t groupCreationTime = time(0);
2248  // capture group type before adding metadata table!
2249  std::string groupType = getTypeNameOfGroup(groupMembers);
2250  std::map<std::string /*name*/, TableVersion /*version*/> groupMembersWithoutMeta =
2251  groupMembers;
2252 
2253  // verify groupNameWithKey and attempt to store
2254  try
2255  {
2256  // save meta data for group; reuse groupMetadataTable_
2257  std::string groupAliasesString = "";
2258  if(groupAliases)
2259  groupAliasesString = StringMacros::mapToString(
2260  *groupAliases, "," /*primary delimeter*/, ":" /*secondary delimeter*/);
2261  __GEN_COUT__ << "Metadata: " << username_ << " " << groupCreationTime << " "
2262  << groupComment << " " << groupAliasesString << " " << groupType
2263  << __E__;
2264 
2265  // to compensate for unusual errors upstream, make sure the metadata table has one
2266  // row
2267  while(groupMetadataTable_.getViewP()->getNumberOfRows() > 1)
2268  groupMetadataTable_.getViewP()->deleteRow(0);
2269  if(groupMetadataTable_.getViewP()->getNumberOfRows() == 0)
2270  groupMetadataTable_.getViewP()->addRow();
2271 
2272  // columns are uid,comment,author,time
2273  groupMetadataTable_.getViewP()->setValue(
2274  groupAliasesString, 0, ConfigurationManager::METADATA_COL_ALIASES);
2275  groupMetadataTable_.getViewP()->setValue(
2276  groupComment, 0, ConfigurationManager::METADATA_COL_COMMENT);
2277  groupMetadataTable_.getViewP()->setValue(
2278  username_, 0, ConfigurationManager::METADATA_COL_AUTHOR);
2279  groupMetadataTable_.getViewP()->setValue(
2280  groupCreationTime, 0, ConfigurationManager::METADATA_COL_TIMESTAMP);
2281 
2282  if(TTEST(2))
2283  {
2284  std::stringstream ss;
2285  groupMetadataTable_.print(ss);
2286  __COUT_MULTI__(2, ss.str());
2287  }
2288 
2289  // save table, and retry on save collision
2290  {
2291  // set version to first available persistent version
2292  TableVersion newVersion = TableVersion::getNextVersion(
2293  theInterface_->findLatestVersion(&groupMetadataTable_));
2294  groupMetadataTable_.getViewP()->setVersion(newVersion);
2295 
2296  uint16_t retries = 0;
2297  while(1)
2298  {
2299  try
2300  {
2301  theInterface_->saveActiveVersion(&groupMetadataTable_);
2302  }
2303  catch(const std::runtime_error& e)
2304  {
2305  __GEN_COUT__ << "Caught runtime_error exception during table save."
2306  << __E__;
2307  if(std::string(e.what()).find("there was a collision") !=
2308  std::string::npos)
2309  {
2310  __GEN_COUT_WARN__
2311  << "There was a collision saving the new table "
2312  << groupMetadataTable_ << "(" << newVersion
2313  << "), trying incremented table version... retries="
2314  << retries << __E__;
2315  if(++retries > 3) //give up
2316  throw;
2317  newVersion = TableVersion::getNextVersion(
2318  newVersion); //increment table version
2319  groupMetadataTable_.getViewP()->setVersion(newVersion);
2320  __GEN_COUT__ << "New version for table: " << groupMetadataTable_
2321  << " found as " << newVersion << __E__;
2322  continue;
2323  }
2324  else
2325  throw;
2326  }
2327 
2328  __GEN_COUT__ << "Created table: " << groupMetadataTable_ << "-v"
2329  << newVersion << __E__;
2330  break;
2331  } //end collission retry loop
2332  }
2333 
2334  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2335  << __E__;
2336 
2337  // force groupMetadataTable_ to be a member for the group
2338  groupMembers[groupMetadataTable_.getTableName()] =
2339  groupMetadataTable_.getViewVersion();
2340 
2341  // save group, and retry on save collision
2342  {
2343  uint16_t retries = 0;
2344  while(1)
2345  {
2346  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2347  << __E__;
2348 
2349  try
2350  {
2351  theInterface_->saveTableGroup(
2352  groupMembers,
2353  TableGroupKey::getFullGroupString(groupName, newKey));
2354  }
2355  catch(const std::runtime_error& e)
2356  {
2357  __GEN_COUT__ << "Caught runtime_error exception during group save."
2358  << __E__;
2359  if(std::string(e.what()).find("there was a collision") !=
2360  std::string::npos)
2361  {
2362  __GEN_COUT_WARN__
2363  << "There was a collision saving the new group " << groupName
2364  << "(" << newKey
2365  << "), trying incremented group key... retries=" << retries
2366  << __E__;
2367  if(++retries > 3) //give up
2368  throw;
2369  newKey = TableGroupKey::getNextKey(newKey); //increment group key
2370  __GEN_COUT__ << "New Key for group: " << groupName << " found as "
2371  << newKey << __E__;
2372  continue;
2373  }
2374  else
2375  throw;
2376  }
2377 
2378  __GEN_COUT__ << "Created table group: " << groupName << "(" << newKey
2379  << ")" << __E__;
2380  break;
2381  } //end collission retry loop
2382  }
2383 
2384  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2385  << __E__;
2386  }
2387  catch(std::runtime_error& e)
2388  {
2389  __GEN_COUT_ERR__ << "Failed to create table group: " << groupName << "(" << newKey
2390  << ")" << __E__;
2391  __GEN_COUT_ERR__ << "\n\n" << e.what() << __E__;
2392  throw;
2393  }
2394  catch(...)
2395  {
2396  __GEN_COUT_ERR__ << "Failed to create table group: " << groupName << ":" << newKey
2397  << __E__;
2398  throw;
2399  }
2400 
2401  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2402 
2403  // store cache of recent groups
2404  allGroupInfo_[groupName].keys_.emplace(newKey);
2405  //update latest group info with this group's info
2406  allGroupInfo_.at(groupName).latestKey_ = newKey;
2407  allGroupInfo_.at(groupName).latestKeyGroupAuthor_ = username_;
2408  allGroupInfo_.at(groupName).latestKeyGroupComment_ = groupComment;
2409  allGroupInfo_.at(groupName).latestKeyGroupCreationTime_ = groupCreationTime;
2410  allGroupInfo_.at(groupName).latestKeyGroupTypeString_ = groupType;
2411  allGroupInfo_.at(groupName).latestKeyMemberMap_ = groupMembersWithoutMeta;
2412 
2413  __GEN_COUT__ << "Saved " << groupName << "(" << newKey << ") of type " << groupType
2414  << __E__;
2415 
2416  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2417 
2418  // at this point succeeded!
2419  return newKey;
2420 } // end saveNewTableGroup()
2421 
2422 //==============================================================================
2426 TableVersion ConfigurationManagerRW::saveNewBackbone(TableVersion temporaryVersion)
2427 {
2428  __GEN_COUT_INFO__ << "Creating new backbone from temporary version "
2429  << temporaryVersion << __E__;
2430 
2431  // find common available temporary version among backbone members
2432  TableVersion newVersion(TableVersion::DEFAULT);
2433  TableVersion retNewVersion;
2434  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
2435  for(auto& name : backboneMemberNames)
2436  {
2437  retNewVersion = ConfigurationManager::getTableByName(name)->getNextVersion();
2438  __GEN_COUT__ << "New version for backbone member (" << name
2439  << "): " << retNewVersion << __E__;
2440  if(retNewVersion > newVersion)
2441  newVersion = retNewVersion;
2442  }
2443 
2444  __GEN_COUT__ << "Common new backbone version found as " << newVersion << __E__;
2445 
2446  // create new views from source temporary version
2447  for(auto& name : backboneMemberNames)
2448  {
2449  // saveNewVersion makes the new version the active version
2450  retNewVersion = getConfigurationInterface()->saveNewVersion(
2451  getTableByName(name), temporaryVersion, newVersion);
2452  if(retNewVersion != newVersion)
2453  {
2454  __SS__ << "Failure! New view requested was " << newVersion
2455  << ". Mismatched new view created: " << retNewVersion << __E__;
2456  __GEN_COUT_ERR__ << ss.str();
2457  __SS_THROW__;
2458  }
2459  }
2460 
2461  return newVersion;
2462 } // end saveNewBackbone()
2463 
2464 //==============================================================================
2469 TableVersion ConfigurationManagerRW::saveModifiedVersion(
2470  const std::string& tableName,
2471  TableVersion originalVersion,
2472  bool makeTemporary,
2473  TableBase* table,
2474  TableVersion temporaryModifiedVersion,
2475  bool ignoreDuplicates /*= false*/,
2476  bool lookForEquivalent /*= false*/,
2477  bool* foundEquivalent /*= nullptr*/)
2478 {
2479  bool needToEraseTemporarySource =
2480  (originalVersion.isTemporaryVersion() && !makeTemporary);
2481 
2482  if(foundEquivalent)
2483  *foundEquivalent = false; // initialize
2484 
2485  // check for duplicate tables already in cache, plus highest version numbers not in cache
2486  if(!ignoreDuplicates)
2487  {
2488  __GEN_COUT__ << "Checking for duplicate '" << tableName << "' tables..." << __E__;
2489 
2490  TableVersion duplicateVersion;
2491 
2492  {
2493  //"DEEP" checking
2494  // load into cache 'recent' versions for this table
2495  // 'recent' := those already in cache, plus highest version numbers not in cache
2496  const std::map<std::string, TableInfo>& allTableInfo =
2497  getAllTableInfo(); // do not refresh
2498 
2499  auto versionReverseIterator =
2500  allTableInfo.at(tableName).versions_.rbegin(); // get reverse iterator
2501  __GEN_COUT__ << "Filling up '" << tableName << "' cache from "
2502  << table->getNumberOfStoredViews() << " to max count of "
2503  << table->MAX_VIEWS_IN_CACHE << __E__;
2504  for(; table->getNumberOfStoredViews() < table->MAX_VIEWS_IN_CACHE &&
2505  versionReverseIterator != allTableInfo.at(tableName).versions_.rend();
2506  ++versionReverseIterator)
2507  {
2508  __GEN_COUTT__ << "'" << tableName << "' versions in reverse order "
2509  << *versionReverseIterator << __E__;
2510  try
2511  {
2512  getVersionedTableByName(tableName,
2513  *versionReverseIterator); // load to cache
2514  }
2515  catch(const std::runtime_error& e)
2516  {
2517  // ignore error
2518  __COUTT__ << "'" << tableName
2519  << "' version failed to load: " << *versionReverseIterator
2520  << __E__;
2521  }
2522  }
2523  }
2524 
2525  __GEN_COUT__ << "Checking '" << tableName << "' for duplicate..." << __E__;
2526 
2527  duplicateVersion = table->checkForDuplicate(
2528  temporaryModifiedVersion,
2529  (!originalVersion.isTemporaryVersion() && !makeTemporary)
2530  ? TableVersion()
2531  : // if from persistent to persistent, then include original version in search
2532  originalVersion);
2533 
2534  if(lookForEquivalent && !duplicateVersion.isInvalid())
2535  {
2536  // found an equivalent!
2537  __GEN_COUT__ << "Equivalent '" << tableName << "' table found in version v"
2538  << duplicateVersion << __E__;
2539 
2540  // if duplicate version was temporary, do not use
2541  if(duplicateVersion.isTemporaryVersion() && !makeTemporary)
2542  {
2543  __GEN_COUT__ << "Need persistent. Duplicate '" << tableName
2544  << "' version was temporary. "
2545  "Abandoning duplicate."
2546  << __E__;
2547  duplicateVersion = TableVersion(); // set invalid
2548  }
2549  else
2550  {
2551  // erase and return equivalent version
2552 
2553  // erase modified equivalent version
2554  eraseTemporaryVersion(tableName, temporaryModifiedVersion);
2555 
2556  // erase original if needed
2557  if(needToEraseTemporarySource)
2558  eraseTemporaryVersion(tableName, originalVersion);
2559 
2560  if(foundEquivalent)
2561  *foundEquivalent = true;
2562 
2563  __GEN_COUT__ << "\t\t Equivalent '" << tableName
2564  << "' assigned version: " << duplicateVersion << __E__;
2565 
2566  return duplicateVersion;
2567  }
2568  }
2569 
2570  if(!duplicateVersion.isInvalid())
2571  {
2572  __SS__ << "This version of table '" << tableName
2573  << "' is identical to another version currently cached v"
2574  << duplicateVersion << ". No reason to save a duplicate." << __E__;
2575  __GEN_COUT_ERR__ << "\n" << ss.str();
2576 
2577  // delete temporaryModifiedVersion
2578  table->eraseView(temporaryModifiedVersion);
2579  __SS_THROW__;
2580  }
2581 
2582  __GEN_COUT__ << "Check for duplicate '" << tableName << "' tables complete."
2583  << __E__;
2584  }
2585 
2586  if(makeTemporary)
2587  __GEN_COUT__ << "\t\t**************************** Save as temporary '"
2588  << tableName << "' table version" << __E__;
2589  else
2590  __GEN_COUT__ << "\t\t**************************** Save as new '" << tableName
2591  << "' table version" << __E__;
2592 
2593  TableVersion newAssignedVersion =
2594  saveNewTable(tableName, temporaryModifiedVersion, makeTemporary);
2595 
2596  __GEN_COUTTV__(table->getView().getComment());
2597 
2598  if(needToEraseTemporarySource)
2599  eraseTemporaryVersion(tableName, originalVersion);
2600 
2601  __GEN_COUT__ << "\t\t '" << tableName
2602  << "' new assigned version: " << newAssignedVersion << __E__;
2603  return newAssignedVersion;
2604 } // end saveModifiedVersion()
2605 
2606 //==============================================================================
2607 GroupEditStruct::GroupEditStruct(const ConfigurationManager::GroupType& groupType,
2608  ConfigurationManagerRW* cfgMgr)
2609  : groupType_(groupType)
2610  , originalGroupName_(cfgMgr->getActiveGroupName(groupType))
2611  , originalGroupKey_(cfgMgr->getActiveGroupKey(groupType))
2612  , cfgMgr_(cfgMgr)
2613  , mfSubject_(cfgMgr->getUsername())
2614 {
2615  if(originalGroupName_ == "" || originalGroupKey_.isInvalid())
2616  {
2617  __SS__ << "Error! No active group found for type '"
2618  << ConfigurationManager::convertGroupTypeToName(groupType)
2619  << ".' There must be an active group to edit the group." << __E__ << __E__
2620  << StringMacros::stackTrace() << __E__;
2621  __SS_THROW__;
2622  }
2623 
2624  __GEN_COUT__ << "Extracting Group-Edit Struct for type "
2625  << ConfigurationManager::convertGroupTypeToName(groupType) << __E__;
2626 
2627  std::map<std::string, TableVersion> activeTables = cfgMgr->getActiveVersions();
2628 
2629  const std::set<std::string>& memberNames =
2630  groupType == ConfigurationManager::GroupType::CONTEXT_TYPE
2631  ? cfgMgr->getActiveContextMemberNames()
2632  : (groupType == ConfigurationManager::GroupType::BACKBONE_TYPE
2633  ? ConfigurationManager::getBackboneMemberNames()
2634  : (groupType == ConfigurationManager::GroupType::ITERATE_TYPE
2635  ? ConfigurationManager::getIterateMemberNames()
2636  : cfgMgr->getConfigurationMemberNames()));
2637 
2638  for(auto& memberName : memberNames)
2639  try
2640  {
2641  groupMembers_.emplace(
2642  std::make_pair(memberName, activeTables.at(memberName)));
2643  groupTables_.emplace(std::make_pair(
2644  memberName,
2645  TableEditStruct(memberName, cfgMgr))); // Table ready for editing!
2646  }
2647  catch(...)
2648  {
2649  __GEN_COUTV__(StringMacros::mapToString(activeTables));
2650  __SS__ << "Error! Could not find group member table '" << memberName
2651  << "' for group type '"
2652  << ConfigurationManager::convertGroupTypeToName(groupType)
2653  << ".' All group members must be present to create the group editing "
2654  "structure."
2655  << __E__ << __E__ << StringMacros::stackTrace() << __E__;
2656  __SS_THROW__;
2657  }
2658 
2659 } // end GroupEditStruct constructor()
2660 
2661 //==============================================================================
2662 GroupEditStruct::~GroupEditStruct()
2663 {
2664  __GEN_COUT__ << "GroupEditStruct from editing '" << originalGroupName_ << "("
2665  << originalGroupKey_ << ")' Destructing..." << __E__;
2666  dropChanges();
2667  __GEN_COUT__ << "GroupEditStruct from editing '" << originalGroupName_ << "("
2668  << originalGroupKey_ << ")' Desctructed." << __E__;
2669 } // end GroupEditStruct destructor()
2670 
2671 //==============================================================================
2673 TableEditStruct& GroupEditStruct::getTableEditStruct(const std::string& tableName,
2674  bool markModified /*= false*/)
2675 {
2676  auto it = groupTables_.find(tableName);
2677  if(it == groupTables_.end())
2678  {
2679  if(groupType_ == ConfigurationManager::GroupType::CONFIGURATION_TYPE &&
2680  markModified)
2681  {
2682  __GEN_COUT__ << "Table '" << tableName
2683  << "' not found in configuration table members from editing '"
2684  << originalGroupName_ << "(" << originalGroupKey_ << ")..."
2685  << " Attempting to add it!" << __E__;
2686 
2687  // emplace returns pair<object,bool wasAdded>
2688  auto newIt = groupTables_.emplace(std::make_pair(
2689  tableName,
2690  TableEditStruct(tableName, cfgMgr_))); // Table ready for editing!
2691  if(newIt.second)
2692  {
2693  newIt.first->second.modified_ =
2694  markModified; // could indicate 'dirty' immediately in user code, which will cause a save of table
2695  groupMembers_.emplace(
2696  std::make_pair(tableName, newIt.first->second.temporaryVersion_));
2697  return newIt.first->second;
2698  }
2699  __GEN_COUT_ERR__ << "Failed to emplace new table..." << __E__;
2700  }
2701 
2702  __SS__ << "Table '" << tableName << "' not found in table members from editing '"
2703  << originalGroupName_ << "(" << originalGroupKey_ << ")!'" << __E__;
2704  __SS_THROW__;
2705  }
2706  it->second.modified_ =
2707  markModified; // could indicate 'dirty' immediately in user code, which will cause a save of table
2708  return it->second;
2709 } // end getTableEditStruct()
2710 
2711 //==============================================================================
2712 void GroupEditStruct::dropChanges()
2713 {
2714  __GEN_COUT__ << "Dropping unsaved changes from editing '" << originalGroupName_ << "("
2715  << originalGroupKey_ << ")'..." << __E__;
2716 
2717  ConfigurationManagerRW* cfgMgr = cfgMgr_;
2718 
2719  // drop all temporary versions
2720  for(auto& groupTable : groupTables_)
2721  if(groupTable.second
2722  .createdTemporaryVersion_) // if temporary version created here
2723  {
2724  // erase with proper version management
2725  cfgMgr->eraseTemporaryVersion(groupTable.second.tableName_,
2726  groupTable.second.temporaryVersion_);
2727  groupTable.second.createdTemporaryVersion_ = false;
2728  groupTable.second.modified_ = false;
2729  }
2730 
2731  __GEN_COUT__ << "Unsaved changes dropped from editing '" << originalGroupName_ << "("
2732  << originalGroupKey_ << ").'" << __E__;
2733 } // end GroupEditStruct::dropChanges()
2734 
2735 //==============================================================================
2736 void GroupEditStruct::saveChanges(const std::string& groupNameToSave,
2737  TableGroupKey& newGroupKey,
2738  bool* foundEquivalentGroupKey /*= nullptr*/,
2739  bool activateNewGroup /*= false*/,
2740  bool updateGroupAliases /*= false*/,
2741  bool updateTableAliases /*= false*/,
2742  TableGroupKey* newBackboneKey /*= nullptr*/,
2743  bool* foundEquivalentBackboneKey /*= nullptr*/,
2744  std::string* accumulatedWarnings /*= nullptr*/)
2745 {
2746  __GEN_COUT__ << "Saving changes..." << __E__;
2747 
2748  newGroupKey = TableGroupKey(); // invalidate reference parameter
2749  if(newBackboneKey)
2750  *newBackboneKey = TableGroupKey(); // invalidate reference parameter
2751  if(foundEquivalentBackboneKey)
2752  *foundEquivalentBackboneKey = false; // clear to start
2753  ConfigurationManagerRW* cfgMgr = cfgMgr_;
2754 
2755  // save all temporary modified versions
2756  bool anyTableNew = false;
2757  for(auto& groupTable : groupTables_)
2758  {
2759  if(!groupTable.second.modified_)
2760  continue; // skip if not modified
2761 
2762  __GEN_COUT__ << "Original version is " << groupTable.second.tableName_ << "-v"
2763  << groupTable.second.originalVersion_ << __E__;
2764 
2765  groupMembers_.at(groupTable.first) = cfgMgr->saveModifiedVersion(
2766  groupTable.second.tableName_,
2767  groupTable.second.originalVersion_,
2768  true /*make temporary*/,
2769  groupTable.second.table_,
2770  groupTable.second.temporaryVersion_,
2771  true /*ignoreDuplicates*/); // make temporary version to save persistent version properly
2772 
2773  __GEN_COUT__ << "Temporary target version is " << groupTable.second.tableName_
2774  << "-v" << groupMembers_.at(groupTable.first) << "-v"
2775  << groupTable.second.temporaryVersion_ << __E__;
2776 
2777  groupMembers_.at(groupTable.first) = cfgMgr->saveModifiedVersion(
2778  groupTable.second.tableName_,
2779  groupTable.second.originalVersion_,
2780  false /*make temporary*/,
2781  groupTable.second.table_,
2782  groupTable.second.temporaryVersion_,
2783  false /*ignoreDuplicates*/,
2784  true /*lookForEquivalent*/); // save persistent version properly
2785 
2786  if(groupTable.second.originalVersion_ != groupMembers_.at(groupTable.first))
2787  {
2788  anyTableNew = true;
2789  __GEN_COUT__ << "Final NEW target version is " << groupTable.second.tableName_
2790  << "-v" << groupMembers_.at(groupTable.first) << __E__;
2791  }
2792  else
2793  __GEN_COUT__ << "Final target version is " << groupTable.second.tableName_
2794  << "-v" << groupMembers_.at(groupTable.first) << __E__;
2795 
2796  groupTable.second.modified_ = false; // clear modified flag
2797  groupTable.second.createdTemporaryVersion_ = false; // modified version is gone
2798  } // loop through table edit structs
2799 
2800  for(auto& table : groupMembers_)
2801  {
2802  __GEN_COUT__ << table.first << " v" << table.second << __E__;
2803  }
2804 
2805  if(!anyTableNew) //then could be duplicate group
2806  {
2807  __GEN_COUT__ << "Checking for duplicate groups..." << __E__;
2808  newGroupKey = cfgMgr->findTableGroup(groupNameToSave, groupMembers_);
2809  }
2810  else
2811  __GEN_COUT__ << "New table found, so no need to check duplicate groups." << __E__;
2812 
2813  if(!newGroupKey.isInvalid())
2814  {
2815  __GEN_COUT__ << "Found equivalent group key (" << newGroupKey << ") for "
2816  << groupNameToSave << "." << __E__;
2817  if(foundEquivalentGroupKey)
2818  *foundEquivalentGroupKey = true;
2819  }
2820  else
2821  {
2822  newGroupKey = cfgMgr->saveNewTableGroup(groupNameToSave, groupMembers_);
2823  __GEN_COUT__ << "Saved new Context group key (" << newGroupKey << ") for "
2824  << groupNameToSave << "." << __E__;
2825  }
2826 
2827  bool groupAliasChange = false;
2828  bool tableAliasChange = false;
2829 
2830  if(groupType_ !=
2831  ConfigurationManager::GroupType::
2832  BACKBONE_TYPE) //if not backbone group save, consider changing aliases (not if it is backbone group, tables not necessarily active yet, which causes error in GroupEditStruct backboneGroupEdit)
2833  {
2834  GroupEditStruct backboneGroupEdit(ConfigurationManager::GroupType::BACKBONE_TYPE,
2835  cfgMgr);
2836 
2837  if(groupType_ != ConfigurationManager::GroupType::BACKBONE_TYPE &&
2838  updateGroupAliases)
2839  {
2840  // check group aliases ... a la
2841  // ConfigurationGUISupervisor::handleSetGroupAliasInBackboneXML
2842 
2843  TableEditStruct& groupAliasTable = backboneGroupEdit.getTableEditStruct(
2844  ConfigurationManager::GROUP_ALIASES_TABLE_NAME, true /*markModified*/);
2845  TableView* tableView = groupAliasTable.tableView_;
2846 
2847  // unsigned int col;
2848  unsigned int row = 0;
2849 
2850  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
2851  cfgMgr->getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
2852  .getChildren();
2853  std::string groupName, groupKey;
2854  for(auto& aliasNodePair : aliasNodePairs)
2855  {
2856  groupName = aliasNodePair.second.getNode("GroupName").getValueAsString();
2857  groupKey = aliasNodePair.second.getNode("GroupKey").getValueAsString();
2858 
2859  __GEN_COUT__ << "Group Alias: " << aliasNodePair.first << " => "
2860  << groupName << "(" << groupKey << "); row=" << row << __E__;
2861 
2862  if(groupName == originalGroupName_ &&
2863  TableGroupKey(groupKey) == originalGroupKey_)
2864  {
2865  __GEN_COUT__ << "Found alias! Changing group key from ("
2866  << originalGroupKey_ << ") to (" << newGroupKey << ")"
2867  << __E__;
2868 
2869  groupAliasChange = true;
2870 
2871  tableView->setValueAsString(
2872  newGroupKey.toString(), row, tableView->findCol("GroupKey"));
2873  }
2874 
2875  ++row;
2876  }
2877 
2878  if(groupAliasChange)
2879  {
2880  std::stringstream ss;
2881  tableView->print(ss);
2882  __GEN_COUT__ << ss.str();
2883  }
2884  } // end updateGroupAliases handling
2885 
2886  if(groupType_ != ConfigurationManager::GroupType::BACKBONE_TYPE &&
2887  updateTableAliases)
2888  {
2889  // update all table version aliases
2890  TableView* tableView =
2891  backboneGroupEdit
2892  .getTableEditStruct(ConfigurationManager::VERSION_ALIASES_TABLE_NAME,
2893  true /*markModified*/)
2894  .tableView_;
2895 
2896  for(auto& groupTable : groupTables_)
2897  {
2898  if(groupTable.second.originalVersion_ ==
2899  groupMembers_.at(groupTable.second.tableName_))
2900  continue; // skip if no change
2901 
2902  __GEN_COUT__ << "Checking alias... original version is "
2903  << groupTable.second.tableName_ << "-v"
2904  << groupTable.second.originalVersion_
2905  << " and new version is v"
2906  << groupMembers_.at(groupTable.second.tableName_) << __E__;
2907 
2908  // unsigned int col;
2909  unsigned int row = 0;
2910 
2911  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
2912  cfgMgr->getNode(ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
2913  .getChildren();
2914  std::string tableName, tableVersion;
2915  for(auto& aliasNodePair : aliasNodePairs)
2916  {
2917  tableName =
2918  aliasNodePair.second.getNode("TableName").getValueAsString();
2919  tableVersion =
2920  aliasNodePair.second.getNode("Version").getValueAsString();
2921 
2922  __GEN_COUT__ << "Table Alias: " << aliasNodePair.first << " => "
2923  << tableName << "-v" << tableVersion << "" << __E__;
2924 
2925  if(tableName == groupTable.second.tableName_ &&
2926  TableVersion(tableVersion) == groupTable.second.originalVersion_)
2927  {
2928  __GEN_COUT__ << "Found alias! Changing icon table version alias."
2929  << __E__;
2930 
2931  tableAliasChange = true;
2932 
2933  tableView->setValueAsString(
2934  groupMembers_.at(groupTable.second.tableName_).toString(),
2935  row,
2936  tableView->findCol("Version"));
2937  }
2938 
2939  ++row;
2940  }
2941  }
2942 
2943  if(tableAliasChange)
2944  {
2945  std::stringstream ss;
2946  tableView->print(ss);
2947  __GEN_COUT__ << ss.str();
2948  }
2949  } // end updateTableAliases handling
2950 
2951  TableGroupKey localNewBackboneKey;
2952  // if backbone modified, save group and activate it
2953  if(groupAliasChange || tableAliasChange)
2954  {
2955  for(auto& table : backboneGroupEdit.groupMembers_)
2956  {
2957  __GEN_COUT__ << table.first << " v" << table.second << __E__;
2958  }
2959  backboneGroupEdit.saveChanges(
2960  backboneGroupEdit.originalGroupName_,
2961  localNewBackboneKey,
2962  foundEquivalentBackboneKey ? foundEquivalentBackboneKey : nullptr);
2963 
2964  if(newBackboneKey)
2965  *newBackboneKey = localNewBackboneKey;
2966  }
2967 
2968  // acquire all active groups and ignore errors, so that activateTableGroup does not
2969  // erase other active groups
2970  {
2971  __GEN_COUT__
2972  << "Restoring active table groups, before activating new groups..."
2973  << __E__;
2974 
2975  std::string localAccumulatedWarnings;
2976  cfgMgr->restoreActiveTableGroups(
2977  false /*throwErrors*/,
2978  "" /*pathToActiveGroupsFile*/,
2979  ConfigurationManager::LoadGroupType::
2980  ALL_TYPES /*onlyLoadIfBackboneOrContext*/,
2981  &localAccumulatedWarnings);
2982  }
2983 
2984  // activate new groups
2985  if(!localNewBackboneKey.isInvalid())
2986  cfgMgr->activateTableGroup(
2987  backboneGroupEdit.originalGroupName_,
2988  localNewBackboneKey,
2989  accumulatedWarnings ? accumulatedWarnings : nullptr);
2990 
2991  } //end non-backbone save type handling
2992  else //is backbone save type
2993  {
2994  // acquire all active groups and ignore errors, so that activateTableGroup does not
2995  // erase other active groups
2996  {
2997  __GEN_COUT__
2998  << "Restoring active table groups, before activating new groups..."
2999  << __E__;
3000 
3001  std::string localAccumulatedWarnings;
3002  cfgMgr->restoreActiveTableGroups(
3003  false /*throwErrors*/,
3004  "" /*pathToActiveGroupsFile*/,
3005  ConfigurationManager::LoadGroupType::
3006  ALL_TYPES /*onlyLoadIfBackboneOrContext*/,
3007  &localAccumulatedWarnings);
3008  }
3009  } //end backbone save type handling
3010 
3011  if(activateNewGroup)
3012  cfgMgr->activateTableGroup(groupNameToSave,
3013  newGroupKey,
3014  accumulatedWarnings ? accumulatedWarnings : nullptr);
3015 
3016  __GEN_COUT__ << "Changes saved." << __E__;
3017 } // end GroupEditStruct::saveChanges()
3018 
3019 //==============================================================================
3021 void ConfigurationManagerRW::testXDAQContext()
3022 {
3023  if(1)
3024  return; //if 0 to debug
3025  __GEN_COUTV__(runTimeSeconds());
3026 
3027  std::string accumulatedWarningsStr;
3028  std::string* accumulatedWarnings = &accumulatedWarningsStr;
3029 
3030  // get Group Info too!
3031  try
3032  {
3033  //test lookup of which groups a table is in
3034  {
3035  std::string documentNameToLoad = "XDAQApplicationTable";
3036  TableVersion documentVersionToLoad(
3037  (int)134); //1 is easy, 134 is hard on daq13 mongodb
3038 
3039  std::set<std::string> groupsContainingTable =
3040  theInterface_->findGroupsWithTable(documentNameToLoad,
3041  documentVersionToLoad);
3042  __GEN_COUT__ << "Groups containing " << documentNameToLoad << "-v"
3043  << documentVersionToLoad
3044  << " count: " << groupsContainingTable.size() << __E__;
3045  for(const auto& group : groupsContainingTable)
3046  {
3047  __GEN_COUT__ << "\t" << group << __E__;
3048  }
3049  }
3050 
3051  std::string debugGroupName = "Mu2eHWEmulatorContext";
3052 
3053  //final solution demo of getting latest group key:
3054  {
3055  TableGroupKey latestGroupKey =
3056  theInterface_->findLatestGroupKey(debugGroupName);
3057  __GEN_COUTV__(latestGroupKey);
3058 
3059  __GEN_COUTV__(runTimeSeconds());
3060  }
3061 
3062  //steps to do time comparison for getting last group key and table key:
3063 
3064  // build allGroupInfo_ for the ConfigurationManagerRW
3065 
3066  std::set<std::string /*name*/> tableGroups =
3067  theInterface_->getAllTableGroupNames();
3068  __GEN_COUT__ << "Number of Groups: " << tableGroups.size() << __E__;
3069 
3070  __GEN_COUTV__(runTimeSeconds());
3071  // return;
3072 
3073  TableGroupKey key;
3074  std::string name;
3075  for(const auto& fullName : tableGroups)
3076  {
3077  TableGroupKey::getGroupNameAndKey(fullName, name, key);
3078  allGroupInfo_[name].keys_.emplace(key);
3079 
3080  if(name == debugGroupName)
3081  {
3082  __GEN_COUTV__(key);
3083  }
3084  }
3085  __GEN_COUTV__(runTimeSeconds());
3086 
3087  std::set<std::string /*name*/> tableNames = theInterface_->getAllTableNames();
3088  __GEN_COUT__ << "Number of Tables: " << tableNames.size() << __E__;
3089 
3090  __GEN_COUTV__(runTimeSeconds());
3091 
3092  for(const auto& fullName : tableNames)
3093  {
3094  if(fullName.find(debugGroupName) != std::string::npos)
3095  {
3096  __GEN_COUTV__(fullName);
3097  }
3098  }
3099  __GEN_COUTV__(runTimeSeconds());
3100 
3101  TableGroupKey latestGroupKey = theInterface_->findLatestGroupKey(debugGroupName);
3102  __GEN_COUTV__(latestGroupKey);
3103 
3104  __GEN_COUTV__(runTimeSeconds());
3105 
3106  TableBase localGroupMemberCacheSaver(
3107  true /*special table*/
3108  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),
3109  TableBase::GROUP_CACHE_PREPEND + debugGroupName);
3110  TableVersion lastestGroupCacheKey =
3111  theInterface_->findLatestVersion(&localGroupMemberCacheSaver);
3112  __GEN_COUTV__(lastestGroupCacheKey);
3113 
3114  __GEN_COUTV__(runTimeSeconds());
3115 
3116  //test a group save that already exists
3117  try
3118  {
3119  TableGroupKey groupKey(int(0));
3120  __GEN_COUT__ << "Testing group save of pre-existing " << debugGroupName << "("
3121  << groupKey << ")" << __E__;
3122  std::map<std::string, TableVersion> groupMembers;
3123  groupMembers["DesktopIconTable"] = TableVersion(123);
3124  theInterface_->saveTableGroup(
3125  groupMembers,
3126  TableGroupKey::getFullGroupString(debugGroupName, groupKey));
3127  }
3128  catch(...)
3129  {
3130  __GEN_COUT__ << "Exception during group save." << __E__;
3131  }
3132  __GEN_COUTV__(runTimeSeconds());
3133 
3134  //test a group save that does not already exists
3135  try
3136  {
3137  std::string debugGroupName = "testGroupSave";
3138  TableGroupKey groupKey(int(2));
3139  __GEN_COUT__ << "Testing group save of non-existing " << debugGroupName << "("
3140  << groupKey << ")" << __E__;
3141  std::map<std::string, TableVersion> groupMembers;
3142  groupMembers["DesktopIconTable"] = TableVersion(123);
3143  groupMembers["MessageFacilityTable"] = TableVersion(7);
3144  theInterface_->saveTableGroup(
3145  groupMembers,
3146  TableGroupKey::getFullGroupString(debugGroupName, groupKey));
3147  }
3148  catch(...)
3149  {
3150  __GEN_COUT__ << "Exception during new group save." << __E__;
3151  }
3152  __GEN_COUTV__(runTimeSeconds());
3153 
3154  //test a table save that already exists
3155  {
3156  std::string documentNameToLoad = "XDAQApplicationTable";
3157  TableVersion documentVersionToLoad(134);
3158 
3159  __GEN_COUT__ << "Testing table save of pre-existing " << documentNameToLoad
3160  << __E__;
3161 
3162  { //load to prove it exists
3163  TableBase localDocLoader(
3164  documentNameToLoad); //can not use special table when filling
3165  localDocLoader.changeVersionAndActivateView(
3166  localDocLoader.createTemporaryView(), documentVersionToLoad);
3167  theInterface_->fill(&localDocLoader, documentVersionToLoad);
3168  __SS__;
3169  localDocLoader.print(ss);
3170  __GEN_COUTV__(ss.str());
3171  }
3172  __GEN_COUTV__(runTimeSeconds());
3173 
3174  try
3175  { //attempt to save over existing version
3176  std::string documentNameToSave = documentNameToLoad;
3177  TableBase
3178  localDocSaver( //true /*special table*/, //special table only allows 1 view in cache and does not load schema (which is perfect for this check),
3179  documentNameToSave); //can not use special table when filling
3180  localDocSaver.changeVersionAndActivateView(
3181  localDocSaver.createTemporaryView(), documentVersionToLoad);
3182 
3183  std::string json = "{ }";
3184  localDocSaver.getViewP()->setCustomStorageData(json);
3185 
3186  __COUTT__ << "Saving JSON string: "
3187  << localDocSaver.getViewP()->getCustomStorageData() << __E__;
3188 
3189  __COUTT__ << "Saving JSON doc as "
3190  << localDocSaver.getView().getTableName() << "("
3191  << localDocSaver.getView().getVersion().toString() << ")"
3192  << __E__;
3193 
3194  // save to db, and do not allow overwrite
3195  theInterface_->saveActiveVersion(&localDocSaver, false /* overwrite */);
3196  }
3197  catch(...)
3198  {
3199  __GEN_COUT__ << "Exception during table save." << __E__;
3200  }
3201  __GEN_COUTV__(runTimeSeconds());
3202 
3203  { //load to prove it exists
3204  TableBase localDocLoader(
3205  documentNameToLoad); //can not use special table when filling
3206  localDocLoader.changeVersionAndActivateView(
3207  localDocLoader.createTemporaryView(), documentVersionToLoad);
3208  theInterface_->fill(&localDocLoader, documentVersionToLoad);
3209  __SS__;
3210  localDocLoader.print(ss);
3211  __GEN_COUTV__(ss.str());
3212  }
3213  __GEN_COUTV__(runTimeSeconds());
3214  }
3215  __GEN_COUTV__(runTimeSeconds());
3216 
3217  //test a table save that does not already exist
3218  {
3219  std::string documentNameToLoad = "MessageFacilityTable";
3220  TableVersion documentVersionToLoad(7);
3221  TableBase localDocLoader(
3222  documentNameToLoad); //can not use special table when filling
3223 
3224  __GEN_COUT__ << "Testing table save of non-existing " << documentNameToLoad
3225  << __E__;
3226 
3227  { //load to prove it exists
3228  localDocLoader.changeVersionAndActivateView(
3229  localDocLoader.createTemporaryView(), documentVersionToLoad);
3230  theInterface_->fill(&localDocLoader, documentVersionToLoad);
3231  __SS__;
3232  localDocLoader.print(ss);
3233  __GEN_COUTV__(ss.str());
3234  __GEN_COUTV__(runTimeSeconds());
3235  }
3236  __GEN_COUTV__(runTimeSeconds());
3237 
3238  try
3239  { //attempt to save new version
3240 
3241  // modify it
3242  TableVersion newVersion = TableVersion::getNextVersion(
3243  theInterface_->findLatestVersion(&localDocLoader));
3244  localDocLoader.getViewP()->setVersion(newVersion);
3245 
3246  __GEN_COUTT__ << "Saving new table as "
3247  << localDocLoader.getView().getTableName() << "("
3248  << localDocLoader.getView().getVersion().toString() << ")"
3249  << __E__;
3250 
3251  localDocLoader.getViewP()->setValueAsString(
3252  "10.226.9.17", 0, 4); //modify value that is 10.226.9.16
3253 
3254  __SS__;
3255  localDocLoader.print(ss);
3256  __GEN_COUTV__(ss.str());
3257 
3258  // save to db, and do not allow overwrite
3259  theInterface_->saveActiveVersion(&localDocLoader, false /* overwrite */);
3260  }
3261  catch(...)
3262  {
3263  __GEN_COUT__ << "Exception during new table save." << __E__;
3264  }
3265  __GEN_COUTV__(runTimeSeconds());
3266  }
3267  __GEN_COUTV__(runTimeSeconds());
3268  return;
3269 
3270  // for each group get member map & comment, author, time, and type for latest key
3271  for(auto& groupInfo : allGroupInfo_)
3272  {
3273  try
3274  {
3275  groupInfo.second.latestKey_ = groupInfo.second.getLastKey();
3276  loadTableGroup(groupInfo.first /*groupName*/,
3277  groupInfo.second.latestKey_,
3278  false /*doActivate*/,
3279  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
3280  0 /*progressBar*/,
3281  0 /*accumulateErrors*/,
3282  &groupInfo.second.latestKeyGroupComment_,
3283  &groupInfo.second.latestKeyGroupAuthor_,
3284  &groupInfo.second.latestKeyGroupCreationTime_,
3285  true /*doNotLoadMember*/,
3286  &groupInfo.second.latestKeyGroupTypeString_);
3287  }
3288  catch(const std::runtime_error& e)
3289  {
3290  __GEN_COUT_WARN__
3291  << "Error occurred loading latest group info into cache for '"
3292  << groupInfo.first << "(" << groupInfo.second.latestKey_ << ")': \n"
3293  << e.what() << __E__;
3294 
3295  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
3296  groupInfo.second.latestKeyGroupComment_ =
3297  ConfigurationManager::UNKNOWN_INFO;
3298  groupInfo.second.latestKeyGroupAuthor_ =
3299  ConfigurationManager::UNKNOWN_INFO;
3300  groupInfo.second.latestKeyGroupCreationTime_ =
3301  ConfigurationManager::UNKNOWN_TIME;
3302  groupInfo.second.latestKeyGroupTypeString_ =
3303  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
3304  groupInfo.second.latestKeyMemberMap_ = {};
3305  }
3306  catch(...)
3307  {
3308  __GEN_COUT_WARN__
3309  << "Error occurred loading latest group info into cache for '"
3310  << groupInfo.first << "(" << groupInfo.second.latestKey_ << ")'..."
3311  << __E__;
3312 
3313  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
3314  groupInfo.second.latestKeyGroupComment_ =
3315  ConfigurationManager::UNKNOWN_INFO;
3316  groupInfo.second.latestKeyGroupAuthor_ =
3317  ConfigurationManager::UNKNOWN_INFO;
3318  groupInfo.second.latestKeyGroupCreationTime_ =
3319  ConfigurationManager::UNKNOWN_TIME;
3320  groupInfo.second.latestKeyGroupTypeString_ =
3321  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
3322  groupInfo.second.latestKeyMemberMap_ = {};
3323  }
3324  } // end group info loop
3325  __GEN_COUTV__(runTimeSeconds());
3326  } // end get group info
3327  catch(const std::runtime_error& e)
3328  {
3329  __SS__ << "A fatal error occurred reading the info for all table groups. Error: "
3330  << e.what() << __E__;
3331  __GEN_COUT_ERR__ << "\n" << ss.str();
3332  if(accumulatedWarnings)
3333  *accumulatedWarnings += ss.str();
3334  else
3335  throw;
3336  }
3337  catch(...)
3338  {
3339  __SS__ << "An unknown fatal error occurred reading the info for all table groups."
3340  << __E__;
3341  __GEN_COUT_ERR__ << "\n" << ss.str();
3342  if(accumulatedWarnings)
3343  *accumulatedWarnings += ss.str();
3344  else
3345  throw;
3346  } //end catch
3347 
3348  __GEN_COUT__ << "testXDAQContext() end runTimeSeconds()=" << runTimeSeconds()
3349  << __E__;
3350  return;
3351 
3352  try
3353  {
3354  __GEN_COUT__ << "Loading table..." << __E__;
3355  loadTableGroup("FETest", TableGroupKey(2)); // Context_1
3356  ConfigurationTree t = getNode("/FETable/DEFAULT/FrontEndType");
3357 
3358  std::string v;
3359 
3360  __GEN_COUT__ << __E__;
3361  t.getValue(v);
3362  __GEN_COUT__ << "Value: " << v << __E__;
3363  __GEN_COUT__ << "Value index: " << t.getValue<int>() << __E__;
3364 
3365  return;
3366  }
3367  catch(...)
3368  {
3369  __GEN_COUT__ << "Failed to load table..." << __E__;
3370  }
3371 } //end testXDAQContext()
ots::ConfigurationInterface::saveNewVersion
TableVersion saveNewVersion(TableBase *table, TableVersion temporaryVersion, TableVersion newVersion=TableVersion())
Definition: ConfigurationInterface.cc:79
ots::ConfigurationManagerRW::saveNewTable
TableVersion saveNewTable(const std::string &tableName, TableVersion temporaryVersion=TableVersion(), bool makeTemporary=false)
modifiers of generic TableBase
Definition: ConfigurationManagerRW.cc:1613
ots::ConfigurationManagerRW::findTableGroup
TableGroupKey findTableGroup(const std::string &groupName, const std::map< std::string, TableVersion > &groupMembers, const std::map< std::string, std::string > &groupAliases=std::map< std::string, std::string >())
Definition: ConfigurationManagerRW.cc:1839
ots::ConfigurationManagerRW::testXDAQContext
void testXDAQContext(void)
for debugging
Definition: ConfigurationManagerRW.cc:3021
ots::ConfigurationManagerRW::saveNewBackbone
TableVersion saveNewBackbone(TableVersion temporaryVersion=TableVersion())
Definition: ConfigurationManagerRW.cc:2426
ots::ConfigurationManagerRW::getGroupInfo
const GroupInfo & getGroupInfo(const std::string &groupName, bool attemptToReloadKeys=false)
public group cache handling
Definition: ConfigurationManagerRW.cc:1771
ots::ConfigurationManagerRW::getAllTableInfo
const std::map< std::string, TableInfo > & getAllTableInfo(bool refresh=false, std::string *accumulatedWarnings=0, const std::string &errorFilterName="", bool getGroupKeys=false, bool getGroupInfo=false, bool initializeActiveGroups=false)
Definition: ConfigurationManagerRW.cc:209
ots::ConfigurationManagerRW::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false)
Definition: ConfigurationManagerRW.cc:900
ots::ConfigurationManagerRW::copyViewToCurrentColumns
TableVersion copyViewToCurrentColumns(const std::string &tableName, TableVersion sourceVersion)
copyViewToCurrentColumns
Definition: ConfigurationManagerRW.cc:1747
ots::ConfigurationManagerRW::saveNewTableGroup
TableGroupKey saveNewTableGroup(const std::string &groupName, std::map< std::string, TableVersion > &groupMembers, const std::string &groupComment=TableViewColumnInfo::DATATYPE_COMMENT_DEFAULT, std::map< std::string, std::string > *groupAliases=0)
modifiers of a table group based on alias, e.g. "Physics"
Definition: ConfigurationManagerRW.cc:2163
ots::ConfigurationManagerRW::activateTableGroup
void activateTableGroup(const std::string &tableGroupName, TableGroupKey tableGroupKey, std::string *accumulatedTreeErrors=0, std::string *groupTypeString=0)
modifiers of table groups
Definition: ConfigurationManagerRW.cc:1150
ots::ConfigurationManagerRW::getMetadataTable
TableBase * getMetadataTable(TableVersion fillVersion=TableVersion())
created for use in otsdaq_flatten_system_aliases and otsdaq_export_system_aliases,...
Definition: ConfigurationManagerRW.cc:2104
ots::ConfigurationManagerRW::saveModifiedVersion
TableVersion saveModifiedVersion(const std::string &tableName, TableVersion originalVersion, bool makeTemporary, TableBase *config, TableVersion temporaryModifiedVersion, bool ignoreDuplicates=false, bool lookForEquivalent=false, bool *foundEquivalent=nullptr)
Definition: ConfigurationManagerRW.cc:2469
ots::ConfigurationManagerRW::createTemporaryBackboneView
TableVersion createTemporaryBackboneView(TableVersion sourceViewVersion=TableVersion())
-1, from MockUp, else from valid backbone view version
Definition: ConfigurationManagerRW.cc:1311
ots::ConfigurationManagerRW::clearCachedVersions
void clearCachedVersions(const std::string &tableName)
Definition: ConfigurationManagerRW.cc:1727
ots::ConfigurationManagerRW::getVersionAliases
std::map< std::string, std::map< std::string, TableVersion > > getVersionAliases(void) const
Definition: ConfigurationManagerRW.cc:1128
ots::ConfigurationManagerRW::eraseTemporaryVersion
void eraseTemporaryVersion(const std::string &tableName, TableVersion targetVersion=TableVersion())
Definition: ConfigurationManagerRW.cc:1678
ots::ConfigurationManagerRW::preloadVersionCreationTimes
void preloadVersionCreationTimes(void)
parallel load of all version creation times into the process-wide cache
Definition: ConfigurationManagerRW.cc:1442
ots::ConfigurationManagerRW::compareTableGroupThread
static void compareTableGroupThread(ConfigurationManagerRW *cfgMgr, std::string groupName, ots::TableGroupKey groupKeyToCompare, const std::map< std::string, TableVersion > &groupMemberMap, const std::map< std::string, std::string > &memberTableAliases, std::atomic< bool > *theFoundIdentical, ots::TableGroupKey *theIdenticalKey, std::mutex *theThreadMutex, std::shared_ptr< std::atomic< bool >> theThreadDone)
Definition: ConfigurationManagerRW.cc:997
ots::ConfigurationManagerRW::loadTableGroupThread
static void loadTableGroupThread(ConfigurationManagerRW *cfgMgr, std::string groupName, ots::TableGroupKey groupKey, std::shared_ptr< ots::GroupInfo > theGroupInfo, std::shared_ptr< std::atomic< bool >> theThreadDone)
loadTableGroupThread()
Definition: ConfigurationManagerRW.cc:955
ots::ConfigurationManagerRW::getVersionCreationTime
time_t getVersionCreationTime(const std::string &tableName, TableVersion version)
Definition: ConfigurationManagerRW.cc:1392
ots::ConfigurationManagerRW::loadTableInfoThread
static void loadTableInfoThread(ConfigurationManagerRW *cfgMgr, std::string tableName, TableBase *existingTable, std::shared_ptr< ots::TableInfo > tableInfo, std::shared_ptr< std::atomic< bool >> threadDone)
loadTableInfoThread()
Definition: ConfigurationManagerRW.cc:761
ots::ConfigurationManager::getVersionAliases
std::map< std::string, std::map< std::string, TableVersion > > getVersionAliases(void) const
Definition: ConfigurationManager.cc:4089
ots::ConfigurationManager::PROCESSOR_COUNT
static const unsigned int PROCESSOR_COUNT
Static members.
Definition: ConfigurationManager.h:37
ots::ConfigurationManager::convertGroupTypeToName
static const std::string & convertGroupTypeToName(const ConfigurationManager::GroupType &groupTypeId)
Definition: ConfigurationManager.cc:652
ots::ConfigurationManager::restoreActiveTableGroups
void restoreActiveTableGroups(bool throwErrors=false, const std::string &pathToActiveGroupsFile="", ConfigurationManager::LoadGroupType onlyLoadIfBackboneOrContext=ConfigurationManager::LoadGroupType::ALL_TYPES, std::string *accumulatedWarnings=0)
Definition: ConfigurationManager.cc:329
ots::ConfigurationManager::getActiveVersions
std::map< std::string, TableVersion > getActiveVersions(void) const
getActiveVersions
Definition: ConfigurationManager.cc:4137
ots::ConfigurationManager::getNode
ConfigurationTree getNode(const std::string &nodeString, bool doNotThrowOnBrokenUIDLinks=false) const
"root/parent/parent/"
Definition: ConfigurationManager.cc:3613
ots::ConfigurationManager::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false, std::map< std::string, TableVersion > mergeInTables={}, std::map< std::string, TableVersion > overrideTables={})
Definition: ConfigurationManager.cc:2024
ots::ConfigurationManager::init
void init(std::string *accumulatedErrors=0, bool initForWriteAccess=false, std::string *accumulatedWarnings=0)
Definition: ConfigurationManager.cc:278
ots::ConfigurationManager::getTypeNameOfGroup
static const std::string & getTypeNameOfGroup(const std::map< std::string, TableVersion > &memberMap)
Definition: ConfigurationManager.cc:850
ots::ConfigurationManager::destroyTableGroup
void destroyTableGroup(const std::string &theGroup="", bool onlyDeactivate=false)
Definition: ConfigurationManager.cc:517
ots::ConfigurationManager::ACTIVE_GROUPS_FILENAME
static const std::string ACTIVE_GROUPS_FILENAME
added env check for otsdaq_flatten_active_to_version to function
Definition: ConfigurationManager.h:40
ots::ConfigurationManager::getTableByName
const TableBase * getTableByName(const std::string &configurationName) const
Definition: ConfigurationManager.cc:3901
ots::ConfigurationTree::getValue
void getValue(T &value) const
ots::ConfigurationTree::getChildren
std::vector< std::pair< std::string, ConfigurationTree > > getChildren(std::map< std::string, std::string > filterMap=std::map< std::string, std::string >(), bool byPriority=false, bool onlyStatusTrue=false) const
Definition: ConfigurationTree.cc:2599
ots::TableBase::createTemporaryView
TableVersion createTemporaryView(TableVersion sourceViewVersion=TableVersion(), TableVersion destTemporaryViewVersion=TableVersion::getNextTemporaryVersion())
source of -1, from MockUp, else from valid view version
Definition: TableBase.cc:1769
ots::TableBase::trimTemporary
void trimTemporary(TableVersion targetVersion=TableVersion())
Definition: TableBase.cc:362
ots::TableBase::getNumberOfStoredViews
unsigned int getNumberOfStoredViews(void) const
Definition: TableBase.cc:856
ots::TableBase::checkForDuplicate
TableVersion checkForDuplicate(TableVersion needleVersion, TableVersion ignoreVersion=TableVersion()) const
Definition: TableBase.cc:404
ots::TableBase::getTemporaryView
TableView * getTemporaryView(TableVersion temporaryVersion)
Definition: TableBase.cc:1880
ots::TableBase::MAX_VIEWS_IN_CACHE
const unsigned int MAX_VIEWS_IN_CACHE
Definition: TableBase.h:30
ots::TableBase::getNextVersion
TableVersion getNextVersion(void) const
Definition: TableBase.cc:1856
ots::TableBase::copyView
TableVersion copyView(const TableView &sourceView, TableVersion destinationVersion, const std::string &author, bool looseColumnMatching=false)
Definition: TableBase.cc:1710
ots::TableBase::print
void print(std::ostream &out=std::cout) const
always prints active view
Definition: TableBase.cc:277
ots::TableBase::getNextTemporaryVersion
TableVersion getNextTemporaryVersion(void) const
Definition: TableBase.cc:1833
ots::TableBase::trimCache
void trimCache(unsigned int trimSize=-1)
Definition: TableBase.cc:322
ots::TableGroupKey::toString
std::string toString(void) const
toString
Definition: TableGroupKey.cc:111
ots::TableGroupKey::getNextKey
static TableGroupKey getNextKey(const TableGroupKey &key=TableGroupKey())
Definition: TableGroupKey.cc:156
ots::TableGroupKey::getGroupNameAndKey
static void getGroupNameAndKey(const std::string &fullGroupString, std::string &groupName, TableGroupKey &key)
requires fullGroupString created as name + "_v" + key + ""
Definition: TableGroupKey.cc:220
ots::TableGroupKey::isInvalid
bool isInvalid(void) const
isInvalid
Definition: TableGroupKey.cc:148
ots::TableGroupKey::getFullGroupString
static std::string getFullGroupString(const std::string &groupName, const TableGroupKey &key, const std::string &preKey="_v", const std::string &postKey="")
Definition: TableGroupKey.cc:174
ots::TableVersion::toString
std::string toString(void) const
toString
Definition: TableVersion.cc:33
ots::TableVersion::isInvalid
bool isInvalid(void) const
isInvalid
Definition: TableVersion.cc:127
ots::TableVersion::getNextVersion
static TableVersion getNextVersion(const TableVersion &version=TableVersion())
Definition: TableVersion.cc:161
ots::TableVersion::isScratchVersion
bool isScratchVersion(void) const
Definition: TableVersion.cc:145
ots::TableVersion::isTemporaryVersion
bool isTemporaryVersion(void) const
Definition: TableVersion.cc:137
ots::TableVersion::getNextTemporaryVersion
static TableVersion getNextTemporaryVersion(const TableVersion &version=TableVersion())
Definition: TableVersion.cc:174
ots::TableView::setValueAsString
void setValueAsString(const std::string &value, unsigned int row, unsigned int col)
Definition: TableView.cc:1087
ots::TableView::setVersion
void setVersion(const T &version)
< in included .icc source
ots::TableView::getCustomStorageData
const std::string & getCustomStorageData(void) const
Getters.
Definition: TableView.h:71
ots::TableView::findCol
unsigned int findCol(const std::string &name) const
Definition: TableView.cc:1970
ots::TableView::setCustomStorageData
void setCustomStorageData(const std::string &storageData)
Definition: TableView.h:168
ots
defines used also by OtsConfigurationWizardSupervisor
Definition: ArtdaqOtsBuildInfo_module.cc:10
ots::GroupEditStruct::getTableEditStruct
TableEditStruct & getTableEditStruct(const std::string &tableName, bool markModified=false)
Note: if markModified, and table not found in group, this function will try to add it to group.
Definition: ConfigurationManagerRW.cc:2673
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
setToString ~
ots::StringMacros::mapToString
static std::string mapToString(const std::map< std::string, T > &mapToReturn, const std::string &primaryDelimeter=", ", const std::string &secondaryDelimeter=": ")
ots::StringMacros::stackTrace
static std::string stackTrace(void)
Definition: StringMacros.cc:1817