otsdaq  3.03.00
ConfigurationManagerRW.cc
1 #include "otsdaq/ConfigurationInterface/ConfigurationManagerRW.h"
2 
3 #include <dirent.h>
4 
5 using namespace ots;
6 
7 #undef __MF_SUBJECT__
8 #define __MF_SUBJECT__ "ConfigurationManagerRW"
9 
10 #define TABLE_INFO_PATH std::string(__ENV__("TABLE_INFO_PATH")) + "/"
11 #define TABLE_INFO_EXT "Info.xml"
12 
13 #define CORE_TABLE_INFO_FILENAME \
14  ((getenv("SERVICE_DATA_PATH") == NULL) \
15  ? (std::string(__ENV__("USER_DATA")) + "/ServiceData") \
16  : (std::string(__ENV__("SERVICE_DATA_PATH")))) + \
17  "/CoreTableInfoNames.dat"
18 
19 std::atomic<bool> ConfigurationManagerRW::firstTimeConstructed_ = true;
20 
21 //==============================================================================
23 ConfigurationManagerRW::ConfigurationManagerRW(const std::string& username)
24  : ConfigurationManager(username) // for use as author of new views
25 {
26  __GEN_COUT__ << "Instantiating Config Manager with Write Access! (for " << username
27  << ") time=" << time(0) << " runTimeSeconds()=" << runTimeSeconds()
28  << __E__;
29 
30  theInterface_ = ConfigurationInterface::getInstance(
31  ConfigurationInterface::CONFIGURATION_MODE::
32  ARTDAQ_DATABASE); // false to use artdaq DB
33 
34  //=========================
35  // dump names of core tables (so UpdateOTS.sh can copy core tables for user)
36  // only if table does not exist
37  if(firstTimeConstructed_)
38  {
39  firstTimeConstructed_ = false;
40 
41  // make table group history directory here and at Gateway Supervisor (just in case)
42  mkdir((ConfigurationManager::LAST_TABLE_GROUP_SAVE_PATH).c_str(), 0755);
43 
44  const std::set<std::string>& contextMemberNames = getContextMemberNames();
45  const std::set<std::string>& backboneMemberNames = getBackboneMemberNames();
46  const std::set<std::string>& iterateMemberNames = getIterateMemberNames();
47 
48  FILE* fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "r");
49 
50  if(fp) // check for all core table names in file, and force their presence
51  {
52  std::vector<unsigned int> foundVector;
53  char line[100];
54  for(const auto& name : contextMemberNames)
55  {
56  foundVector.push_back(false);
57  rewind(fp);
58  while(fgets(line, 100, fp))
59  {
60  if(strlen(line) < 1)
61  continue;
62  line[strlen(line) - 1] = '\0'; // remove endline
63  if(strcmp(line, ("ContextGroup/" + name).c_str()) == 0) // is match?
64  {
65  foundVector.back() = true;
66  break;
67  }
68  }
69  }
70 
71  for(const auto& name : backboneMemberNames)
72  {
73  foundVector.push_back(false);
74  rewind(fp);
75  while(fgets(line, 100, fp))
76  {
77  if(strlen(line) < 1)
78  continue;
79  line[strlen(line) - 1] = '\0'; // remove endline
80  if(strcmp(line, ("BackboneGroup/" + name).c_str()) == 0) // is match?
81  {
82  foundVector.back() = true;
83  break;
84  }
85  }
86  }
87 
88  for(const auto& name : iterateMemberNames)
89  {
90  foundVector.push_back(false);
91  rewind(fp);
92  while(fgets(line, 100, fp))
93  {
94  if(strlen(line) < 1)
95  continue;
96  line[strlen(line) - 1] = '\0'; // remove endline
97  if(strcmp(line, ("IterateGroup/" + name).c_str()) == 0) // is match?
98  {
99  foundVector.back() = true;
100  break;
101  }
102  }
103  }
104 
105  //look for optional Context table ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE
106  {
107  foundVector.push_back(false);
108  rewind(fp);
109  while(fgets(line, 100, fp))
110  {
111  if(strlen(line) < 1)
112  continue;
113  line[strlen(line) - 1] = '\0'; // remove endline
114  if(strcmp(line,
115  ("ContextGroup/" +
116  ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE)
117  .c_str()) == 0) // is match?
118  {
119  foundVector.back() = true;
120  break;
121  }
122  }
123  }
124 
125  fclose(fp);
126 
127  // open file for appending the missing names
128  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "a");
129  if(fp)
130  {
131  unsigned int i = 0;
132  for(const auto& name : contextMemberNames)
133  {
134  if(!foundVector[i])
135  fprintf(fp, "\nContextGroup/%s", name.c_str());
136 
137  ++i;
138  }
139  for(const auto& name : backboneMemberNames)
140  {
141  if(!foundVector[i])
142  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
143 
144  ++i;
145  }
146  for(const auto& name : iterateMemberNames)
147  {
148  if(!foundVector[i])
149  fprintf(fp, "\nIterateGroup/%s", name.c_str());
150 
151  ++i;
152  }
153 
154  //last is optional Context table ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE
155  if(!foundVector[i])
156  fprintf(
157  fp,
158  "\nContextGroup/%s",
159  ConfigurationManager::CONTEXT_SUBSYSTEM_OPTIONAL_TABLE.c_str());
160 
161  fclose(fp);
162  }
163  else
164  {
165  __SS__ << "Failed to open core table info file for appending: "
166  << CORE_TABLE_INFO_FILENAME << __E__;
167  __SS_THROW__;
168  }
169  }
170  else
171  {
172  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "w");
173  if(fp)
174  {
175  fprintf(fp, "ARTDAQ/*");
176  fprintf(fp, "\nConfigCore/*");
177  for(const auto& name : contextMemberNames)
178  fprintf(fp, "\nContextGroup/%s", name.c_str());
179  for(const auto& name : backboneMemberNames)
180  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
181  for(const auto& name : iterateMemberNames)
182  fprintf(fp, "\nIterateGroup/%s", name.c_str());
183  fclose(fp);
184  }
185  else
186  {
187  __SS__ << "Failed to open core table info file: "
188  << CORE_TABLE_INFO_FILENAME << __E__;
189  __SS_THROW__;
190  }
191  }
192  } // end dump names of core tables
193 
194  __GEN_COUTV__(runTimeSeconds());
195 } // end constructor
196 
197 //==============================================================================
204 const std::map<std::string, TableInfo>& ConfigurationManagerRW::getAllTableInfo(
205  bool refresh /* = false */,
206  std::string* accumulatedWarnings /* = 0 */,
207  const std::string& errorFilterName /* = "" */,
208  bool getGroupKeys /* = false */,
209  bool getGroupInfo /* = false */,
210  bool initializeActiveGroups /* = false */)
211 {
212  // allTableInfo_ is container to be returned
213 
214  if(!refresh)
215  return allTableInfo_;
216 
217  // else refresh!
218  allTableInfo_.clear();
219 
220  TableBase* table;
221 
222  // existing configurations are defined by which infos are in TABLE_INFO_PATH
223  // can test that the class exists based on this
224  // and then which versions
225  __GEN_COUT__ << "======================================================== "
226  "getAllTableInfo start runTimeSeconds()="
227  << runTimeSeconds() << __E__;
228  {
229  __GEN_COUT__ << "Refreshing all! Extracting list of tables..." << __E__;
230  DIR* pDIR;
231  struct dirent* entry;
232  std::string path = TABLE_INFO_PATH;
233  char fileExt[] = TABLE_INFO_EXT;
234  const unsigned char MIN_TABLE_NAME_SZ = 3;
235 
236  const int numOfThreads = PROCESSOR_COUNT / 2;
237  __GEN_COUT__ << " PROCESSOR_COUNT " << PROCESSOR_COUNT << " ==> " << numOfThreads
238  << " threads." << __E__;
239  if(numOfThreads < 2) // no multi-threading
240  {
241  if((pDIR = opendir(path.c_str())) != 0)
242  {
243  while((entry = readdir(pDIR)) != 0)
244  {
245  // enforce table name length
246  if(strlen(entry->d_name) < strlen(fileExt) + MIN_TABLE_NAME_SZ)
247  continue;
248 
249  // find file names with correct file extenstion
250  if(strcmp(&(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
251  fileExt) != 0)
252  continue; // skip different extentions
253 
254  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] =
255  '\0'; // remove file extension to get table name
256 
257  // 0 will force the creation of new instance (and reload from Info)
258  table = 0;
259 
260  try // only add valid table instances to maps
261  {
262  theInterface_->get(table,
263  entry->d_name,
264  0,
265  0,
266  true); // dont fill
267  }
268  catch(cet::exception const&)
269  {
270  if(table)
271  delete table;
272  table = 0;
273 
274  __GEN_COUT__ << "Skipping! No valid class found for... "
275  << entry->d_name << "\n";
276  continue;
277  }
278  catch(std::runtime_error& e)
279  {
280  if(table)
281  delete table;
282  table = 0;
283 
284  __GEN_COUT__ << "Skipping! No valid class found for... "
285  << entry->d_name << "\n";
286  __GEN_COUT__ << "Error: " << e.what() << __E__;
287 
288  // for a runtime_error, it is likely that columns are the problem
289  // the Table Editor needs to still fix these.. so attempt to
290  // proceed.
291  if(accumulatedWarnings)
292  {
293  if(errorFilterName == "" || errorFilterName == entry->d_name)
294  {
295  *accumulatedWarnings += std::string("\nIn table '") +
296  entry->d_name + "'..." +
297  e.what(); // global accumulate
298 
299  __SS__ << "Attempting to allow illegal columns!" << __E__;
300  *accumulatedWarnings += ss.str();
301  }
302 
303  // attempt to recover and build a mock-up
304  __GEN_COUT__ << "Attempting to allow illegal columns!"
305  << __E__;
306 
307  std::string returnedAccumulatedErrors;
308  try
309  {
310  table = new TableBase(entry->d_name,
311  &returnedAccumulatedErrors);
312  }
313  catch(...)
314  {
315  __GEN_COUT__ << "Skipping! Allowing illegal columns "
316  "didn't work either... "
317  << entry->d_name << "\n";
318  continue;
319  }
320  __GEN_COUT__
321  << "Error (but allowed): " << returnedAccumulatedErrors
322  << __E__;
323 
324  if(errorFilterName == "" || errorFilterName == entry->d_name)
325  *accumulatedWarnings +=
326  std::string("\nIn table '") + entry->d_name + "'..." +
327  returnedAccumulatedErrors; // global accumulate
328  }
329  else
330  continue;
331  }
332 
333  if(nameToTableMap_[entry->d_name]) // handle if instance existed
334  {
335  // copy the existing temporary versions! (or else all is lost)
336  std::set<TableVersion> versions =
337  nameToTableMap_[entry->d_name]->getStoredVersions();
338  for(auto& version : versions)
339  if(version.isTemporaryVersion())
340  {
341  try // do NOT let TableView::init() throw here
342  {
343  nameToTableMap_[entry->d_name]->setActiveView(
344  version);
345  table->copyView( // this calls TableView::init()
346  nameToTableMap_[entry->d_name]->getView(),
347  version,
348  username_);
349  }
350  catch(
351  ...) // do NOT let invalid temporary version throw at this
352  // point
353  {
354  } // just trust configurationBase throws out the failed version
355  }
356 
357  delete nameToTableMap_[entry->d_name];
358  nameToTableMap_[entry->d_name] = 0;
359  }
360 
361  nameToTableMap_[entry->d_name] = table;
362 
363  allTableInfo_[entry->d_name].tablePtr_ = table;
364  allTableInfo_[entry->d_name].versions_ =
365  theInterface_->getVersions(table);
366 
367  // also add any existing temporary versions to all table info
368  // because the interface wont find those versions
369  std::set<TableVersion> versions =
370  nameToTableMap_[entry->d_name]->getStoredVersions();
371  for(auto& version : versions)
372  if(version.isTemporaryVersion())
373  {
374  allTableInfo_[entry->d_name].versions_.emplace(version);
375  }
376  } //end table name handling from directory
377  closedir(pDIR);
378  }
379  }
380  else //multi-threading
381  {
382  int threadsLaunched = 0;
383  int foundThreadIndex = 0;
384  std::string tableName;
385 
386  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
387  for(int i = 0; i < numOfThreads; ++i)
388  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
389 
390  std::vector<std::shared_ptr<ots::TableInfo>> sharedTableInfoPtrs;
391 
392  if((pDIR = opendir(path.c_str())) != 0)
393  {
394  while((entry = readdir(pDIR)) != 0)
395  {
396  // enforce table name length
397  if(strlen(entry->d_name) < strlen(fileExt) + MIN_TABLE_NAME_SZ)
398  continue;
399 
400  // find file names with correct file extenstion
401  if(strcmp(&(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
402  fileExt) != 0)
403  continue; // skip different extentions
404 
405  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] =
406  '\0'; // remove file extension to get table name
407  tableName =
408  entry
409  ->d_name; //copy immediate, thread does not like using pointer to char*, corrupts immediately on wraparound
410 
411  //make temporary table info for thread
412  sharedTableInfoPtrs.push_back(std::make_shared<ots::TableInfo>());
413  sharedTableInfoPtrs.back()->accumulatedWarnings_ =
414  accumulatedWarnings ? "ALLOW"
415  : ""; //mark to allow accumulated warnings
416 
417  if(threadsLaunched >= numOfThreads)
418  {
419  //find availableThreadIndex
420  foundThreadIndex = -1;
421  size_t ii = 0;
422  while(foundThreadIndex == -1)
423  {
424  for(int i = 0; i < numOfThreads; ++i)
425  if(*(threadDone[i]))
426  {
427  foundThreadIndex = i;
428  break;
429  }
430  if(foundThreadIndex == -1)
431  {
432  __GEN_COUT_TYPE__(TLVL_DEBUG + 2)
433  << __COUT_HDR__
434  << "Waiting for available thread... iteration # "
435  << ii << __E__;
436  if(++ii > 100 /* 1s */ * 10)
437  {
438  __GEN_SS__ << "Threads seem to be stuck getting "
439  "table info! Timeout while waiting..."
440  << __E__;
441  __GEN_SS_THROW__;
442  }
443  usleep(10000);
444  }
445  } //end thread search loop
446  threadsLaunched = numOfThreads - 1;
447  }
448  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex
449  << " for table " << tableName << __E__;
450 
451  *(threadDone[foundThreadIndex]) = false;
452  std::thread(
453  [](ConfigurationManagerRW* theCfgMgr,
454  std::string theTableName,
455  TableBase* existingTable,
456  std::shared_ptr<ots::TableInfo> theTableInfo,
457  std::shared_ptr<std::atomic<bool>> theThreadDone) {
458  ConfigurationManagerRW::loadTableInfoThread(theCfgMgr,
459  theTableName,
460  existingTable,
461  theTableInfo,
462  theThreadDone);
463  },
464  this,
465  tableName,
466  nameToTableMap_[tableName],
467  sharedTableInfoPtrs.back(),
468  threadDone[foundThreadIndex])
469  .detach();
470 
471  ++threadsLaunched;
472  ++foundThreadIndex;
473  } //end table name handling from directory
474  closedir(pDIR);
475  } //end tableInfo thread loop
476 
477  //check for all threads done
478  do
479  {
480  foundThreadIndex = -1;
481  for(int i = 0; i < numOfThreads; ++i)
482  if(!*(threadDone[i]))
483  {
484  foundThreadIndex = i;
485  break;
486  }
487  if(foundThreadIndex != -1)
488  {
489  __GEN_COUTT__ << "Waiting for thread to finish... "
490  << foundThreadIndex << __E__;
491  usleep(10000);
492  }
493  } while(foundThreadIndex != -1); //end thread done search loop
494 
495  //threads done now, so copy table info
496  for(auto& tableInfo : sharedTableInfoPtrs)
497  {
498  __GEN_COUT_TYPE__(TLVL_DEBUG + 3)
499  << __COUT_HDR__ << "Copying table info for "
500  << tableInfo->tablePtr_->getTableName() << __E__;
501  nameToTableMap_[tableInfo->tablePtr_->getTableName()] =
502  tableInfo->tablePtr_;
503  allTableInfo_[tableInfo->tablePtr_->getTableName()].tablePtr_ =
504  tableInfo->tablePtr_;
505  allTableInfo_[tableInfo->tablePtr_->getTableName()].versions_ =
506  tableInfo->versions_;
507  } //end copy group info loop
508  }
509  __GEN_COUT__ << "Extracting list of tables complete." << __E__;
510  } //end Extracting list of tables
511 
512  // call init to load active versions by default, activate with warnings allowed (assuming development going on)
513  if(initializeActiveGroups)
514  {
515  __GEN_COUT__ << "Now initializing..." << __E__;
516  // if there is a filter name, do not include init warnings (it just scares people in the table editor)
517  std::string tmpAccumulateWarnings;
518  init(0 /*accumulatedErrors*/,
519  false /*initForWriteAccess*/,
520  accumulatedWarnings ? &tmpAccumulateWarnings : nullptr);
521 
522  if(accumulatedWarnings && errorFilterName == "")
523  *accumulatedWarnings += tmpAccumulateWarnings;
524  }
525  __GEN_COUT__ << "======================================================== "
526  "getAllTableInfo end runTimeSeconds()="
527  << runTimeSeconds() << __E__;
528 
529  // get Group Info too!
530  if(getGroupKeys || getGroupInfo)
531  {
532  allGroupInfo_.clear();
533  try
534  {
535  // build allGroupInfo_ for the ConfigurationManagerRW
536 
537  std::set<std::string /*name*/> tableGroups =
538  theInterface_->getAllTableGroupNames();
539  __GEN_COUT__ << "Number of Groups: " << tableGroups.size() << __E__;
540 
541  __GEN_COUTT__ << "Group Info start runTimeSeconds()=" << runTimeSeconds()
542  << __E__;
543 
544  TableGroupKey key;
545  std::string name;
546  for(const auto& fullName : tableGroups)
547  {
548  TableGroupKey::getGroupNameAndKey(fullName, name, key);
549  allGroupInfo_[name].keys_.emplace(key); //store in cache
550  }
551 
552  __GEN_COUTT__ << "Group Keys end runTimeSeconds()=" << runTimeSeconds()
553  << __E__;
554 
555  // for each group get member map & comment, author, time, and type for latest key
556  if(getGroupInfo)
557  {
558  const int numOfThreads = PROCESSOR_COUNT / 2;
559  __GEN_COUT__ << " PROCESSOR_COUNT " << PROCESSOR_COUNT << " ==> "
560  << numOfThreads << " threads." << __E__;
561  if(numOfThreads < 2) // no multi-threading
562  for(auto& groupInfo : allGroupInfo_)
563  {
564  try
565  {
566  groupInfo.second.latestKey_ = groupInfo.second.getLastKey();
567  loadTableGroup(
568  groupInfo.first /*groupName*/,
569  groupInfo.second.latestKey_,
570  false /*doActivate*/,
571  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
572  0 /*progressBar*/,
573  0 /*accumulateErrors*/,
574  &groupInfo.second.latestKeyGroupComment_,
575  &groupInfo.second.latestKeyGroupAuthor_,
576  &groupInfo.second.latestKeyGroupCreationTime_,
577  true /*doNotLoadMember*/,
578  &groupInfo.second.latestKeyGroupTypeString_);
579  }
580  catch(...)
581  {
582  __GEN_COUT_WARN__ << "Error occurred loading latest group "
583  "info into cache for '"
584  << groupInfo.first << "("
585  << groupInfo.second.latestKey_ << ")'..."
586  << __E__;
587  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
588  groupInfo.second.latestKeyGroupComment_ =
589  ConfigurationManager::UNKNOWN_INFO;
590  groupInfo.second.latestKeyGroupAuthor_ =
591  ConfigurationManager::UNKNOWN_INFO;
592  groupInfo.second.latestKeyGroupCreationTime_ =
593  ConfigurationManager::UNKNOWN_TIME;
594  groupInfo.second.latestKeyGroupTypeString_ =
595  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
596  groupInfo.second.latestKeyMemberMap_ = {};
597  }
598  } // end group info loop
599  else //multi-threading
600  {
601  int threadsLaunched = 0;
602  int foundThreadIndex = 0;
603 
604  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
605  for(int i = 0; i < numOfThreads; ++i)
606  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
607 
608  std::vector<std::shared_ptr<ots::GroupInfo>> sharedGroupInfoPtrs;
609 
610  for(auto& groupInfo : allGroupInfo_)
611  {
612  //make temporary group info for thread
613  sharedGroupInfoPtrs.push_back(std::make_shared<ots::GroupInfo>());
614 
615  if(threadsLaunched >= numOfThreads)
616  {
617  //find availableThreadIndex
618  foundThreadIndex = -1;
619  while(foundThreadIndex == -1)
620  {
621  for(int i = 0; i < numOfThreads; ++i)
622  if(*(threadDone[i]))
623  {
624  foundThreadIndex = i;
625  break;
626  }
627  if(foundThreadIndex == -1)
628  {
629  __GEN_COUTT__ << "Waiting for available thread..."
630  << __E__;
631  usleep(10000);
632  }
633  } //end thread search loop
634  threadsLaunched = numOfThreads - 1;
635  }
636  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex
637  << " for " << groupInfo.first << "("
638  << groupInfo.second.getLastKey() << ")" << __E__;
639 
640  *(threadDone[foundThreadIndex]) = false;
641 
642  std::thread(
643  [](ConfigurationManagerRW* theCfgMgr,
644  std::string theGroupName,
645  ots::TableGroupKey theGroupKey,
646  std::shared_ptr<ots::GroupInfo> theGroupInfo,
647  std::shared_ptr<std::atomic<bool>> theThreadDone) {
648  ConfigurationManagerRW::loadTableGroupThread(
649  theCfgMgr,
650  theGroupName,
651  theGroupKey,
652  theGroupInfo,
653  theThreadDone);
654  },
655  this,
656  groupInfo.first,
657  groupInfo.second.getLastKey(),
658  sharedGroupInfoPtrs.back(),
659  threadDone[foundThreadIndex])
660  .detach();
661 
662  ++threadsLaunched;
663  ++foundThreadIndex;
664  } //end groupInfo thread loop
665 
666  //check for all threads done
667  do
668  {
669  foundThreadIndex = -1;
670  for(int i = 0; i < numOfThreads; ++i)
671  if(!*(threadDone[i]))
672  {
673  foundThreadIndex = i;
674  break;
675  }
676  if(foundThreadIndex != -1)
677  {
678  __GEN_COUTT__ << "Waiting for thread to finish... "
679  << foundThreadIndex << __E__;
680  usleep(10000);
681  }
682  } while(foundThreadIndex != -1); //end thread done search loop
683 
684  //threads done now, so copy group info
685  size_t i = 0;
686  for(auto& groupInfo : allGroupInfo_)
687  {
688  groupInfo.second.latestKey_ = sharedGroupInfoPtrs[i]->latestKey_;
689  groupInfo.second.latestKeyGroupComment_ =
690  sharedGroupInfoPtrs[i]->latestKeyGroupComment_;
691  groupInfo.second.latestKeyGroupAuthor_ =
692  sharedGroupInfoPtrs[i]->latestKeyGroupAuthor_;
693  groupInfo.second.latestKeyGroupCreationTime_ =
694  sharedGroupInfoPtrs[i]->latestKeyGroupCreationTime_;
695  groupInfo.second.latestKeyGroupTypeString_ =
696  sharedGroupInfoPtrs[i]->latestKeyGroupTypeString_;
697  groupInfo.second.latestKeyMemberMap_ =
698  sharedGroupInfoPtrs[i]->latestKeyMemberMap_;
699  ++i;
700  } //end copy group info loop
701 
702  } //end multi-thread handling
703  }
704  } // end get group info
705  catch(const std::runtime_error& e)
706  {
707  __SS__
708  << "A fatal error occurred reading the info for all table groups. Error: "
709  << e.what() << __E__;
710  __GEN_COUT_ERR__ << "\n" << ss.str();
711  if(accumulatedWarnings)
712  *accumulatedWarnings += ss.str();
713  else
714  throw;
715  }
716  catch(...)
717  {
718  __SS__ << "An unknown fatal error occurred reading the info for all table "
719  "groups."
720  << __E__;
721  try
722  {
723  throw;
724  } //one more try to printout extra info
725  catch(const std::exception& e)
726  {
727  ss << "Exception message: " << e.what();
728  }
729  catch(...)
730  {
731  }
732  __GEN_COUT_ERR__ << "\n" << ss.str();
733  if(accumulatedWarnings)
734  *accumulatedWarnings += ss.str();
735  else
736  throw;
737  }
738  __GEN_COUTT__ << "Group Info end runTimeSeconds()=" << runTimeSeconds() << __E__;
739  } //end getGroupInfo
740  else
741  __GEN_COUTT__ << "Table Info end runTimeSeconds()=" << runTimeSeconds() << __E__;
742 
743  return allTableInfo_;
744 } // end getAllTableInfo()
745 
746 //==============================================================================
748 void ConfigurationManagerRW::loadTableInfoThread(
749  ConfigurationManagerRW* cfgMgr,
750  std::string tableName,
751  TableBase* existingTable,
752  std::shared_ptr<ots::TableInfo> tableInfo,
753  std::shared_ptr<std::atomic<bool>> threadDone)
754 try
755 {
756  __COUTT__ << "Thread started... table " << tableName << __E__;
757 
758  // 0 will force the creation of new instance (and reload from Info)
759  tableInfo->tablePtr_ = 0;
760 
761  try // only add valid table instances to maps
762  {
763  cfgMgr->theInterface_->get(tableInfo->tablePtr_,
764  tableName,
765  0,
766  0,
767  true); // dont fill
768  }
769  catch(cet::exception const&)
770  {
771  if(tableInfo->tablePtr_)
772  delete tableInfo->tablePtr_;
773  tableInfo->tablePtr_ = 0;
774 
775  __COUT__ << "Skipping! No valid class found for... " << tableName << "\n";
776  *(threadDone) = true;
777  return;
778  }
779  catch(std::runtime_error& e)
780  {
781  if(tableInfo->tablePtr_)
782  delete tableInfo->tablePtr_;
783  tableInfo->tablePtr_ = 0;
784 
785  __COUT__ << "Skipping! No valid class found for... " << tableName << "\n";
786  __COUTT__ << "Error: " << e.what() << __E__;
787 
788  // for a runtime_error, it is likely that columns are the problem
789  // the Table Editor needs to still fix these.. so attempt to
790  // proceed.
791  if(tableInfo->accumulatedWarnings_ == "ALLOW")
792  {
793  tableInfo->accumulatedWarnings_ = "";
794  if(1) //errorFilterName == "" || errorFilterName == tableName)
795  {
796  tableInfo->accumulatedWarnings_ += std::string("\nIn table '") +
797  tableName + "'..." +
798  e.what(); // global accumulate
799 
800  __SS__ << "Attempting to allow illegal columns!" << __E__;
801  tableInfo->accumulatedWarnings_ += ss.str();
802  }
803 
804  // attempt to recover and build a mock-up
805  __COUT__ << "Attempting to allow illegal columns!" << __E__;
806 
807  std::string returnedAccumulatedErrors;
808  try
809  {
810  tableInfo->tablePtr_ =
811  new TableBase(tableName, &returnedAccumulatedErrors);
812  }
813  catch(...)
814  {
815  __COUT__ << "Skipping! Allowing illegal columns didn't work either... "
816  << tableName << "\n";
817  *(threadDone) = true;
818  return;
819  }
820  __COUT_WARN__ << "Error (but allowed): " << returnedAccumulatedErrors
821  << __E__;
822 
823  if(1) //errorFilterName == "" || errorFilterName == entry->d_name)
824  tableInfo->accumulatedWarnings_ +=
825  std::string("\nIn table '") + tableName + "'..." +
826  returnedAccumulatedErrors; // global accumulate
827  }
828  else
829  {
830  tableInfo->accumulatedWarnings_ = "";
831  *(threadDone) = true;
832  return;
833  }
834  }
835 
836  if(existingTable) // handle if instance existed
837  {
838  __COUTT__ << "Copying temporary version from existing table object for "
839  << tableName << __E__;
840  // copy the existing temporary versions! (or else all is lost)
841  std::set<TableVersion> versions = existingTable->getStoredVersions();
842  for(auto& version : versions)
843  if(version.isTemporaryVersion())
844  {
845  try // do NOT let TableView::init() throw here
846  {
847  existingTable->setActiveView(version);
848  tableInfo->tablePtr_->copyView( // this calls TableView::init()
849  existingTable->getView(),
850  version,
851  cfgMgr->username_);
852  }
853  catch(...) // do NOT let invalid temporary version throw at this
854  // point
855  {
856  } // just trust configurationBase throws out the failed version
857  }
858 
859  delete existingTable;
860  existingTable = 0;
861  }
862 
863  tableInfo->versions_ = cfgMgr->theInterface_->getVersions(tableInfo->tablePtr_);
864 
865  // also add any existing temporary versions to all table info
866  // because the interface wont find those versions
867  std::set<TableVersion> versions = tableInfo->tablePtr_->getStoredVersions();
868  for(auto& version : versions)
869  if(version.isTemporaryVersion())
870  {
871  tableInfo->versions_.emplace(version);
872  }
873 
874  __COUTT__ << "Thread done... table " << tableName << __E__;
875  *(threadDone) = true;
876 } // end loadTableInfoThread
877 catch(...)
878 {
879  __COUT_ERR__ << "Error occurred loading latest table info into cache for '"
880  << tableName << "'..." << __E__;
881  *(threadDone) = true;
882 } // end loadTableInfoThread catch
883 
884 //==============================================================================
887 void ConfigurationManagerRW::loadTableGroup(
888  const std::string& groupName,
889  const TableGroupKey& groupKey,
890  bool doActivate /*=false*/,
891  std::map<std::string /*table name*/, TableVersion>*
892  groupMembers /*=0 , note: db time intensive! */,
893  ProgressBar* progressBar /*=0*/,
894  std::string* accumulatedWarnings /*=0*/,
895  std::string* groupComment /*=0 , note: in metadata */,
896  std::string* groupAuthor /*=0 , note: in metadata */,
897  std::string* groupCreateTime /*=0 , note: in metadata */,
898  bool doNotLoadMembers /*=false*/,
899  std::string* groupTypeString /*=0 , note: db time intensive! */,
900  std::map<std::string /*name*/, std::string /*alias*/>*
901  groupAliases /*=0 , note: in metadata */,
902  ConfigurationManager::LoadGroupType
903  groupTypeToLoad /*=ConfigurationManager::LoadGroupType::ALL_TYPES*/,
904  bool ignoreVersionTracking /*=false*/)
905 {
906  ConfigurationManager::loadTableGroup(groupName,
907  groupKey,
908  doActivate,
909  groupMembers,
910  progressBar,
911  accumulatedWarnings,
912  groupComment,
913  groupAuthor,
914  groupCreateTime,
915  doNotLoadMembers,
916  groupTypeString,
917  groupAliases,
918  groupTypeToLoad,
919  ignoreVersionTracking);
920 
921  if(!groupMembers || !groupMembers->size() || !groupComment || groupKey.isInvalid() ||
922  !groupAuthor || !groupCreateTime)
923  return;
924 
925  //treat successfull load as latest group key
926  auto groupInfo = allGroupInfo_.find(groupName);
927  if(groupInfo == allGroupInfo_.end())
928  return; //ignore if no group info cache
929 
930  groupInfo->second.latestKey_ = groupKey;
931  groupInfo->second.latestKeyGroupComment_ = *groupComment;
932  groupInfo->second.latestKeyGroupAuthor_ = *groupAuthor;
933  groupInfo->second.latestKeyGroupCreationTime_ = *groupCreateTime;
934  if(groupTypeString) //assume unlikely to change types
935  groupInfo->second.latestKeyGroupTypeString_ = *groupTypeString;
936  groupInfo->second.latestKeyMemberMap_ = *groupMembers;
937 
938 } //end loadTableGroup() RW version
939 
940 //==============================================================================
942 void ConfigurationManagerRW::loadTableGroupThread(
943  ConfigurationManagerRW* cfgMgr,
944  std::string groupName,
945  ots::TableGroupKey groupKey,
946  std::shared_ptr<ots::GroupInfo> groupInfo,
947  std::shared_ptr<std::atomic<bool>> threadDone)
948 try
949 {
950  __COUTT__ << "Thread started... " << groupName << "(" << groupKey << ")" << __E__;
951 
952  groupInfo->latestKey_ = groupKey;
953  cfgMgr->loadTableGroup(groupName,
954  groupKey,
955  false /*doActivate*/,
956  &(groupInfo->latestKeyMemberMap_) /*groupMembers*/,
957  0 /*progressBar*/,
958  0 /*accumulateErrors*/,
959  &(groupInfo->latestKeyGroupComment_),
960  &(groupInfo->latestKeyGroupAuthor_),
961  &(groupInfo->latestKeyGroupCreationTime_),
962  true /*doNotLoadMember*/,
963  &(groupInfo->latestKeyGroupTypeString_));
964 
965  *(threadDone) = true;
966 } // end loadTableGroupThread
967 catch(...)
968 {
969  __COUT_WARN__ << "Error occurred loading latest group info into cache for '"
970  << groupName << "(" << groupInfo->latestKey_ << ")'..." << __E__;
971  groupInfo->latestKey_ = TableGroupKey::INVALID;
972  groupInfo->latestKeyGroupComment_ = ConfigurationManager::UNKNOWN_INFO;
973  groupInfo->latestKeyGroupAuthor_ = ConfigurationManager::UNKNOWN_INFO;
974  groupInfo->latestKeyGroupCreationTime_ = ConfigurationManager::UNKNOWN_TIME;
975  groupInfo->latestKeyGroupTypeString_ = ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
976  groupInfo->latestKeyMemberMap_ = {};
977  *(threadDone) = true;
978 } // end loadTableGroupThread catch
979 
980 //==============================================================================
984 void ConfigurationManagerRW::compareTableGroupThread(
985  ConfigurationManagerRW* cfgMgr,
986  std::string groupName,
987  ots::TableGroupKey groupKeyToCompare,
988  const std::map<std::string, TableVersion>& groupMemberMap,
989  const std::map<std::string /*name*/, std::string /*alias*/>& memberTableAliases,
990  std::atomic<bool>* foundIdentical,
991  ots::TableGroupKey* identicalKey,
992  std::mutex* threadMutex,
993  std::shared_ptr<std::atomic<bool>> threadDone)
994 try
995 {
996  std::map<std::string /*name*/, TableVersion /*version*/> compareToMemberMap;
997  std::map<std::string /*name*/, std::string /*alias*/> compareToMemberTableAliases;
998  std::map<std::string /*name*/, std::string /*alias*/>*
999  compareToMemberTableAliasesPtr = nullptr;
1000  if(memberTableAliases
1001  .size()) //only give pointer if necessary, without will load group faster
1002  compareToMemberTableAliasesPtr = &compareToMemberTableAliases;
1003 
1004  cfgMgr->loadTableGroup(groupName,
1005  groupKeyToCompare,
1006  false /*doActivate*/,
1007  &compareToMemberMap /*memberMap*/,
1008  0, /*progressBar*/
1009  0, /*accumulatedWarnings*/
1010  0, /*groupComment*/
1011  0,
1012  0, /*null pointers*/
1013  true /*doNotLoadMember*/,
1014  0 /*groupTypeString*/,
1015  compareToMemberTableAliasesPtr);
1016 
1017  bool debug = false;
1018  if(TTEST(9)) // = DEBUG+9
1019  {
1020  debug = true;
1021  for(auto& memberPair : groupMemberMap)
1022  __COUTS__(9) << "member " << memberPair.first << " (" << memberPair.second
1023  << ")" << __E__;
1024  for(auto& memberPair : compareToMemberMap)
1025  __COUTS__(9) << "compare " << groupName << " (" << groupKeyToCompare
1026  << ") member:" << memberPair.first << " (" << memberPair.second
1027  << ")" << __E__;
1028  }
1029 
1030  bool isDifferent = false;
1031  for(auto& memberPair : groupMemberMap)
1032  {
1033  if(memberTableAliases.find(memberPair.first) != memberTableAliases.end())
1034  {
1035  // handle this table as alias, not version
1036  if(compareToMemberTableAliases.find(memberPair.first) ==
1037  compareToMemberTableAliases.end() || // alias is missing
1038  memberTableAliases.at(memberPair.first) !=
1039  compareToMemberTableAliases.at(memberPair.first))
1040  { // then different
1041  isDifferent = true;
1042  if(debug)
1043  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1044  << ") on alias " << memberPair.first << __E__;
1045  break;
1046  }
1047  else
1048  continue;
1049  } // else check if compareTo group is using an alias for table
1050  else if(compareToMemberTableAliases.find(memberPair.first) !=
1051  compareToMemberTableAliases.end())
1052  {
1053  // then different
1054  isDifferent = true;
1055  if(debug)
1056  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1057  << ") on reverse alias " << memberPair.first << __E__;
1058  break;
1059 
1060  } // else handle as table version comparison
1061  else if(compareToMemberMap.find(memberPair.first) ==
1062  compareToMemberMap.end() || // name is missing
1063  memberPair.second !=
1064  compareToMemberMap.at(memberPair.first)) // or version mismatch
1065  {
1066  // then different
1067  isDifferent = true;
1068  if(debug)
1069  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare
1070  << ") on mismatch " << memberPair.first << __E__;
1071  break;
1072  }
1073  }
1074 
1075  // check member size for exact match
1076  if(!isDifferent &&
1077  groupMemberMap.size() !=
1078  compareToMemberMap
1079  .size()) // different size, so not same (groupMemberMap is a subset of memberPairs)
1080  {
1081  isDifferent = true;
1082 
1083  if(debug)
1084  __COUTT__ << "diff " << groupName << " (" << groupKeyToCompare << ") on size "
1085  << __E__;
1086  }
1087 
1088  if(!isDifferent) //found an exact match!
1089  {
1090  *foundIdentical = true;
1091  __COUT__ << "=====> Found exact match with key: " << groupKeyToCompare << __E__;
1092 
1093  std::lock_guard<std::mutex> lock(*threadMutex);
1094  *identicalKey = groupKeyToCompare;
1095  }
1096 
1097  *(threadDone) = true;
1098 } // end compareTableGroupThread
1099 catch(...)
1100 {
1101  __COUT_WARN__ << "Error occurred comparing group '" << groupName << "("
1102  << groupKeyToCompare << ")'..." << __E__;
1103 
1104  *(threadDone) = true;
1105 } // end compareTableGroupThread catch
1106 
1107 //==============================================================================
1111 std::map<std::string /*table name*/,
1112  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
1113 ConfigurationManagerRW::getVersionAliases(void) const
1114 {
1115  std::map<std::string /*table name*/,
1116  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
1117  retMap = ConfigurationManager::getVersionAliases();
1118 
1119  // always have scratch alias for each table that has a scratch version
1120  // overwrite map entry if necessary
1121  if(!ConfigurationInterface::isVersionTrackingEnabled())
1122  for(const auto& tableInfo : allTableInfo_)
1123  for(const auto& version : tableInfo.second.versions_)
1124  if(version.isScratchVersion())
1125  retMap[tableInfo.first][ConfigurationManager::SCRATCH_VERSION_ALIAS] =
1126  TableVersion(TableVersion::SCRATCH);
1127 
1128  return retMap;
1129 } // end getVersionAliases()
1130 
1131 //==============================================================================
1135 void ConfigurationManagerRW::activateTableGroup(const std::string& tableGroupName,
1136  TableGroupKey tableGroupKey,
1137  std::string* accumulatedTreeErrors,
1138  std::string* groupTypeString)
1139 {
1140  try
1141  {
1142  loadTableGroup(tableGroupName,
1143  tableGroupKey,
1144  true, // loads and activates
1145  0, // no members needed
1146  0, // no progress bar
1147  accumulatedTreeErrors, // accumulate warnings or not
1148  0 /* groupComment */,
1149  0 /* groupAuthor */,
1150  0 /* groupCreateTime */,
1151  false /* doNotLoadMember */,
1152  groupTypeString);
1153  }
1154  catch(...)
1155  {
1156  __GEN_COUT_ERR__ << "There were errors, so de-activating group: "
1157  << tableGroupName << " (" << tableGroupKey << ")" << __E__;
1158  try // just in case any lingering pieces, let's deactivate
1159  {
1160  destroyTableGroup(tableGroupName, true);
1161  }
1162  catch(...)
1163  {
1164  }
1165  throw; // re-throw original exception
1166  }
1167 
1168  __GEN_COUT_INFO__ << "Updating persistent active groups to "
1169  << ConfigurationManager::ACTIVE_GROUPS_FILENAME << " ..." << __E__;
1170 
1171  __COUT_INFO__ << "Active Context table group: " << theContextTableGroup_ << "("
1172  << (theContextTableGroupKey_
1173  ? theContextTableGroupKey_->toString().c_str()
1174  : "-1")
1175  << ")" << __E__;
1176  __COUT_INFO__ << "Active Backbone table group: " << theBackboneTableGroup_ << "("
1177  << (theBackboneTableGroupKey_
1178  ? theBackboneTableGroupKey_->toString().c_str()
1179  : "-1")
1180  << ")" << __E__;
1181  __COUT_INFO__ << "Active Iterate table group: " << theIterateTableGroup_ << "("
1182  << (theIterateTableGroupKey_
1183  ? theIterateTableGroupKey_->toString().c_str()
1184  : "-1")
1185  << ")" << __E__;
1186  __COUT_INFO__ << "Active Configuration table group: " << theConfigurationTableGroup_
1187  << "("
1188  << (theConfigurationTableGroupKey_
1189  ? theConfigurationTableGroupKey_->toString().c_str()
1190  : "-1")
1191  << ")" << __E__;
1192 
1193  std::string fn = ConfigurationManager::ACTIVE_GROUPS_FILENAME;
1194  FILE* fp = fopen(fn.c_str(), "w");
1195  if(!fp)
1196  {
1197  __SS__ << "Fatal Error! Unable to open the file "
1198  << ConfigurationManager::ACTIVE_GROUPS_FILENAME
1199  << " for editing! Is there a permissions problem?" << __E__;
1200  __GEN_COUT_ERR__ << ss.str();
1201  __SS_THROW__;
1202  return;
1203  }
1204  fprintf(fp, "%s\n", theContextTableGroup_.c_str());
1205  fprintf(
1206  fp,
1207  "%s\n",
1208  theContextTableGroupKey_ ? theContextTableGroupKey_->toString().c_str() : "-1");
1209  fprintf(fp, "%s\n", theBackboneTableGroup_.c_str());
1210  fprintf(
1211  fp,
1212  "%s\n",
1213  theBackboneTableGroupKey_ ? theBackboneTableGroupKey_->toString().c_str() : "-1");
1214  fprintf(fp, "%s\n", theIterateTableGroup_.c_str());
1215  fprintf(
1216  fp,
1217  "%s\n",
1218  theIterateTableGroupKey_ ? theIterateTableGroupKey_->toString().c_str() : "-1");
1219  fprintf(fp, "%s\n", theConfigurationTableGroup_.c_str());
1220  fprintf(fp,
1221  "%s\n",
1222  theConfigurationTableGroupKey_
1223  ? theConfigurationTableGroupKey_->toString().c_str()
1224  : "-1");
1225  fclose(fp);
1226 
1227  // save last activated group
1228  {
1229  std::pair<std::string /*group name*/, TableGroupKey> activatedGroup(
1230  std::string(tableGroupName), tableGroupKey);
1231  if(theConfigurationTableGroupKey_ &&
1232  theConfigurationTableGroup_ == tableGroupName &&
1233  *theConfigurationTableGroupKey_ == tableGroupKey)
1234  ConfigurationManager::saveGroupNameAndKey(activatedGroup,
1235  LAST_ACTIVATED_CONFIG_GROUP_FILE);
1236  else if(theContextTableGroupKey_ && theContextTableGroup_ == tableGroupName &&
1237  *theContextTableGroupKey_ == tableGroupKey)
1238  ConfigurationManager::saveGroupNameAndKey(activatedGroup,
1239  LAST_ACTIVATED_CONTEXT_GROUP_FILE);
1240  else if(theBackboneTableGroupKey_ && theBackboneTableGroup_ == tableGroupName &&
1241  *theBackboneTableGroupKey_ == tableGroupKey)
1242  ConfigurationManager::saveGroupNameAndKey(activatedGroup,
1243  LAST_ACTIVATED_BACKBONE_GROUP_FILE);
1244  else if(theIterateTableGroupKey_ && theIterateTableGroup_ == tableGroupName &&
1245  *theIterateTableGroupKey_ == tableGroupKey)
1246  ConfigurationManager::saveGroupNameAndKey(activatedGroup,
1247  LAST_ACTIVATED_ITERATOR_GROUP_FILE);
1248  } // end save last activated group
1249 
1250 } // end activateTableGroup()
1251 
1252 //==============================================================================
1256 TableVersion ConfigurationManagerRW::createTemporaryBackboneView(
1257  TableVersion sourceViewVersion)
1258 {
1259  __GEN_COUT_INFO__ << "Creating temporary backbone view from version "
1260  << sourceViewVersion << __E__;
1261 
1262  // find common available temporary version among backbone members
1263  TableVersion tmpVersion =
1264  TableVersion::getNextTemporaryVersion(); // get the default temporary version
1265  TableVersion retTmpVersion;
1266  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
1267  for(auto& name : backboneMemberNames)
1268  {
1269  retTmpVersion =
1270  ConfigurationManager::getTableByName(name)->getNextTemporaryVersion();
1271  if(retTmpVersion < tmpVersion)
1272  tmpVersion = retTmpVersion;
1273  }
1274 
1275  __GEN_COUT__ << "Common temporary backbone version found as " << tmpVersion << __E__;
1276 
1277  // create temporary views from source version to destination temporary version
1278  for(auto& name : backboneMemberNames)
1279  {
1280  retTmpVersion =
1281  getTableByName(name)->createTemporaryView(sourceViewVersion, tmpVersion);
1282  if(retTmpVersion != tmpVersion)
1283  {
1284  __SS__ << "Failure! Temporary view requested was " << tmpVersion
1285  << ". Mismatched temporary view created: " << retTmpVersion << __E__;
1286  __GEN_COUT_ERR__ << ss.str();
1287  __SS_THROW__;
1288  }
1289  }
1290 
1291  return tmpVersion;
1292 } // end createTemporaryBackboneView()
1293 
1294 //==============================================================================
1295 TableBase* ConfigurationManagerRW::getTableByName(const std::string& tableName)
1296 {
1297  if(nameToTableMap_.find(tableName) == nameToTableMap_.end())
1298  {
1299  if(tableName == ConfigurationManager::ARTDAQ_TOP_TABLE_NAME)
1300  {
1301  __GEN_COUT_WARN__
1302  << "Since target table was the artdaq top configuration level, "
1303  "attempting to help user by appending to core tables file: "
1304  << CORE_TABLE_INFO_FILENAME << __E__;
1305  FILE* fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "a");
1306  if(fp)
1307  {
1308  fprintf(fp, "\nARTDAQ/*");
1309  fclose(fp);
1310  }
1311  }
1312 
1313  __SS__ << "Table not found with name: " << tableName << __E__;
1314  size_t f;
1315  if((f = tableName.find(' ')) != std::string::npos)
1316  ss << "There was a space character found in the table name needle at "
1317  "position "
1318  << f << " in the string (was this intended?). " << __E__;
1319 
1320  ss << "\nIf you think this table should exist in the core set of tables, try "
1321  "running 'UpdateOTS.sh --tables' to update your tables, then relaunch ots."
1322  << __E__;
1323  ss << "\nTables must be defined at path $USER_DATA/TableInfo/ to exist in ots. "
1324  "Please verify your table definitions, and then restart ots."
1325  << __E__;
1326  __GEN_COUT_ERR__ << "\n" << ss.str();
1327  __SS_THROW__;
1328  }
1329  return nameToTableMap_[tableName];
1330 } // end getTableByName()
1331 
1332 //==============================================================================
1337 TableBase* ConfigurationManagerRW::getVersionedTableByName(
1338  const std::string& tableName,
1339  TableVersion version,
1340  bool looseColumnMatching /* = false */,
1341  std::string* accumulatedErrors /* = 0 */,
1342  bool getRawData /* = false */)
1343 {
1344  auto it = nameToTableMap_.find(tableName);
1345  if(it == nameToTableMap_.end())
1346  {
1347  __SS__ << "\nCan not find table named '" << tableName
1348  << "'\n\n\n\nYou need to load the table before it can be used."
1349  << "It probably is missing from the member list of the Table "
1350  "Group that was loaded?\n\n\n\n\n"
1351  << __E__;
1352  __SS_THROW__;
1353  }
1354  TableBase* table = it->second;
1355 
1356  if(version.isTemporaryVersion())
1357  {
1358  table->setActiveView(version);
1359 
1360  if(getRawData)
1361  {
1362  std::stringstream jsonSs;
1363  table->getViewP()->printJSON(jsonSs);
1364  table->getViewP()->doGetSourceRawData(true);
1365  table->getViewP()->fillFromJSON(jsonSs.str());
1366  }
1367  }
1368  else
1369  {
1370  theInterface_->get(table,
1371  tableName,
1372  0 /* groupKey */,
1373  0 /* groupName */,
1374  false /* dontFill */, // false to fill w/version
1375  version,
1376  false /* resetConfiguration*/, // false to not reset
1377  looseColumnMatching,
1378  getRawData,
1379  accumulatedErrors);
1380  }
1381  return table;
1382 } // end getVersionedTableByName()
1383 
1384 //==============================================================================
1387 TableVersion ConfigurationManagerRW::saveNewTable(const std::string& tableName,
1388  TableVersion temporaryVersion,
1389  bool makeTemporary) //,
1391 {
1392  TableVersion newVersion(temporaryVersion);
1393 
1394  // set author of version
1395  TableBase* table = getTableByName(tableName);
1396  table->getTemporaryView(temporaryVersion)->setAuthor(username_);
1397  // NOTE: author is assigned to permanent versions when saved to DBI
1398 
1399  if(!makeTemporary) // saveNewVersion makes the new version the active version
1400  newVersion = theInterface_->saveNewVersion(table, temporaryVersion);
1401  else // make the temporary version active
1402  table->setActiveView(newVersion);
1403 
1404  // if there is a problem, try to recover
1405  while(!makeTemporary && !newVersion.isScratchVersion() &&
1406  allTableInfo_[tableName].versions_.find(newVersion) !=
1407  allTableInfo_[tableName].versions_.end())
1408  {
1409  __GEN_COUT_ERR__
1410  << "What happenened!?? ERROR::: new persistent version v" << newVersion
1411  << " already exists!? How is it possible? Retrace your steps and "
1412  "tell an admin."
1413  << __E__;
1414 
1415  // create a new temporary version of the target view
1416  temporaryVersion = table->createTemporaryView(newVersion);
1417 
1418  if(newVersion.isTemporaryVersion())
1419  newVersion = temporaryVersion;
1420  else
1421  newVersion = TableVersion::getNextVersion(newVersion);
1422 
1423  __GEN_COUT_WARN__ << "Attempting to recover and use v" << newVersion << __E__;
1424 
1425  if(!makeTemporary) // saveNewVersion makes the new version the active version
1426  newVersion =
1427  theInterface_->saveNewVersion(table, temporaryVersion, newVersion);
1428  else // make the temporary version active
1429  table->setActiveView(newVersion);
1430  }
1431 
1432  if(newVersion.isInvalid())
1433  {
1434  __SS__ << "Something went wrong saving the new version v" << newVersion
1435  << ". What happened?! (duplicates? database error?)" << __E__;
1436  __GEN_COUT_ERR__ << "\n" << ss.str();
1437  __SS_THROW__;
1438  }
1439 
1440  // update allTableInfo_ with the new version
1441  allTableInfo_[tableName].versions_.insert(newVersion);
1442 
1443  // table->getView().print();
1444  return newVersion;
1445 } // end saveNewTable()
1446 
1447 //==============================================================================
1452 void ConfigurationManagerRW::eraseTemporaryVersion(const std::string& tableName,
1453  TableVersion targetVersion)
1454 {
1455  TableBase* table = getTableByName(tableName);
1456 
1457  table->trimTemporary(targetVersion);
1458 
1459  // if allTableInfo_ is not setup, then done
1460  if(allTableInfo_.find(tableName) == allTableInfo_.end())
1461  return;
1462  // else cleanup table info
1463 
1464  if(targetVersion.isInvalid())
1465  {
1466  // erase all temporary versions!
1467  for(auto it = allTableInfo_[tableName].versions_.begin();
1468  it != allTableInfo_[tableName].versions_.end();
1469  /*no increment*/)
1470  {
1471  if(it->isTemporaryVersion())
1472  {
1473  __GEN_COUT__ << "Removing '" << tableName << "' version info: " << *it
1474  << __E__;
1475  allTableInfo_[tableName].versions_.erase(it++);
1476  }
1477  else
1478  ++it;
1479  }
1480  }
1481  else // erase target version only
1482  {
1483  //__GEN_COUT__ << "Removing '" << tableName << "' version info: " << targetVersion << __E__;
1484  auto it = allTableInfo_[tableName].versions_.find(targetVersion);
1485  if(it == allTableInfo_[tableName].versions_.end())
1486  {
1487  __GEN_COUT__ << "Target '" << tableName << "' version v" << targetVersion
1488  << " was not found in info versions..." << __E__;
1489  return;
1490  }
1491  allTableInfo_[tableName].versions_.erase(
1492  allTableInfo_[tableName].versions_.find(targetVersion));
1493  }
1494 } // end eraseTemporaryVersion()
1495 
1496 //==============================================================================
1501 void ConfigurationManagerRW::clearCachedVersions(const std::string& tableName)
1502 {
1503  TableBase* table = getTableByName(tableName);
1504 
1505  table->trimCache(0);
1506 } // end clearCachedVersions()
1507 
1508 //==============================================================================
1513 void ConfigurationManagerRW::clearAllCachedVersions()
1514 {
1515  for(auto configInfo : allTableInfo_)
1516  configInfo.second.tablePtr_->trimCache(0);
1517 } // end clearAllCachedVersions()
1518 
1519 //==============================================================================
1521 TableVersion ConfigurationManagerRW::copyViewToCurrentColumns(
1522  const std::string& tableName, TableVersion sourceVersion)
1523 {
1524  getTableByName(tableName)->reset();
1525 
1526  // make sure source version is loaded
1527  // need to load with loose column rules!
1528  TableBase* table =
1529  getVersionedTableByName(tableName, TableVersion(sourceVersion), true);
1530 
1531  // copy from source version to a new temporary version
1532  TableVersion newTemporaryVersion =
1533  table->copyView(table->getView(), TableVersion(), username_);
1534 
1535  // update allTableInfo_ with the new version
1536  allTableInfo_[tableName].versions_.insert(newTemporaryVersion);
1537 
1538  return newTemporaryVersion;
1539 } // end copyViewToCurrentColumns()
1540 
1541 //==============================================================================
1545 const GroupInfo& ConfigurationManagerRW::getGroupInfo(
1546  const std::string& groupName, bool attemptToReloadKeys /* = false */)
1547 {
1548  // //NOTE: seems like this filter is taking the long amount of time
1549  // std::set<std::string /*name*/> fullGroupNames =
1550  // theInterface_->getAllTableGroupNames(groupName); //db filter by group name
1551 
1552  // so instead caching ourselves...
1553  auto it = allGroupInfo_.find(groupName);
1554  if(it == allGroupInfo_.end())
1555  {
1556  __SS__ << "Group name '" << groupName
1557  << "' not found in group info! (creating empty info)" << __E__;
1558  __GEN_COUT_WARN__ << ss.str();
1559  //__SS_THROW__;
1560  return allGroupInfo_[groupName];
1561  }
1562 
1563  if(attemptToReloadKeys) //load keys from Interface group cache
1564  {
1565  __GEN_COUT__ << "Reloading keys from special db group cache if it exists..."
1566  << __E__;
1567 
1568  std::set<TableGroupKey> keys;
1569  { //load keys from special db group cache (this avoids pre-cache filling and avoids long db lookup, unless speed table cache missing for this group)
1570  //attempt to use cache first! (potentially way faster .04 s vs 4 s)
1571  bool cacheFailed = false;
1572  try
1573  {
1574  TableBase localGroupMemberCacheLoader(
1575  true /*special table*/
1576  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),,
1577  TableBase::GROUP_CACHE_PREPEND + groupName);
1578  auto versions = theInterface_->getVersions(&localGroupMemberCacheLoader);
1579  for(const auto& version : versions)
1580  keys.emplace(TableGroupKey(version.version()));
1581  }
1582  catch(...)
1583  {
1584  __GEN_COUT__ << "Ignoring cache loading error. Doing full load of keys..."
1585  << __E__;
1586  cacheFailed = true;
1587  }
1588 
1589  if(cacheFailed && 0) //could consider full load if cache failed
1590  keys = theInterface_->getKeys(groupName);
1591 
1592  if(!cacheFailed) //take keys
1593  {
1594  __GEN_COUT__ << "Key from special db group cache were loaded." << __E__;
1595  it->second.keys_ = keys; //update ConfigManager cache!
1596  }
1597  }
1598  }
1599 
1600  return it->second;
1601 } // end getGroupInfo()
1602 
1603 //==============================================================================
1613 TableGroupKey ConfigurationManagerRW::findTableGroup(
1614  const std::string& groupName,
1615  const std::map<std::string, TableVersion>& groupMemberMap,
1616  const std::map<std::string /*name*/, std::string /*alias*/>& memberTableAliases)
1617 {
1618  // //NOTE: seems like this filter is taking the long amount of time
1619  // std::set<std::string /*name*/> fullGroupNames =
1620  // theInterface_->getAllTableGroupNames(groupName); //db filter bygroup name
1621  // const GroupInfo& groupInfo = getGroupInfo(groupName); // Note this also seems to take too long because requires a pre-cache load!
1622  std::set<TableGroupKey> keys;
1623  { //so instead load keys from special db group cache (this avoids pre-cache filling and avoids long db lookup, unless speed table cache missing for this group)
1624  //attempt to use cache first! (potentially way faster .04 s vs 4 s)
1625  bool cacheFailed = false;
1626  try
1627  {
1628  TableBase localGroupMemberCacheLoader(
1629  true /*special table*/
1630  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),,
1631  TableBase::GROUP_CACHE_PREPEND + groupName);
1632  auto versions = theInterface_->getVersions(&localGroupMemberCacheLoader);
1633  for(const auto& version : versions)
1634  keys.emplace(TableGroupKey(version.version()));
1635  }
1636  catch(...)
1637  {
1638  __COUT__ << "Ignoring cache loading error. Doing full load of keys..."
1639  << __E__;
1640  cacheFailed = true;
1641  }
1642 
1643  if(cacheFailed) //since cache failed, do full load
1644  keys = theInterface_->getKeys(groupName);
1645  }
1646 
1647  __COUTTV__(StringMacros::setToString(keys));
1648 
1649  const unsigned int MAX_DEPTH_TO_CHECK = 20;
1650  unsigned int keyMinToCheck = 0;
1651 
1652  if(keys.size())
1653  keyMinToCheck = keys.rbegin()->key();
1654  if(keyMinToCheck > MAX_DEPTH_TO_CHECK)
1655  {
1656  keyMinToCheck -= MAX_DEPTH_TO_CHECK;
1657  __GEN_COUT__ << "Checking groups back to key... " << keyMinToCheck << __E__;
1658  }
1659  else
1660  {
1661  keyMinToCheck = 0;
1662  __GEN_COUT__ << "Checking all groups." << __E__;
1663  }
1664 
1665  __GEN_COUTTV__(StringMacros::mapToString(groupMemberMap));
1666 
1667  // have min key to check, now loop through and check groups
1668 
1669  const int numOfThreads = PROCESSOR_COUNT / 2;
1670  __GEN_COUT__ << " PROCESSOR_COUNT " << PROCESSOR_COUNT << " ==> " << numOfThreads
1671  << " threads." << __E__;
1672  if(numOfThreads < 2) // no multi-threading
1673  {
1674  std::map<std::string /*name*/, TableVersion /*version*/> compareToMemberMap;
1675  std::map<std::string /*name*/, std::string /*alias*/> compareToMemberTableAliases;
1676  std::map<std::string /*name*/, std::string /*alias*/>*
1677  compareToMemberTableAliasesPtr = nullptr;
1678  if(memberTableAliases.size())
1679  compareToMemberTableAliasesPtr = &compareToMemberTableAliases;
1680 
1681  bool isDifferent;
1682  for(const auto& key : keys)
1683  {
1684  if(key.key() < keyMinToCheck)
1685  continue; // skip keys that are too old
1686 
1687  loadTableGroup(groupName,
1688  key,
1689  false /*doActivate*/,
1690  &compareToMemberMap /*memberMap*/,
1691  0, /*progressBar*/
1692  0, /*accumulatedWarnings*/
1693  0, /*groupComment*/
1694  0, /*groupAuthor*/
1695  0, /*groupCreateTime*/
1696  true /*doNotLoadMember*/,
1697  0 /*groupTypeString*/,
1698  compareToMemberTableAliasesPtr);
1699 
1700  isDifferent = false;
1701  for(auto& memberPair : groupMemberMap)
1702  {
1703  if(memberTableAliases.find(memberPair.first) != memberTableAliases.end())
1704  {
1705  // handle this table as alias, not version
1706  if(compareToMemberTableAliases.find(memberPair.first) ==
1707  compareToMemberTableAliases.end() || // alias is missing
1708  memberTableAliases.at(memberPair.first) !=
1709  compareToMemberTableAliases.at(memberPair.first))
1710  { // then different
1711  isDifferent = true;
1712  break;
1713  }
1714  else
1715  continue;
1716  } // else check if compareTo group is using an alias for table
1717  else if(compareToMemberTableAliases.find(memberPair.first) !=
1718  compareToMemberTableAliases.end())
1719  {
1720  // then different
1721  isDifferent = true;
1722  break;
1723 
1724  } // else handle as table version comparison
1725  else if(compareToMemberMap.find(memberPair.first) ==
1726  compareToMemberMap.end() || // name is missing
1727  memberPair.second !=
1728  compareToMemberMap.at(
1729  memberPair.first)) // or version mismatch
1730  {
1731  // then different
1732  isDifferent = true;
1733  break;
1734  }
1735  }
1736  if(isDifferent)
1737  continue;
1738 
1739  // check member size for exact match
1740  if(groupMemberMap.size() != compareToMemberMap.size())
1741  continue; // different size, so not same (groupMemberMap is a subset of
1742  // memberPairs)
1743 
1744  __GEN_COUT__ << "Found exact match with key: " << key << __E__;
1745  // else found an exact match!
1746  return key;
1747  }
1748  __GEN_COUT__ << "No match found - this group is new!" << __E__;
1749  // if here, then no match found
1750  return TableGroupKey(); // return invalid key
1751  }
1752  else //multi-threading
1753  {
1754  int threadsLaunched = 0;
1755  int foundThreadIndex = 0;
1756  std::atomic<bool> foundIdentical = false;
1757  ots::TableGroupKey identicalKey;
1758  std::mutex threadMutex;
1759 
1760  std::vector<std::shared_ptr<std::atomic<bool>>> threadDone;
1761  for(int i = 0; i < numOfThreads; ++i)
1762  threadDone.push_back(std::make_shared<std::atomic<bool>>(true));
1763 
1764  for(const auto& key : keys)
1765  {
1766  if(foundIdentical)
1767  break;
1768  if(key.key() < keyMinToCheck)
1769  continue; // skip keys that are too old
1770 
1771  if(threadsLaunched >= numOfThreads)
1772  {
1773  //find availableThreadIndex
1774  foundThreadIndex = -1;
1775  while(foundThreadIndex == -1)
1776  {
1777  if(foundIdentical)
1778  break;
1779 
1780  for(int i = 0; i < numOfThreads; ++i)
1781  if(*(threadDone[i]))
1782  {
1783  foundThreadIndex = i;
1784  break;
1785  }
1786  if(foundThreadIndex == -1)
1787  {
1788  __GEN_COUTT__ << "Waiting for available thread..." << __E__;
1789  usleep(10000);
1790  }
1791  } //end thread search loop
1792  threadsLaunched = numOfThreads - 1;
1793  }
1794  if(foundIdentical)
1795  break;
1796 
1797  __GEN_COUTT__ << "Starting thread... " << foundThreadIndex << __E__;
1798  *(threadDone[foundThreadIndex]) = false;
1799 
1800  std::thread(
1801  [](ConfigurationManagerRW* cfgMgr,
1802  std::string theGroupName,
1803  ots::TableGroupKey groupKeyToCompare,
1804  const std::map<std::string, TableVersion>& groupMemberMap,
1805  const std::map<std::string /*name*/, std::string /*alias*/>&
1806  memberTableAliases,
1807  std::atomic<bool>* theFoundIdentical,
1808  ots::TableGroupKey* theIdenticalKey,
1809  std::mutex* theThreadMutex,
1810  std::shared_ptr<std::atomic<bool>> theThreadDone) {
1811  ConfigurationManagerRW::compareTableGroupThread(cfgMgr,
1812  theGroupName,
1813  groupKeyToCompare,
1814  groupMemberMap,
1815  memberTableAliases,
1816  theFoundIdentical,
1817  theIdenticalKey,
1818  theThreadMutex,
1819  theThreadDone);
1820  },
1821  this,
1822  groupName,
1823  key,
1824  groupMemberMap,
1825  memberTableAliases,
1826  &foundIdentical,
1827  &identicalKey,
1828  &threadMutex,
1829  threadDone[foundThreadIndex])
1830  .detach();
1831 
1832  ++threadsLaunched;
1833  ++foundThreadIndex;
1834  } //end group key check thread loop
1835 
1836  //check for all threads done
1837  do
1838  {
1839  foundThreadIndex = -1;
1840  for(int i = 0; i < numOfThreads; ++i)
1841  if(!*(threadDone[i]))
1842  {
1843  foundThreadIndex = i;
1844  break;
1845  }
1846  if(foundThreadIndex != -1)
1847  {
1848  __GEN_COUTT__ << "Waiting for thread to finish... " << foundThreadIndex
1849  << __E__;
1850  usleep(10000);
1851  }
1852  } while(foundThreadIndex != -1); //end thread done search loop
1853 
1854  if(foundIdentical)
1855  {
1856  __GEN_COUT__ << "Found exact match with key: " << identicalKey << __E__;
1857  return identicalKey;
1858  }
1859 
1860  // if here, then no match found
1861  return TableGroupKey(); // return invalid key
1862  } //end multi-thread handling
1863 } // end findTableGroup()
1864 
1865 //==============================================================================
1870 TableBase* ConfigurationManagerRW::getMetadataTable(
1871  TableVersion fillVersion /* = TableVersion()*/)
1872 {
1873  if(fillVersion.isInvalid())
1874  return &groupMetadataTable_;
1875  //else load specified fill version
1876 
1877  //only lock metadata table since it is shared by all group accesses
1878  std::lock_guard<std::mutex> lock(metaDataTableMutex_);
1879 
1880  // clear table
1881  while(groupMetadataTable_.getView().getNumberOfRows())
1882  groupMetadataTable_.getViewP()->deleteRow(0);
1883 
1884  // retrieve metadata from database
1885  try
1886  {
1887  theInterface_->fill(&groupMetadataTable_, fillVersion);
1888  }
1889  catch(const std::runtime_error& e)
1890  {
1891  __GEN_COUT_WARN__ << "Failed to load " << groupMetadataTable_.getTableName()
1892  << "-v" << fillVersion << ". Metadata error: " << e.what()
1893  << __E__;
1894  }
1895  catch(...)
1896  {
1897  __GEN_COUT_WARN__ << "Failed to load " << groupMetadataTable_.getTableName()
1898  << "-v" << fillVersion << ". Ignoring unknown metadata error. "
1899  << __E__;
1900  }
1901 
1902  // check that there is only 1 row
1903  if(groupMetadataTable_.getView().getNumberOfRows() != 1)
1904  {
1905  groupMetadataTable_.print();
1906  __GEN_COUT_ERR__ << "Ignoring that groupMetadataTable_ v" << fillVersion
1907  << " has wrong "
1908  "number of rows!' Must "
1909  "be 1. Going with anonymous defaults."
1910  << __E__;
1911 
1912  // fix metadata table
1913  while(groupMetadataTable_.getViewP()->getNumberOfRows() > 1)
1914  groupMetadataTable_.getViewP()->deleteRow(0);
1915  if(groupMetadataTable_.getViewP()->getNumberOfRows() == 0)
1916  groupMetadataTable_.getViewP()->addRow();
1917  }
1918 
1919  return &groupMetadataTable_;
1920 } // end getMetadataTable()
1921 
1922 //==============================================================================
1929 TableGroupKey ConfigurationManagerRW::saveNewTableGroup(
1930  const std::string& groupName,
1931  std::map<std::string, TableVersion>& groupMembers,
1932  const std::string& groupComment,
1933  std::map<std::string /*table*/, std::string /*alias*/>* groupAliases)
1934 {
1935  // steps:
1936  // determine new group key
1937  // verify group members
1938  // verify groupNameWithKey
1939  // verify store
1940 
1941  if(groupMembers.size() == 0) // do not allow empty groups
1942  {
1943  __SS__ << "Empty group member list. Can not create a group without members!"
1944  << __E__;
1945  __SS_THROW__;
1946  }
1947 
1948  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
1949 
1950  // verify group members
1951  // - use all table info
1952  std::map<std::string, TableInfo> allCfgInfo = getAllTableInfo();
1953  for(auto& memberPair : groupMembers)
1954  {
1955  // check member name
1956  if(allCfgInfo.find(memberPair.first) == allCfgInfo.end())
1957  {
1958  __GEN_COUT_ERR__ << "Group member \"" << memberPair.first
1959  << "\" not found in database!";
1960 
1961  if(groupMetadataTable_.getTableName() == memberPair.first)
1962  {
1963  __GEN_COUT_WARN__
1964  << "Looks like this is the groupMetadataTable_ '"
1965  << TableBase::GROUP_METADATA_TABLE_NAME
1966  << ".' Note that this table is added to the member map when groups "
1967  "are saved."
1968  << "It should not be part of member map when calling this function."
1969  << __E__;
1970  __GEN_COUT__ << "Attempting to recover." << __E__;
1971  groupMembers.erase(groupMembers.find(memberPair.first));
1972  }
1973  else
1974  {
1975  __SS__ << ("Group member not found!") << __E__;
1976  __SS_THROW__;
1977  }
1978  }
1979  // check member version
1980  if(allCfgInfo[memberPair.first].versions_.find(memberPair.second) ==
1981  allCfgInfo[memberPair.first].versions_.end())
1982  {
1983  __SS__ << "Group member \"" << memberPair.first << "\" version \""
1984  << memberPair.second << "\" not found in database!";
1985  __SS_THROW__;
1986  }
1987  } // end verify members
1988 
1989  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
1990 
1991  // verify group aliases
1992  if(groupAliases)
1993  {
1994  for(auto& aliasPair : *groupAliases)
1995  {
1996  // check for alias table in member names
1997  if(groupMembers.find(aliasPair.first) == groupMembers.end())
1998  {
1999  __GEN_COUT_ERR__ << "Group member \"" << aliasPair.first
2000  << "\" not found in group member map!";
2001 
2002  __SS__ << ("Alias table not found in member list!") << __E__;
2003  __SS_THROW__;
2004  }
2005  }
2006  } // end verify group aliases
2007 
2008  TableGroupKey newKey =
2009  TableGroupKey::getNextKey(theInterface_->findLatestGroupKey(groupName));
2010  __GEN_COUT__ << "New Key for group: " << groupName << " found as " << newKey << __E__;
2011  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2012 
2013  time_t groupCreationTime = time(0);
2014  // capture group type before adding metadata table!
2015  std::string groupType = getTypeNameOfGroup(groupMembers);
2016  std::map<std::string /*name*/, TableVersion /*version*/> groupMembersWithoutMeta =
2017  groupMembers;
2018 
2019  // verify groupNameWithKey and attempt to store
2020  try
2021  {
2022  // save meta data for group; reuse groupMetadataTable_
2023  std::string groupAliasesString = "";
2024  if(groupAliases)
2025  groupAliasesString = StringMacros::mapToString(
2026  *groupAliases, "," /*primary delimeter*/, ":" /*secondary delimeter*/);
2027  __GEN_COUT__ << "Metadata: " << username_ << " " << groupCreationTime << " "
2028  << groupComment << " " << groupAliasesString << " " << groupType
2029  << __E__;
2030 
2031  // to compensate for unusual errors upstream, make sure the metadata table has one
2032  // row
2033  while(groupMetadataTable_.getViewP()->getNumberOfRows() > 1)
2034  groupMetadataTable_.getViewP()->deleteRow(0);
2035  if(groupMetadataTable_.getViewP()->getNumberOfRows() == 0)
2036  groupMetadataTable_.getViewP()->addRow();
2037 
2038  // columns are uid,comment,author,time
2039  groupMetadataTable_.getViewP()->setValue(
2040  groupAliasesString, 0, ConfigurationManager::METADATA_COL_ALIASES);
2041  groupMetadataTable_.getViewP()->setValue(
2042  groupComment, 0, ConfigurationManager::METADATA_COL_COMMENT);
2043  groupMetadataTable_.getViewP()->setValue(
2044  username_, 0, ConfigurationManager::METADATA_COL_AUTHOR);
2045  groupMetadataTable_.getViewP()->setValue(
2046  groupCreationTime, 0, ConfigurationManager::METADATA_COL_TIMESTAMP);
2047 
2048  if(TTEST(2))
2049  {
2050  std::stringstream ss;
2051  groupMetadataTable_.print(ss);
2052  __COUT_MULTI__(2, ss.str());
2053  }
2054 
2055  // save table, and retry on save collision
2056  {
2057  // set version to first available persistent version
2058  TableVersion newVersion = TableVersion::getNextVersion(
2059  theInterface_->findLatestVersion(&groupMetadataTable_));
2060  groupMetadataTable_.getViewP()->setVersion(newVersion);
2061 
2062  uint16_t retries = 0;
2063  while(1)
2064  {
2065  try
2066  {
2067  theInterface_->saveActiveVersion(&groupMetadataTable_);
2068  }
2069  catch(const std::runtime_error& e)
2070  {
2071  __GEN_COUT__ << "Caught runtime_error exception during table save."
2072  << __E__;
2073  if(std::string(e.what()).find("there was a collision") !=
2074  std::string::npos)
2075  {
2076  __GEN_COUT_WARN__
2077  << "There was a collision saving the new table "
2078  << groupMetadataTable_ << "(" << newVersion
2079  << "), trying incremented table version... retries="
2080  << retries << __E__;
2081  if(++retries > 3) //give up
2082  throw;
2083  newVersion = TableVersion::getNextVersion(
2084  newVersion); //increment table version
2085  groupMetadataTable_.getViewP()->setVersion(newVersion);
2086  __GEN_COUT__ << "New version for table: " << groupMetadataTable_
2087  << " found as " << newVersion << __E__;
2088  continue;
2089  }
2090  else
2091  throw;
2092  }
2093 
2094  __GEN_COUT__ << "Created table: " << groupMetadataTable_ << "-v"
2095  << newVersion << __E__;
2096  break;
2097  } //end collission retry loop
2098  }
2099 
2100  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2101  << __E__;
2102 
2103  // force groupMetadataTable_ to be a member for the group
2104  groupMembers[groupMetadataTable_.getTableName()] =
2105  groupMetadataTable_.getViewVersion();
2106 
2107  // save group, and retry on save collision
2108  {
2109  uint16_t retries = 0;
2110  while(1)
2111  {
2112  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2113  << __E__;
2114 
2115  try
2116  {
2117  theInterface_->saveTableGroup(
2118  groupMembers,
2119  TableGroupKey::getFullGroupString(groupName, newKey));
2120  }
2121  catch(const std::runtime_error& e)
2122  {
2123  __GEN_COUT__ << "Caught runtime_error exception during group save."
2124  << __E__;
2125  if(std::string(e.what()).find("there was a collision") !=
2126  std::string::npos)
2127  {
2128  __GEN_COUT_WARN__
2129  << "There was a collision saving the new group " << groupName
2130  << "(" << newKey
2131  << "), trying incremented group key... retries=" << retries
2132  << __E__;
2133  if(++retries > 3) //give up
2134  throw;
2135  newKey = TableGroupKey::getNextKey(newKey); //increment group key
2136  __GEN_COUT__ << "New Key for group: " << groupName << " found as "
2137  << newKey << __E__;
2138  continue;
2139  }
2140  else
2141  throw;
2142  }
2143 
2144  __GEN_COUT__ << "Created table group: " << groupName << "(" << newKey
2145  << ")" << __E__;
2146  break;
2147  } //end collission retry loop
2148  }
2149 
2150  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds()
2151  << __E__;
2152  }
2153  catch(std::runtime_error& e)
2154  {
2155  __GEN_COUT_ERR__ << "Failed to create table group: " << groupName << "(" << newKey
2156  << ")" << __E__;
2157  __GEN_COUT_ERR__ << "\n\n" << e.what() << __E__;
2158  throw;
2159  }
2160  catch(...)
2161  {
2162  __GEN_COUT_ERR__ << "Failed to create table group: " << groupName << ":" << newKey
2163  << __E__;
2164  throw;
2165  }
2166 
2167  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2168 
2169  // store cache of recent groups
2170  allGroupInfo_[groupName].keys_.emplace(newKey);
2171  //update latest group info with this group's info
2172  allGroupInfo_.at(groupName).latestKey_ = newKey;
2173  allGroupInfo_.at(groupName).latestKeyGroupAuthor_ = username_;
2174  allGroupInfo_.at(groupName).latestKeyGroupComment_ = groupComment;
2175  allGroupInfo_.at(groupName).latestKeyGroupCreationTime_ = groupCreationTime;
2176  allGroupInfo_.at(groupName).latestKeyGroupTypeString_ = groupType;
2177  allGroupInfo_.at(groupName).latestKeyMemberMap_ = groupMembersWithoutMeta;
2178 
2179  __GEN_COUT__ << "Saved " << groupName << "(" << newKey << ") of type " << groupType
2180  << __E__;
2181 
2182  __GEN_COUTT__ << "saveNewTableGroup runTimeSeconds()=" << runTimeSeconds() << __E__;
2183 
2184  // at this point succeeded!
2185  return newKey;
2186 } // end saveNewTableGroup()
2187 
2188 //==============================================================================
2192 TableVersion ConfigurationManagerRW::saveNewBackbone(TableVersion temporaryVersion)
2193 {
2194  __GEN_COUT_INFO__ << "Creating new backbone from temporary version "
2195  << temporaryVersion << __E__;
2196 
2197  // find common available temporary version among backbone members
2198  TableVersion newVersion(TableVersion::DEFAULT);
2199  TableVersion retNewVersion;
2200  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
2201  for(auto& name : backboneMemberNames)
2202  {
2203  retNewVersion = ConfigurationManager::getTableByName(name)->getNextVersion();
2204  __GEN_COUT__ << "New version for backbone member (" << name
2205  << "): " << retNewVersion << __E__;
2206  if(retNewVersion > newVersion)
2207  newVersion = retNewVersion;
2208  }
2209 
2210  __GEN_COUT__ << "Common new backbone version found as " << newVersion << __E__;
2211 
2212  // create new views from source temporary version
2213  for(auto& name : backboneMemberNames)
2214  {
2215  // saveNewVersion makes the new version the active version
2216  retNewVersion = getConfigurationInterface()->saveNewVersion(
2217  getTableByName(name), temporaryVersion, newVersion);
2218  if(retNewVersion != newVersion)
2219  {
2220  __SS__ << "Failure! New view requested was " << newVersion
2221  << ". Mismatched new view created: " << retNewVersion << __E__;
2222  __GEN_COUT_ERR__ << ss.str();
2223  __SS_THROW__;
2224  }
2225  }
2226 
2227  return newVersion;
2228 } // end saveNewBackbone()
2229 
2230 //==============================================================================
2235 TableVersion ConfigurationManagerRW::saveModifiedVersion(
2236  const std::string& tableName,
2237  TableVersion originalVersion,
2238  bool makeTemporary,
2239  TableBase* table,
2240  TableVersion temporaryModifiedVersion,
2241  bool ignoreDuplicates /*= false*/,
2242  bool lookForEquivalent /*= false*/,
2243  bool* foundEquivalent /*= nullptr*/)
2244 {
2245  bool needToEraseTemporarySource =
2246  (originalVersion.isTemporaryVersion() && !makeTemporary);
2247 
2248  if(foundEquivalent)
2249  *foundEquivalent = false; // initialize
2250 
2251  // check for duplicate tables already in cache, plus highest version numbers not in cache
2252  if(!ignoreDuplicates)
2253  {
2254  __GEN_COUT__ << "Checking for duplicate '" << tableName << "' tables..." << __E__;
2255 
2256  TableVersion duplicateVersion;
2257 
2258  {
2259  //"DEEP" checking
2260  // load into cache 'recent' versions for this table
2261  // 'recent' := those already in cache, plus highest version numbers not in cache
2262  const std::map<std::string, TableInfo>& allTableInfo =
2263  getAllTableInfo(); // do not refresh
2264 
2265  auto versionReverseIterator =
2266  allTableInfo.at(tableName).versions_.rbegin(); // get reverse iterator
2267  __GEN_COUT__ << "Filling up '" << tableName << "' cache from "
2268  << table->getNumberOfStoredViews() << " to max count of "
2269  << table->MAX_VIEWS_IN_CACHE << __E__;
2270  for(; table->getNumberOfStoredViews() < table->MAX_VIEWS_IN_CACHE &&
2271  versionReverseIterator != allTableInfo.at(tableName).versions_.rend();
2272  ++versionReverseIterator)
2273  {
2274  __GEN_COUTT__ << "'" << tableName << "' versions in reverse order "
2275  << *versionReverseIterator << __E__;
2276  try
2277  {
2278  getVersionedTableByName(tableName,
2279  *versionReverseIterator); // load to cache
2280  }
2281  catch(const std::runtime_error& e)
2282  {
2283  // ignore error
2284  __COUTT__ << "'" << tableName
2285  << "' version failed to load: " << *versionReverseIterator
2286  << __E__;
2287  }
2288  }
2289  }
2290 
2291  __GEN_COUT__ << "Checking '" << tableName << "' for duplicate..." << __E__;
2292 
2293  duplicateVersion = table->checkForDuplicate(
2294  temporaryModifiedVersion,
2295  (!originalVersion.isTemporaryVersion() && !makeTemporary)
2296  ? TableVersion()
2297  : // if from persistent to persistent, then include original version in search
2298  originalVersion);
2299 
2300  if(lookForEquivalent && !duplicateVersion.isInvalid())
2301  {
2302  // found an equivalent!
2303  __GEN_COUT__ << "Equivalent '" << tableName << "' table found in version v"
2304  << duplicateVersion << __E__;
2305 
2306  // if duplicate version was temporary, do not use
2307  if(duplicateVersion.isTemporaryVersion() && !makeTemporary)
2308  {
2309  __GEN_COUT__ << "Need persistent. Duplicate '" << tableName
2310  << "' version was temporary. "
2311  "Abandoning duplicate."
2312  << __E__;
2313  duplicateVersion = TableVersion(); // set invalid
2314  }
2315  else
2316  {
2317  // erase and return equivalent version
2318 
2319  // erase modified equivalent version
2320  eraseTemporaryVersion(tableName, temporaryModifiedVersion);
2321 
2322  // erase original if needed
2323  if(needToEraseTemporarySource)
2324  eraseTemporaryVersion(tableName, originalVersion);
2325 
2326  if(foundEquivalent)
2327  *foundEquivalent = true;
2328 
2329  __GEN_COUT__ << "\t\t Equivalent '" << tableName
2330  << "' assigned version: " << duplicateVersion << __E__;
2331 
2332  return duplicateVersion;
2333  }
2334  }
2335 
2336  if(!duplicateVersion.isInvalid())
2337  {
2338  __SS__ << "This version of table '" << tableName
2339  << "' is identical to another version currently cached v"
2340  << duplicateVersion << ". No reason to save a duplicate." << __E__;
2341  __GEN_COUT_ERR__ << "\n" << ss.str();
2342 
2343  // delete temporaryModifiedVersion
2344  table->eraseView(temporaryModifiedVersion);
2345  __SS_THROW__;
2346  }
2347 
2348  __GEN_COUT__ << "Check for duplicate '" << tableName << "' tables complete."
2349  << __E__;
2350  }
2351 
2352  if(makeTemporary)
2353  __GEN_COUT__ << "\t\t**************************** Save as temporary '"
2354  << tableName << "' table version" << __E__;
2355  else
2356  __GEN_COUT__ << "\t\t**************************** Save as new '" << tableName
2357  << "' table version" << __E__;
2358 
2359  TableVersion newAssignedVersion =
2360  saveNewTable(tableName, temporaryModifiedVersion, makeTemporary);
2361 
2362  __GEN_COUTTV__(table->getView().getComment());
2363 
2364  if(needToEraseTemporarySource)
2365  eraseTemporaryVersion(tableName, originalVersion);
2366 
2367  __GEN_COUT__ << "\t\t '" << tableName
2368  << "' new assigned version: " << newAssignedVersion << __E__;
2369  return newAssignedVersion;
2370 } // end saveModifiedVersion()
2371 
2372 //==============================================================================
2373 GroupEditStruct::GroupEditStruct(const ConfigurationManager::GroupType& groupType,
2374  ConfigurationManagerRW* cfgMgr)
2375  : groupType_(groupType)
2376  , originalGroupName_(cfgMgr->getActiveGroupName(groupType))
2377  , originalGroupKey_(cfgMgr->getActiveGroupKey(groupType))
2378  , cfgMgr_(cfgMgr)
2379  , mfSubject_(cfgMgr->getUsername())
2380 {
2381  if(originalGroupName_ == "" || originalGroupKey_.isInvalid())
2382  {
2383  __SS__ << "Error! No active group found for type '"
2384  << ConfigurationManager::convertGroupTypeToName(groupType)
2385  << ".' There must be an active group to edit the group." << __E__ << __E__
2386  << StringMacros::stackTrace() << __E__;
2387  __SS_THROW__;
2388  }
2389 
2390  __GEN_COUT__ << "Extracting Group-Edit Struct for type "
2391  << ConfigurationManager::convertGroupTypeToName(groupType) << __E__;
2392 
2393  std::map<std::string, TableVersion> activeTables = cfgMgr->getActiveVersions();
2394 
2395  const std::set<std::string>& memberNames =
2396  groupType == ConfigurationManager::GroupType::CONTEXT_TYPE
2397  ? ConfigurationManager::getContextMemberNames()
2398  : (groupType == ConfigurationManager::GroupType::BACKBONE_TYPE
2399  ? ConfigurationManager::getBackboneMemberNames()
2400  : (groupType == ConfigurationManager::GroupType::ITERATE_TYPE
2401  ? ConfigurationManager::getIterateMemberNames()
2402  : cfgMgr->getConfigurationMemberNames()));
2403 
2404  for(auto& memberName : memberNames)
2405  try
2406  {
2407  groupMembers_.emplace(
2408  std::make_pair(memberName, activeTables.at(memberName)));
2409  groupTables_.emplace(std::make_pair(
2410  memberName,
2411  TableEditStruct(memberName, cfgMgr))); // Table ready for editing!
2412  }
2413  catch(...)
2414  {
2415  __GEN_COUTV__(StringMacros::mapToString(activeTables));
2416  __SS__ << "Error! Could not find group member table '" << memberName
2417  << "' for group type '"
2418  << ConfigurationManager::convertGroupTypeToName(groupType)
2419  << ".' All group members must be present to create the group editing "
2420  "structure."
2421  << __E__ << __E__ << StringMacros::stackTrace() << __E__;
2422  __SS_THROW__;
2423  }
2424 
2425 } // end GroupEditStruct constructor()
2426 
2427 //==============================================================================
2428 GroupEditStruct::~GroupEditStruct()
2429 {
2430  __GEN_COUT__ << "GroupEditStruct from editing '" << originalGroupName_ << "("
2431  << originalGroupKey_ << ")' Destructing..." << __E__;
2432  dropChanges();
2433  __GEN_COUT__ << "GroupEditStruct from editing '" << originalGroupName_ << "("
2434  << originalGroupKey_ << ")' Desctructed." << __E__;
2435 } // end GroupEditStruct destructor()
2436 
2437 //==============================================================================
2439 TableEditStruct& GroupEditStruct::getTableEditStruct(const std::string& tableName,
2440  bool markModified /*= false*/)
2441 {
2442  auto it = groupTables_.find(tableName);
2443  if(it == groupTables_.end())
2444  {
2445  if(groupType_ == ConfigurationManager::GroupType::CONFIGURATION_TYPE &&
2446  markModified)
2447  {
2448  __GEN_COUT__ << "Table '" << tableName
2449  << "' not found in configuration table members from editing '"
2450  << originalGroupName_ << "(" << originalGroupKey_ << ")..."
2451  << " Attempting to add it!" << __E__;
2452 
2453  // emplace returns pair<object,bool wasAdded>
2454  auto newIt = groupTables_.emplace(std::make_pair(
2455  tableName,
2456  TableEditStruct(tableName, cfgMgr_))); // Table ready for editing!
2457  if(newIt.second)
2458  {
2459  newIt.first->second.modified_ =
2460  markModified; // could indicate 'dirty' immediately in user code, which will cause a save of table
2461  groupMembers_.emplace(
2462  std::make_pair(tableName, newIt.first->second.temporaryVersion_));
2463  return newIt.first->second;
2464  }
2465  __GEN_COUT_ERR__ << "Failed to emplace new table..." << __E__;
2466  }
2467 
2468  __SS__ << "Table '" << tableName << "' not found in table members from editing '"
2469  << originalGroupName_ << "(" << originalGroupKey_ << ")!'" << __E__;
2470  __SS_THROW__;
2471  }
2472  it->second.modified_ =
2473  markModified; // could indicate 'dirty' immediately in user code, which will cause a save of table
2474  return it->second;
2475 } // end getTableEditStruct()
2476 
2477 //==============================================================================
2478 void GroupEditStruct::dropChanges()
2479 {
2480  __GEN_COUT__ << "Dropping unsaved changes from editing '" << originalGroupName_ << "("
2481  << originalGroupKey_ << ")'..." << __E__;
2482 
2483  ConfigurationManagerRW* cfgMgr = cfgMgr_;
2484 
2485  // drop all temporary versions
2486  for(auto& groupTable : groupTables_)
2487  if(groupTable.second
2488  .createdTemporaryVersion_) // if temporary version created here
2489  {
2490  // erase with proper version management
2491  cfgMgr->eraseTemporaryVersion(groupTable.second.tableName_,
2492  groupTable.second.temporaryVersion_);
2493  groupTable.second.createdTemporaryVersion_ = false;
2494  groupTable.second.modified_ = false;
2495  }
2496 
2497  __GEN_COUT__ << "Unsaved changes dropped from editing '" << originalGroupName_ << "("
2498  << originalGroupKey_ << ").'" << __E__;
2499 } // end GroupEditStruct::dropChanges()
2500 
2501 //==============================================================================
2502 void GroupEditStruct::saveChanges(const std::string& groupNameToSave,
2503  TableGroupKey& newGroupKey,
2504  bool* foundEquivalentGroupKey /*= nullptr*/,
2505  bool activateNewGroup /*= false*/,
2506  bool updateGroupAliases /*= false*/,
2507  bool updateTableAliases /*= false*/,
2508  TableGroupKey* newBackboneKey /*= nullptr*/,
2509  bool* foundEquivalentBackboneKey /*= nullptr*/,
2510  std::string* accumulatedWarnings /*= nullptr*/)
2511 {
2512  __GEN_COUT__ << "Saving changes..." << __E__;
2513 
2514  newGroupKey = TableGroupKey(); // invalidate reference parameter
2515  if(newBackboneKey)
2516  *newBackboneKey = TableGroupKey(); // invalidate reference parameter
2517  if(foundEquivalentBackboneKey)
2518  *foundEquivalentBackboneKey = false; // clear to start
2519  ConfigurationManagerRW* cfgMgr = cfgMgr_;
2520 
2521  // save all temporary modified versions
2522  bool anyTableNew = false;
2523  for(auto& groupTable : groupTables_)
2524  {
2525  if(!groupTable.second.modified_)
2526  continue; // skip if not modified
2527 
2528  __GEN_COUT__ << "Original version is " << groupTable.second.tableName_ << "-v"
2529  << groupTable.second.originalVersion_ << __E__;
2530 
2531  groupMembers_.at(groupTable.first) = cfgMgr->saveModifiedVersion(
2532  groupTable.second.tableName_,
2533  groupTable.second.originalVersion_,
2534  true /*make temporary*/,
2535  groupTable.second.table_,
2536  groupTable.second.temporaryVersion_,
2537  true /*ignoreDuplicates*/); // make temporary version to save persistent version properly
2538 
2539  __GEN_COUT__ << "Temporary target version is " << groupTable.second.tableName_
2540  << "-v" << groupMembers_.at(groupTable.first) << "-v"
2541  << groupTable.second.temporaryVersion_ << __E__;
2542 
2543  groupMembers_.at(groupTable.first) = cfgMgr->saveModifiedVersion(
2544  groupTable.second.tableName_,
2545  groupTable.second.originalVersion_,
2546  false /*make temporary*/,
2547  groupTable.second.table_,
2548  groupTable.second.temporaryVersion_,
2549  false /*ignoreDuplicates*/,
2550  true /*lookForEquivalent*/); // save persistent version properly
2551 
2552  if(groupTable.second.originalVersion_ != groupMembers_.at(groupTable.first))
2553  {
2554  anyTableNew = true;
2555  __GEN_COUT__ << "Final NEW target version is " << groupTable.second.tableName_
2556  << "-v" << groupMembers_.at(groupTable.first) << __E__;
2557  }
2558  else
2559  __GEN_COUT__ << "Final target version is " << groupTable.second.tableName_
2560  << "-v" << groupMembers_.at(groupTable.first) << __E__;
2561 
2562  groupTable.second.modified_ = false; // clear modified flag
2563  groupTable.second.createdTemporaryVersion_ = false; // modified version is gone
2564  } // loop through table edit structs
2565 
2566  for(auto& table : groupMembers_)
2567  {
2568  __GEN_COUT__ << table.first << " v" << table.second << __E__;
2569  }
2570 
2571  if(!anyTableNew) //then could be duplicate group
2572  {
2573  __GEN_COUT__ << "Checking for duplicate groups..." << __E__;
2574  newGroupKey = cfgMgr->findTableGroup(groupNameToSave, groupMembers_);
2575  }
2576  else
2577  __GEN_COUT__ << "New table found, so no need to check duplicate groups." << __E__;
2578 
2579  if(!newGroupKey.isInvalid())
2580  {
2581  __GEN_COUT__ << "Found equivalent group key (" << newGroupKey << ") for "
2582  << groupNameToSave << "." << __E__;
2583  if(foundEquivalentGroupKey)
2584  *foundEquivalentGroupKey = true;
2585  }
2586  else
2587  {
2588  newGroupKey = cfgMgr->saveNewTableGroup(groupNameToSave, groupMembers_);
2589  __GEN_COUT__ << "Saved new Context group key (" << newGroupKey << ") for "
2590  << groupNameToSave << "." << __E__;
2591  }
2592 
2593  bool groupAliasChange = false;
2594  bool tableAliasChange = false;
2595 
2596  if(groupType_ !=
2597  ConfigurationManager::GroupType::
2598  BACKBONE_TYPE) //if not backbone group save, consider changing aliases (not if it is backbone group, tables not necessarily active yet, which causes error in GroupEditStruct backboneGroupEdit)
2599  {
2600  GroupEditStruct backboneGroupEdit(ConfigurationManager::GroupType::BACKBONE_TYPE,
2601  cfgMgr);
2602 
2603  if(groupType_ != ConfigurationManager::GroupType::BACKBONE_TYPE &&
2604  updateGroupAliases)
2605  {
2606  // check group aliases ... a la
2607  // ConfigurationGUISupervisor::handleSetGroupAliasInBackboneXML
2608 
2609  TableEditStruct& groupAliasTable = backboneGroupEdit.getTableEditStruct(
2610  ConfigurationManager::GROUP_ALIASES_TABLE_NAME, true /*markModified*/);
2611  TableView* tableView = groupAliasTable.tableView_;
2612 
2613  // unsigned int col;
2614  unsigned int row = 0;
2615 
2616  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
2617  cfgMgr->getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
2618  .getChildren();
2619  std::string groupName, groupKey;
2620  for(auto& aliasNodePair : aliasNodePairs)
2621  {
2622  groupName = aliasNodePair.second.getNode("GroupName").getValueAsString();
2623  groupKey = aliasNodePair.second.getNode("GroupKey").getValueAsString();
2624 
2625  __GEN_COUT__ << "Group Alias: " << aliasNodePair.first << " => "
2626  << groupName << "(" << groupKey << "); row=" << row << __E__;
2627 
2628  if(groupName == originalGroupName_ &&
2629  TableGroupKey(groupKey) == originalGroupKey_)
2630  {
2631  __GEN_COUT__ << "Found alias! Changing group key from ("
2632  << originalGroupKey_ << ") to (" << newGroupKey << ")"
2633  << __E__;
2634 
2635  groupAliasChange = true;
2636 
2637  tableView->setValueAsString(
2638  newGroupKey.toString(), row, tableView->findCol("GroupKey"));
2639  }
2640 
2641  ++row;
2642  }
2643 
2644  if(groupAliasChange)
2645  {
2646  std::stringstream ss;
2647  tableView->print(ss);
2648  __GEN_COUT__ << ss.str();
2649  }
2650  } // end updateGroupAliases handling
2651 
2652  if(groupType_ != ConfigurationManager::GroupType::BACKBONE_TYPE &&
2653  updateTableAliases)
2654  {
2655  // update all table version aliases
2656  TableView* tableView =
2657  backboneGroupEdit
2658  .getTableEditStruct(ConfigurationManager::VERSION_ALIASES_TABLE_NAME,
2659  true /*markModified*/)
2660  .tableView_;
2661 
2662  for(auto& groupTable : groupTables_)
2663  {
2664  if(groupTable.second.originalVersion_ ==
2665  groupMembers_.at(groupTable.second.tableName_))
2666  continue; // skip if no change
2667 
2668  __GEN_COUT__ << "Checking alias... original version is "
2669  << groupTable.second.tableName_ << "-v"
2670  << groupTable.second.originalVersion_
2671  << " and new version is v"
2672  << groupMembers_.at(groupTable.second.tableName_) << __E__;
2673 
2674  // unsigned int col;
2675  unsigned int row = 0;
2676 
2677  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
2678  cfgMgr->getNode(ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
2679  .getChildren();
2680  std::string tableName, tableVersion;
2681  for(auto& aliasNodePair : aliasNodePairs)
2682  {
2683  tableName =
2684  aliasNodePair.second.getNode("TableName").getValueAsString();
2685  tableVersion =
2686  aliasNodePair.second.getNode("Version").getValueAsString();
2687 
2688  __GEN_COUT__ << "Table Alias: " << aliasNodePair.first << " => "
2689  << tableName << "-v" << tableVersion << "" << __E__;
2690 
2691  if(tableName == groupTable.second.tableName_ &&
2692  TableVersion(tableVersion) == groupTable.second.originalVersion_)
2693  {
2694  __GEN_COUT__ << "Found alias! Changing icon table version alias."
2695  << __E__;
2696 
2697  tableAliasChange = true;
2698 
2699  tableView->setValueAsString(
2700  groupMembers_.at(groupTable.second.tableName_).toString(),
2701  row,
2702  tableView->findCol("Version"));
2703  }
2704 
2705  ++row;
2706  }
2707  }
2708 
2709  if(tableAliasChange)
2710  {
2711  std::stringstream ss;
2712  tableView->print(ss);
2713  __GEN_COUT__ << ss.str();
2714  }
2715  } // end updateTableAliases handling
2716 
2717  TableGroupKey localNewBackboneKey;
2718  // if backbone modified, save group and activate it
2719  if(groupAliasChange || tableAliasChange)
2720  {
2721  for(auto& table : backboneGroupEdit.groupMembers_)
2722  {
2723  __GEN_COUT__ << table.first << " v" << table.second << __E__;
2724  }
2725  backboneGroupEdit.saveChanges(
2726  backboneGroupEdit.originalGroupName_,
2727  localNewBackboneKey,
2728  foundEquivalentBackboneKey ? foundEquivalentBackboneKey : nullptr);
2729 
2730  if(newBackboneKey)
2731  *newBackboneKey = localNewBackboneKey;
2732  }
2733 
2734  // acquire all active groups and ignore errors, so that activateTableGroup does not
2735  // erase other active groups
2736  {
2737  __GEN_COUT__
2738  << "Restoring active table groups, before activating new groups..."
2739  << __E__;
2740 
2741  std::string localAccumulatedWarnings;
2742  cfgMgr->restoreActiveTableGroups(
2743  false /*throwErrors*/,
2744  "" /*pathToActiveGroupsFile*/,
2745  ConfigurationManager::LoadGroupType::
2746  ALL_TYPES /*onlyLoadIfBackboneOrContext*/,
2747  &localAccumulatedWarnings);
2748  }
2749 
2750  // activate new groups
2751  if(!localNewBackboneKey.isInvalid())
2752  cfgMgr->activateTableGroup(
2753  backboneGroupEdit.originalGroupName_,
2754  localNewBackboneKey,
2755  accumulatedWarnings ? accumulatedWarnings : nullptr);
2756 
2757  } //end non-backbone save type handling
2758  else //is backbone save type
2759  {
2760  // acquire all active groups and ignore errors, so that activateTableGroup does not
2761  // erase other active groups
2762  {
2763  __GEN_COUT__
2764  << "Restoring active table groups, before activating new groups..."
2765  << __E__;
2766 
2767  std::string localAccumulatedWarnings;
2768  cfgMgr->restoreActiveTableGroups(
2769  false /*throwErrors*/,
2770  "" /*pathToActiveGroupsFile*/,
2771  ConfigurationManager::LoadGroupType::
2772  ALL_TYPES /*onlyLoadIfBackboneOrContext*/,
2773  &localAccumulatedWarnings);
2774  }
2775  } //end backbone save type handling
2776 
2777  if(activateNewGroup)
2778  cfgMgr->activateTableGroup(groupNameToSave,
2779  newGroupKey,
2780  accumulatedWarnings ? accumulatedWarnings : nullptr);
2781 
2782  __GEN_COUT__ << "Changes saved." << __E__;
2783 } // end GroupEditStruct::saveChanges()
2784 
2785 //==============================================================================
2787 void ConfigurationManagerRW::testXDAQContext()
2788 {
2789  if(1)
2790  return; //if 0 to debug
2791  __GEN_COUTV__(runTimeSeconds());
2792 
2793  std::string accumulatedWarningsStr;
2794  std::string* accumulatedWarnings = &accumulatedWarningsStr;
2795 
2796  // get Group Info too!
2797  try
2798  {
2799  std::string debugGroupName = "Mu2eHWEmulatorContext";
2800 
2801  //final solution demo of getting latest group key:
2802  {
2803  TableGroupKey latestGroupKey =
2804  theInterface_->findLatestGroupKey(debugGroupName);
2805  __GEN_COUTV__(latestGroupKey);
2806 
2807  __GEN_COUTV__(runTimeSeconds());
2808  }
2809 
2810  //steps to do time comparison for getting last group key and table key:
2811 
2812  // build allGroupInfo_ for the ConfigurationManagerRW
2813 
2814  std::set<std::string /*name*/> tableGroups =
2815  theInterface_->getAllTableGroupNames();
2816  __GEN_COUT__ << "Number of Groups: " << tableGroups.size() << __E__;
2817 
2818  __GEN_COUTV__(runTimeSeconds());
2819  // return;
2820 
2821  TableGroupKey key;
2822  std::string name;
2823  for(const auto& fullName : tableGroups)
2824  {
2825  TableGroupKey::getGroupNameAndKey(fullName, name, key);
2826  allGroupInfo_[name].keys_.emplace(key);
2827 
2828  if(name == debugGroupName)
2829  {
2830  __GEN_COUTV__(key);
2831  }
2832  }
2833  __GEN_COUTV__(runTimeSeconds());
2834 
2835  std::set<std::string /*name*/> tableNames = theInterface_->getAllTableNames();
2836  __GEN_COUT__ << "Number of Tables: " << tableNames.size() << __E__;
2837 
2838  __GEN_COUTV__(runTimeSeconds());
2839 
2840  for(const auto& fullName : tableNames)
2841  {
2842  if(fullName.find(debugGroupName) != std::string::npos)
2843  {
2844  __GEN_COUTV__(fullName);
2845  }
2846  }
2847  __GEN_COUTV__(runTimeSeconds());
2848 
2849  TableGroupKey latestGroupKey = theInterface_->findLatestGroupKey(debugGroupName);
2850  __GEN_COUTV__(latestGroupKey);
2851 
2852  __GEN_COUTV__(runTimeSeconds());
2853 
2854  TableBase localGroupMemberCacheSaver(
2855  true /*special table*/
2856  , //special table only allows 1 view in cache and does not load schema (which is perfect for this temporary table),
2857  TableBase::GROUP_CACHE_PREPEND + debugGroupName);
2858  TableVersion lastestGroupCacheKey =
2859  theInterface_->findLatestVersion(&localGroupMemberCacheSaver);
2860  __GEN_COUTV__(lastestGroupCacheKey);
2861 
2862  __GEN_COUTV__(runTimeSeconds());
2863 
2864  //test a group save that already exists
2865  try
2866  {
2867  TableGroupKey groupKey(int(0));
2868  __GEN_COUT__ << "Testing group save of pre-existing " << debugGroupName << "("
2869  << groupKey << ")" << __E__;
2870  std::map<std::string, TableVersion> groupMembers;
2871  groupMembers["DesktopIconTable"] = TableVersion(123);
2872  theInterface_->saveTableGroup(
2873  groupMembers,
2874  TableGroupKey::getFullGroupString(debugGroupName, groupKey));
2875  }
2876  catch(...)
2877  {
2878  __GEN_COUT__ << "Exception during group save." << __E__;
2879  }
2880  __GEN_COUTV__(runTimeSeconds());
2881 
2882  //test a group save that does not already exists
2883  try
2884  {
2885  std::string debugGroupName = "testGroupSave";
2886  TableGroupKey groupKey(int(1));
2887  __GEN_COUT__ << "Testing group save of non-existing " << debugGroupName << "("
2888  << groupKey << ")" << __E__;
2889  std::map<std::string, TableVersion> groupMembers;
2890  groupMembers["DesktopIconTable"] = TableVersion(123);
2891  groupMembers["MessageFacilityTable"] = TableVersion(7);
2892  theInterface_->saveTableGroup(
2893  groupMembers,
2894  TableGroupKey::getFullGroupString(debugGroupName, groupKey));
2895  }
2896  catch(...)
2897  {
2898  __GEN_COUT__ << "Exception during new group save." << __E__;
2899  }
2900  __GEN_COUTV__(runTimeSeconds());
2901 
2902  //test a table save that already exists
2903  {
2904  std::string documentNameToLoad = "XDAQApplicationTable";
2905  TableVersion documentVersionToLoad(134);
2906 
2907  __GEN_COUT__ << "Testing table save of pre-existing " << documentNameToLoad
2908  << __E__;
2909 
2910  { //load to prove it exists
2911  TableBase localDocLoader(
2912  documentNameToLoad); //can not use special table when filling
2913  localDocLoader.changeVersionAndActivateView(
2914  localDocLoader.createTemporaryView(), documentVersionToLoad);
2915  theInterface_->fill(&localDocLoader, documentVersionToLoad);
2916  __SS__;
2917  localDocLoader.print(ss);
2918  __GEN_COUTV__(ss.str());
2919  }
2920  __GEN_COUTV__(runTimeSeconds());
2921 
2922  try
2923  { //attempt to save over existing version
2924  std::string documentNameToSave = documentNameToLoad;
2925  TableBase
2926  localDocSaver( //true /*special table*/, //special table only allows 1 view in cache and does not load schema (which is perfect for this check),
2927  documentNameToSave); //can not use special table when filling
2928  localDocSaver.changeVersionAndActivateView(
2929  localDocSaver.createTemporaryView(), documentVersionToLoad);
2930 
2931  std::string json = "{ }";
2932  localDocSaver.getViewP()->setCustomStorageData(json);
2933 
2934  __COUTT__ << "Saving JSON string: "
2935  << localDocSaver.getViewP()->getCustomStorageData() << __E__;
2936 
2937  __COUTT__ << "Saving JSON doc as "
2938  << localDocSaver.getView().getTableName() << "("
2939  << localDocSaver.getView().getVersion().toString() << ")"
2940  << __E__;
2941 
2942  // save to db, and do not allow overwrite
2943  theInterface_->saveActiveVersion(&localDocSaver, false /* overwrite */);
2944  }
2945  catch(...)
2946  {
2947  __GEN_COUT__ << "Exception during table save." << __E__;
2948  }
2949  __GEN_COUTV__(runTimeSeconds());
2950 
2951  { //load to prove it exists
2952  TableBase localDocLoader(
2953  documentNameToLoad); //can not use special table when filling
2954  localDocLoader.changeVersionAndActivateView(
2955  localDocLoader.createTemporaryView(), documentVersionToLoad);
2956  theInterface_->fill(&localDocLoader, documentVersionToLoad);
2957  __SS__;
2958  localDocLoader.print(ss);
2959  __GEN_COUTV__(ss.str());
2960  }
2961  __GEN_COUTV__(runTimeSeconds());
2962  }
2963  __GEN_COUTV__(runTimeSeconds());
2964 
2965  //test a table save that does not already exist
2966  {
2967  std::string documentNameToLoad = "MessageFacilityTable";
2968  TableVersion documentVersionToLoad(7);
2969  TableBase localDocLoader(
2970  documentNameToLoad); //can not use special table when filling
2971 
2972  __GEN_COUT__ << "Testing table save of non-existing " << documentNameToLoad
2973  << __E__;
2974 
2975  { //load to prove it exists
2976  localDocLoader.changeVersionAndActivateView(
2977  localDocLoader.createTemporaryView(), documentVersionToLoad);
2978  theInterface_->fill(&localDocLoader, documentVersionToLoad);
2979  __SS__;
2980  localDocLoader.print(ss);
2981  __GEN_COUTV__(ss.str());
2982  __GEN_COUTV__(runTimeSeconds());
2983  }
2984  __GEN_COUTV__(runTimeSeconds());
2985 
2986  try
2987  { //attempt to save new version
2988 
2989  // modify it
2990  TableVersion newVersion = TableVersion::getNextVersion(
2991  theInterface_->findLatestVersion(&localDocLoader));
2992  localDocLoader.getViewP()->setVersion(newVersion);
2993 
2994  __GEN_COUTT__ << "Saving new table as "
2995  << localDocLoader.getView().getTableName() << "("
2996  << localDocLoader.getView().getVersion().toString() << ")"
2997  << __E__;
2998 
2999  localDocLoader.getViewP()->setValueAsString(
3000  "10.226.9.17", 0, 4); //modify value that is 10.226.9.16
3001 
3002  __SS__;
3003  localDocLoader.print(ss);
3004  __GEN_COUTV__(ss.str());
3005 
3006  // save to db, and do not allow overwrite
3007  // theInterface_->saveActiveVersion(&localDocLoader, false /* overwrite */);
3008  }
3009  catch(...)
3010  {
3011  __GEN_COUT__ << "Exception during new table save." << __E__;
3012  }
3013  __GEN_COUTV__(runTimeSeconds());
3014  }
3015  __GEN_COUTV__(runTimeSeconds());
3016  return;
3017 
3018  // for each group get member map & comment, author, time, and type for latest key
3019  for(auto& groupInfo : allGroupInfo_)
3020  {
3021  try
3022  {
3023  groupInfo.second.latestKey_ = groupInfo.second.getLastKey();
3024  loadTableGroup(groupInfo.first /*groupName*/,
3025  groupInfo.second.latestKey_,
3026  false /*doActivate*/,
3027  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
3028  0 /*progressBar*/,
3029  0 /*accumulateErrors*/,
3030  &groupInfo.second.latestKeyGroupComment_,
3031  &groupInfo.second.latestKeyGroupAuthor_,
3032  &groupInfo.second.latestKeyGroupCreationTime_,
3033  true /*doNotLoadMember*/,
3034  &groupInfo.second.latestKeyGroupTypeString_);
3035  }
3036  catch(const std::runtime_error& e)
3037  {
3038  __GEN_COUT_WARN__
3039  << "Error occurred loading latest group info into cache for '"
3040  << groupInfo.first << "(" << groupInfo.second.latestKey_ << ")': \n"
3041  << e.what() << __E__;
3042 
3043  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
3044  groupInfo.second.latestKeyGroupComment_ =
3045  ConfigurationManager::UNKNOWN_INFO;
3046  groupInfo.second.latestKeyGroupAuthor_ =
3047  ConfigurationManager::UNKNOWN_INFO;
3048  groupInfo.second.latestKeyGroupCreationTime_ =
3049  ConfigurationManager::UNKNOWN_TIME;
3050  groupInfo.second.latestKeyGroupTypeString_ =
3051  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
3052  groupInfo.second.latestKeyMemberMap_ = {};
3053  }
3054  catch(...)
3055  {
3056  __GEN_COUT_WARN__
3057  << "Error occurred loading latest group info into cache for '"
3058  << groupInfo.first << "(" << groupInfo.second.latestKey_ << ")'..."
3059  << __E__;
3060 
3061  groupInfo.second.latestKey_ = TableGroupKey::INVALID;
3062  groupInfo.second.latestKeyGroupComment_ =
3063  ConfigurationManager::UNKNOWN_INFO;
3064  groupInfo.second.latestKeyGroupAuthor_ =
3065  ConfigurationManager::UNKNOWN_INFO;
3066  groupInfo.second.latestKeyGroupCreationTime_ =
3067  ConfigurationManager::UNKNOWN_TIME;
3068  groupInfo.second.latestKeyGroupTypeString_ =
3069  ConfigurationManager::GROUP_TYPE_NAME_UNKNOWN;
3070  groupInfo.second.latestKeyMemberMap_ = {};
3071  }
3072  } // end group info loop
3073  __GEN_COUTV__(runTimeSeconds());
3074  } // end get group info
3075  catch(const std::runtime_error& e)
3076  {
3077  __SS__ << "A fatal error occurred reading the info for all table groups. Error: "
3078  << e.what() << __E__;
3079  __GEN_COUT_ERR__ << "\n" << ss.str();
3080  if(accumulatedWarnings)
3081  *accumulatedWarnings += ss.str();
3082  else
3083  throw;
3084  }
3085  catch(...)
3086  {
3087  __SS__ << "An unknown fatal error occurred reading the info for all table groups."
3088  << __E__;
3089  __GEN_COUT_ERR__ << "\n" << ss.str();
3090  if(accumulatedWarnings)
3091  *accumulatedWarnings += ss.str();
3092  else
3093  throw;
3094  } //end catch
3095 
3096  __GEN_COUT__ << "testXDAQContext() end runTimeSeconds()=" << runTimeSeconds()
3097  << __E__;
3098  return;
3099 
3100  try
3101  {
3102  __GEN_COUT__ << "Loading table..." << __E__;
3103  loadTableGroup("FETest", TableGroupKey(2)); // Context_1
3104  ConfigurationTree t = getNode("/FETable/DEFAULT/FrontEndType");
3105 
3106  std::string v;
3107 
3108  __GEN_COUT__ << __E__;
3109  t.getValue(v);
3110  __GEN_COUT__ << "Value: " << v << __E__;
3111  __GEN_COUT__ << "Value index: " << t.getValue<int>() << __E__;
3112 
3113  return;
3114  }
3115  catch(...)
3116  {
3117  __GEN_COUT__ << "Failed to load table..." << __E__;
3118  }
3119 } //end testXDAQContext()
ots::ConfigurationInterface::saveNewVersion
TableVersion saveNewVersion(TableBase *configuration, TableVersion temporaryVersion, TableVersion newVersion=TableVersion())
Definition: ConfigurationInterface.cc:79
ots::ConfigurationManagerRW::saveNewTable
TableVersion saveNewTable(const std::string &tableName, TableVersion temporaryVersion=TableVersion(), bool makeTemporary=false)
modifiers of generic TableBase
Definition: ConfigurationManagerRW.cc:1387
ots::ConfigurationManagerRW::findTableGroup
TableGroupKey findTableGroup(const std::string &groupName, const std::map< std::string, TableVersion > &groupMembers, const std::map< std::string, std::string > &groupAliases=std::map< std::string, std::string >())
Definition: ConfigurationManagerRW.cc:1613
ots::ConfigurationManagerRW::testXDAQContext
void testXDAQContext(void)
for debugging
Definition: ConfigurationManagerRW.cc:2787
ots::ConfigurationManagerRW::saveNewBackbone
TableVersion saveNewBackbone(TableVersion temporaryVersion=TableVersion())
Definition: ConfigurationManagerRW.cc:2192
ots::ConfigurationManagerRW::getGroupInfo
const GroupInfo & getGroupInfo(const std::string &groupName, bool attemptToReloadKeys=false)
public group cache handling
Definition: ConfigurationManagerRW.cc:1545
ots::ConfigurationManagerRW::getAllTableInfo
const std::map< std::string, TableInfo > & getAllTableInfo(bool refresh=false, std::string *accumulatedWarnings=0, const std::string &errorFilterName="", bool getGroupKeys=false, bool getGroupInfo=false, bool initializeActiveGroups=false)
Definition: ConfigurationManagerRW.cc:204
ots::ConfigurationManagerRW::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false)
Definition: ConfigurationManagerRW.cc:887
ots::ConfigurationManagerRW::copyViewToCurrentColumns
TableVersion copyViewToCurrentColumns(const std::string &tableName, TableVersion sourceVersion)
copyViewToCurrentColumns
Definition: ConfigurationManagerRW.cc:1521
ots::ConfigurationManagerRW::saveNewTableGroup
TableGroupKey saveNewTableGroup(const std::string &groupName, std::map< std::string, TableVersion > &groupMembers, const std::string &groupComment=TableViewColumnInfo::DATATYPE_COMMENT_DEFAULT, std::map< std::string, std::string > *groupAliases=0)
modifiers of a table group based on alias, e.g. "Physics"
Definition: ConfigurationManagerRW.cc:1929
ots::ConfigurationManagerRW::activateTableGroup
void activateTableGroup(const std::string &tableGroupName, TableGroupKey tableGroupKey, std::string *accumulatedTreeErrors=0, std::string *groupTypeString=0)
modifiers of table groups
Definition: ConfigurationManagerRW.cc:1135
ots::ConfigurationManagerRW::getMetadataTable
TableBase * getMetadataTable(TableVersion fillVersion=TableVersion())
created for use in otsdaq_flatten_system_aliases and otsdaq_export_system_aliases,...
Definition: ConfigurationManagerRW.cc:1870
ots::ConfigurationManagerRW::saveModifiedVersion
TableVersion saveModifiedVersion(const std::string &tableName, TableVersion originalVersion, bool makeTemporary, TableBase *config, TableVersion temporaryModifiedVersion, bool ignoreDuplicates=false, bool lookForEquivalent=false, bool *foundEquivalent=nullptr)
Definition: ConfigurationManagerRW.cc:2235
ots::ConfigurationManagerRW::createTemporaryBackboneView
TableVersion createTemporaryBackboneView(TableVersion sourceViewVersion=TableVersion())
-1, from MockUp, else from valid backbone view version
Definition: ConfigurationManagerRW.cc:1256
ots::ConfigurationManagerRW::clearCachedVersions
void clearCachedVersions(const std::string &tableName)
Definition: ConfigurationManagerRW.cc:1501
ots::ConfigurationManagerRW::getVersionAliases
std::map< std::string, std::map< std::string, TableVersion > > getVersionAliases(void) const
Definition: ConfigurationManagerRW.cc:1113
ots::ConfigurationManagerRW::eraseTemporaryVersion
void eraseTemporaryVersion(const std::string &tableName, TableVersion targetVersion=TableVersion())
Definition: ConfigurationManagerRW.cc:1452
ots::ConfigurationManagerRW::getVersionedTableByName
TableBase * getVersionedTableByName(const std::string &tableName, TableVersion version, bool looseColumnMatching=false, std::string *accumulatedErrors=0, bool getRawData=false)
Definition: ConfigurationManagerRW.cc:1337
ots::ConfigurationManagerRW::compareTableGroupThread
static void compareTableGroupThread(ConfigurationManagerRW *cfgMgr, std::string groupName, ots::TableGroupKey groupKeyToCompare, const std::map< std::string, TableVersion > &groupMemberMap, const std::map< std::string, std::string > &memberTableAliases, std::atomic< bool > *theFoundIdentical, ots::TableGroupKey *theIdenticalKey, std::mutex *theThreadMutex, std::shared_ptr< std::atomic< bool >> theThreadDone)
Definition: ConfigurationManagerRW.cc:984
ots::ConfigurationManagerRW::loadTableGroupThread
static void loadTableGroupThread(ConfigurationManagerRW *cfgMgr, std::string groupName, ots::TableGroupKey groupKey, std::shared_ptr< ots::GroupInfo > theGroupInfo, std::shared_ptr< std::atomic< bool >> theThreadDone)
loadTableGroupThread()
Definition: ConfigurationManagerRW.cc:942
ots::ConfigurationManagerRW::loadTableInfoThread
static void loadTableInfoThread(ConfigurationManagerRW *cfgMgr, std::string tableName, TableBase *existingTable, std::shared_ptr< ots::TableInfo > tableInfo, std::shared_ptr< std::atomic< bool >> threadDone)
loadTableInfoThread()
Definition: ConfigurationManagerRW.cc:748
ots::ConfigurationManager::getVersionAliases
std::map< std::string, std::map< std::string, TableVersion > > getVersionAliases(void) const
Definition: ConfigurationManager.cc:3290
ots::ConfigurationManager::PROCESSOR_COUNT
static const unsigned int PROCESSOR_COUNT
Static members.
Definition: ConfigurationManager.h:37
ots::ConfigurationManager::convertGroupTypeToName
static const std::string & convertGroupTypeToName(const ConfigurationManager::GroupType &groupTypeId)
Definition: ConfigurationManager.cc:595
ots::ConfigurationManager::restoreActiveTableGroups
void restoreActiveTableGroups(bool throwErrors=false, const std::string &pathToActiveGroupsFile="", ConfigurationManager::LoadGroupType onlyLoadIfBackboneOrContext=ConfigurationManager::LoadGroupType::ALL_TYPES, std::string *accumulatedWarnings=0)
Definition: ConfigurationManager.cc:280
ots::ConfigurationManager::getActiveVersions
std::map< std::string, TableVersion > getActiveVersions(void) const
getActiveVersions
Definition: ConfigurationManager.cc:3338
ots::ConfigurationManager::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType groupTypeToLoad=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false)
Definition: ConfigurationManager.cc:1511
ots::ConfigurationManager::getNode
ConfigurationTree getNode(const std::string &nodeString, bool doNotThrowOnBrokenUIDLinks=false) const
"root/parent/parent/"
Definition: ConfigurationManager.cc:2843
ots::ConfigurationManager::init
void init(std::string *accumulatedErrors=0, bool initForWriteAccess=false, std::string *accumulatedWarnings=0)
Definition: ConfigurationManager.cc:229
ots::ConfigurationManager::getTypeNameOfGroup
static const std::string & getTypeNameOfGroup(const std::map< std::string, TableVersion > &memberMap)
Definition: ConfigurationManager.cc:792
ots::ConfigurationManager::destroyTableGroup
void destroyTableGroup(const std::string &theGroup="", bool onlyDeactivate=false)
Definition: ConfigurationManager.cc:463
ots::ConfigurationManager::ACTIVE_GROUPS_FILENAME
static const std::string ACTIVE_GROUPS_FILENAME
added env check for otsdaq_flatten_active_to_version to function
Definition: ConfigurationManager.h:40
ots::ConfigurationManager::getTableByName
const TableBase * getTableByName(const std::string &configurationName) const
Definition: ConfigurationManager.cc:3131
ots::ConfigurationTree::getValue
void getValue(T &value) const
ots::ConfigurationTree::getChildren
std::vector< std::pair< std::string, ConfigurationTree > > getChildren(std::map< std::string, std::string > filterMap=std::map< std::string, std::string >(), bool byPriority=false, bool onlyStatusTrue=false) const
Definition: ConfigurationTree.cc:2495
ots::TableBase::createTemporaryView
TableVersion createTemporaryView(TableVersion sourceViewVersion=TableVersion(), TableVersion destTemporaryViewVersion=TableVersion::getNextTemporaryVersion())
source of -1, from MockUp, else from valid view version
Definition: TableBase.cc:1734
ots::TableBase::trimTemporary
void trimTemporary(TableVersion targetVersion=TableVersion())
Definition: TableBase.cc:362
ots::TableBase::getNumberOfStoredViews
unsigned int getNumberOfStoredViews(void) const
Definition: TableBase.cc:856
ots::TableBase::checkForDuplicate
TableVersion checkForDuplicate(TableVersion needleVersion, TableVersion ignoreVersion=TableVersion()) const
Definition: TableBase.cc:404
ots::TableBase::getTemporaryView
TableView * getTemporaryView(TableVersion temporaryVersion)
Definition: TableBase.cc:1845
ots::TableBase::MAX_VIEWS_IN_CACHE
const unsigned int MAX_VIEWS_IN_CACHE
Definition: TableBase.h:22
ots::TableBase::getNextVersion
TableVersion getNextVersion(void) const
Definition: TableBase.cc:1821
ots::TableBase::copyView
TableVersion copyView(const TableView &sourceView, TableVersion destinationVersion, const std::string &author, bool looseColumnMatching=false)
Definition: TableBase.cc:1675
ots::TableBase::print
void print(std::ostream &out=std::cout) const
always prints active view
Definition: TableBase.cc:277
ots::TableBase::getNextTemporaryVersion
TableVersion getNextTemporaryVersion(void) const
Definition: TableBase.cc:1798
ots::TableBase::trimCache
void trimCache(unsigned int trimSize=-1)
Definition: TableBase.cc:322
ots::TableGroupKey::toString
std::string toString(void) const
toString
Definition: TableGroupKey.cc:111
ots::TableGroupKey::getNextKey
static TableGroupKey getNextKey(const TableGroupKey &key=TableGroupKey())
Definition: TableGroupKey.cc:156
ots::TableGroupKey::getGroupNameAndKey
static void getGroupNameAndKey(const std::string &fullGroupString, std::string &groupName, TableGroupKey &key)
requires fullGroupString created as name + "_v" + key + ""
Definition: TableGroupKey.cc:220
ots::TableGroupKey::isInvalid
bool isInvalid(void) const
isInvalid
Definition: TableGroupKey.cc:148
ots::TableGroupKey::getFullGroupString
static std::string getFullGroupString(const std::string &groupName, const TableGroupKey &key, const std::string &preKey="_v", const std::string &postKey="")
Definition: TableGroupKey.cc:174
ots::TableVersion::toString
std::string toString(void) const
toString
Definition: TableVersion.cc:33
ots::TableVersion::isInvalid
bool isInvalid(void) const
isInvalid
Definition: TableVersion.cc:127
ots::TableVersion::getNextVersion
static TableVersion getNextVersion(const TableVersion &version=TableVersion())
Definition: TableVersion.cc:161
ots::TableVersion::isScratchVersion
bool isScratchVersion(void) const
Definition: TableVersion.cc:145
ots::TableVersion::isTemporaryVersion
bool isTemporaryVersion(void) const
Definition: TableVersion.cc:137
ots::TableVersion::getNextTemporaryVersion
static TableVersion getNextTemporaryVersion(const TableVersion &version=TableVersion())
Definition: TableVersion.cc:174
ots::TableView::setValueAsString
void setValueAsString(const std::string &value, unsigned int row, unsigned int col)
Definition: TableView.cc:1090
ots::TableView::setVersion
void setVersion(const T &version)
< in included .icc source
ots::TableView::getCustomStorageData
const std::string & getCustomStorageData(void) const
Getters.
Definition: TableView.h:71
ots::TableView::fillFromJSON
int fillFromJSON(const std::string &json)
Definition: TableView.cc:2401
ots::TableView::findCol
unsigned int findCol(const std::string &name) const
Definition: TableView.cc:1952
ots::TableView::setCustomStorageData
void setCustomStorageData(const std::string &storageData)
Definition: TableView.h:168
ots::GroupEditStruct::getTableEditStruct
TableEditStruct & getTableEditStruct(const std::string &tableName, bool markModified=false)
Note: if markModified, and table not found in group, this function will try to add it to group.
Definition: ConfigurationManagerRW.cc:2439
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
setToString ~
ots::StringMacros::mapToString
static std::string mapToString(const std::map< std::string, T > &mapToReturn, const std::string &primaryDelimeter=", ", const std::string &secondaryDelimeter=": ")
ots::StringMacros::stackTrace
static std::string stackTrace(void)
Definition: StringMacros.cc:1455