otsdaq  3.09.00
TableView.cc
1 #include "otsdaq/TableCore/TableView.h"
2 #include "otsdaq/Macros/StringMacros.h"
3 #include "otsdaq/TableCore/TableBase.h"
4 
5 #include <cstdlib>
6 #include <iostream>
7 #include <regex>
8 #include <sstream>
9 
10 using namespace ots;
11 
12 #undef __MF_SUBJECT__
13 #define __MF_SUBJECT__ "TableView"
14 #undef __COUT_HDR__
15 #define __COUT_HDR__ (tableName_ + "v" + version_.toString() + "\t<> ")
16 
17 const unsigned int TableView::INVALID = -1;
18 
19 //==============================================================================
20 TableView::TableView(const std::string& tableName)
21  : storageData_(tableName) // hijack momentarily for convert to caps
22  , tableName_(TableBase::convertToCaps(storageData_))
23  , version_(TableVersion::INVALID)
24  , comment_("")
25  , author_("")
26  , creationTime_(time(0))
27  , lastAccessTime_(0)
28  , colUID_(INVALID)
29  , colStatus_(INVALID)
30  , colPriority_(INVALID)
31  , fillWithLooseColumnMatching_(false)
32  , getSourceRawData_(false)
33  , sourceColumnMismatchCount_(0)
34  , sourceColumnMissingCount_(0)
35 {
36  storageData_ = ""; // unhijack
37 
38  if(tableName == "")
39  {
40  __SS__ << "Do not allow anonymous table view construction!" << __E__;
41  ss << StringMacros::stackTrace() << __E__;
42  __SS_THROW__;
43  }
44 
45 } // end constructor
46 
47 //==============================================================================
48 TableView::~TableView(void) {}
49 
50 //==============================================================================
54 TableView& TableView::operator=(const TableView /*src*/)
55 {
56  __SS__ << "Invalid use of operator=... Should not directly copy a TableView. Please "
57  "use TableView::copy(sourceView,author,comment)";
58  ss << StringMacros::stackTrace() << __E__;
59 
60  __COUT__ << ss.str() << __E__;
61  exit(0);
62  __SS_THROW__;
63 }
64 
65 //==============================================================================
66 TableView& TableView::copy(const TableView& src,
67  TableVersion destinationVersion,
68  const std::string& author)
69 {
70  // tableName_ = src.tableName_;
71  version_ = destinationVersion;
72  comment_ = src.comment_;
73  author_ = author; // take new author
74  // creationTime_ = time(0); //don't change creation time
75  lastAccessTime_ = time(0);
76 
77  // can not use operator= for TableViewColumn (it is a const class)
78  // columnsInfo_ = src.columnsInfo_;
79  columnsInfo_.clear();
80  for(auto& c : src.columnsInfo_)
81  columnsInfo_.push_back(c);
82 
83  theDataView_ = src.theDataView_;
84  sourceColumnNames_ = src.sourceColumnNames_;
85 
86  // RAR remove init() check, because usually copy() is only the first step
87  // in a series of changes that result in another call to init()
88  // init(); // verify consistency
89 
90  std::string tmpCachePrepend = TableBase::GROUP_CACHE_PREPEND;
91  tmpCachePrepend = TableBase::convertToCaps(tmpCachePrepend);
92  std::string tmpJsonDocPrepend = TableBase::JSON_DOC_PREPEND;
93  tmpJsonDocPrepend = TableBase::convertToCaps(tmpJsonDocPrepend);
94 
95  //if special GROUP CACHE table, handle construction in a special way
96  if(tableName_.substr(0, tmpCachePrepend.length()) == tmpCachePrepend ||
97  tableName_.substr(0, tmpJsonDocPrepend.length()) == tmpJsonDocPrepend)
98  {
99  __COUTT__ << "TableView copy for '" << tableName_ << "' done." << __E__;
100  return *this;
101  } //end special GROUP CACHE table construction
102 
103  initColUID(); // setup UID column
104  initRowDefaults();
105  try
106  {
107  initColStatus(); // setup Status column
108  }
109  catch(...)
110  {
111  } // ignore no Status column
112  try
113  {
114  initColPriority(); // setup Priority column
115  }
116  catch(...)
117  {
118  } // ignore no Priority column
119 
120  return *this;
121 } // end copy()
122 
123 //==============================================================================
126 unsigned int TableView::copyRows(const std::string& author,
127  const TableView& src,
128  unsigned int srcOffsetRow /* = 0 */,
129  unsigned int srcRowsToCopy /* = -1 */,
130  unsigned int destOffsetRow /* = -1 */,
131  unsigned char generateUniqueDataColumns /* = false */,
132  const std::string& baseNameAutoUID /*= "" */)
133 {
134  __COUTTV__(destOffsetRow);
135  __COUTTV__(srcOffsetRow);
136  __COUTTV__(srcRowsToCopy);
137 
138  unsigned int retRow = (unsigned int)-1;
139 
140  // check that column sizes match
141  if(src.getNumberOfColumns() != getNumberOfColumns())
142  {
143  __SS__ << "Error! Number of Columns of source view must match destination view. "
144  << "Source table=" << src.getTableName() << "_v" << src.getVersion()
145  << " cols=" << src.getNumberOfColumns() << " dest table=" << getTableName()
146  << "_v" << getVersion() << " cols=" << getNumberOfColumns() << __E__;
147  __SS_THROW__;
148  }
149 
150  unsigned int srcRows = src.getNumberOfRows();
151 
152  for(unsigned int r = 0; r < srcRowsToCopy; ++r)
153  {
154  if(r + srcOffsetRow >= srcRows)
155  break; // end when no more source rows to copy (past bounds)
156 
157  destOffsetRow = addRow(author,
158  generateUniqueDataColumns /*incrementUniqueData*/,
159  baseNameAutoUID /*baseNameAutoUID*/,
160  destOffsetRow); // add and get row created
161 
162  if(retRow == (unsigned int)-1)
163  retRow = destOffsetRow; // save row of first copied entry
164 
165  // copy data
166  for(unsigned int col = 0; col < getNumberOfColumns(); ++col)
167  if(generateUniqueDataColumns &&
168  (columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_UID ||
169  columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_UNIQUE_DATA ||
170  columnsInfo_[col].getType() ==
171  TableViewColumnInfo::TYPE_UNIQUE_GROUP_DATA))
172  continue; // if leaving unique data, then skip copy
173  else
174  {
175  __COUTT__ << "Copying [" << r + srcOffsetRow << "][" << col << "] to ["
176  << destOffsetRow << "][" << col
177  << "] = " << src.theDataView_[r + srcOffsetRow][col] << __E__;
178  theDataView_[destOffsetRow][col] =
179  src.theDataView_[r + srcOffsetRow][col];
180  }
181 
182  // prepare for next row
183  ++destOffsetRow;
184  }
185 
186  return retRow;
187 } // end copyRows()
188 
189 //==============================================================================
196 void TableView::init(void)
197 {
198  //__COUT__ << "Starting table verification..." << StringMacros::stackTrace() << __E__;
199 
200  try
201  {
202  // verify column names are unique
203  // make set of names,.. and CommentDescription == COMMENT
204  std::set<std::string> colNameSet;
205  std::string capsColName, colName;
206  for(auto& colInfo : columnsInfo_)
207  {
208  colName = colInfo.getStorageName();
209  if(colName == "COMMENT_DESCRIPTION")
210  colName = "COMMENT";
211  capsColName = "";
212  for(unsigned int i = 0; i < colName.size(); ++i)
213  {
214  if(colName[i] == '_')
215  continue;
216  capsColName += colName[i];
217  }
218 
219  colNameSet.emplace(capsColName);
220  }
221 
222  if(colNameSet.size() != columnsInfo_.size())
223  {
224  __SS__ << "Table Error:\t"
225  << " Columns names must be unique! There are " << columnsInfo_.size()
226  << " columns and the unique name count is " << colNameSet.size()
227  << __E__;
228  __SS_THROW__;
229  }
230 
231  initColUID(); // setup UID column
232  try
233  {
234  initColStatus(); // setup Status column
235  }
236  catch(...)
237  {
238  } // ignore no Status column
239  try
240  {
241  initColPriority(); // setup Priority column
242  }
243  catch(...)
244  {
245  } // ignore no Priority column
246 
247  // fix source columns if not already populated
248  if(sourceColumnNames_.size() == 0) // setup sourceColumnNames_ to be correct
249  for(unsigned int i = 0; i < getNumberOfColumns(); ++i)
250  sourceColumnNames_.emplace(getColumnsInfo()[i].getStorageName());
251 
252  // require one comment column
253  unsigned int colPos;
254  if((colPos = findColByType(TableViewColumnInfo::TYPE_COMMENT)) != INVALID)
255  {
256  if(columnsInfo_[colPos].getName() != TableViewColumnInfo::COL_NAME_COMMENT)
257  {
258  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_COMMENT
259  << " data type column must have name="
260  << TableViewColumnInfo::COL_NAME_COMMENT << __E__;
261  __SS_THROW__;
262  }
263 
264  if(findColByType(TableViewColumnInfo::TYPE_COMMENT, colPos + 1) !=
265  INVALID) // found two!
266  {
267  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_COMMENT
268  << " data type in column " << columnsInfo_[colPos].getName()
269  << " is repeated. This is not allowed." << __E__;
270  __SS_THROW__;
271  }
272 
273  if(colPos != getNumberOfColumns() - 3)
274  {
275  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_COMMENT
276  << " data type column must be 3rd to last (in column "
277  << getNumberOfColumns() - 3 << ")." << __E__;
278  __SS_THROW__;
279  }
280  }
281  else
282  {
283  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_COMMENT
284  << " data type column "
285  << " is missing. This is not allowed." << __E__;
286  __SS_THROW__;
287  }
288 
289  // require one author column
290  if((colPos = findColByType(TableViewColumnInfo::TYPE_AUTHOR)) != INVALID)
291  {
292  if(findColByType(TableViewColumnInfo::TYPE_AUTHOR, colPos + 1) !=
293  INVALID) // found two!
294  {
295  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_AUTHOR
296  << " data type in column " << columnsInfo_[colPos].getName()
297  << " is repeated. This is not allowed." << __E__;
298  __SS_THROW__;
299  }
300 
301  if(colPos != getNumberOfColumns() - 2)
302  {
303  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_AUTHOR
304  << " data type column must be 2nd to last (in column "
305  << getNumberOfColumns() - 2 << ")." << __E__;
306  __SS_THROW__;
307  }
308  }
309  else
310  {
311  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_AUTHOR
312  << " data type column "
313  << " is missing. This is not allowed." << __E__;
314  __SS_THROW__;
315  }
316 
317  // require one timestamp column
318  if((colPos = findColByType(TableViewColumnInfo::TYPE_TIMESTAMP)) != INVALID)
319  {
320  if(findColByType(TableViewColumnInfo::TYPE_TIMESTAMP, colPos + 1) !=
321  INVALID) // found two!
322  {
323  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_TIMESTAMP
324  << " data type in column " << columnsInfo_[colPos].getName()
325  << " is repeated. This is not allowed." << __E__;
326  __SS_THROW__;
327  }
328 
329  if(colPos != getNumberOfColumns() - 1)
330  {
331  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_TIMESTAMP
332  << " data type column must be last (in column "
333  << getNumberOfColumns() - 1 << ")." << __E__;
334  __COUT_ERR__ << "\n" << ss.str();
335  __SS_THROW__;
336  }
337  }
338  else
339  {
340  __SS__ << "Table Error:\t" << TableViewColumnInfo::TYPE_TIMESTAMP
341  << " data type column "
342  << " is missing. This is not allowed." << __E__;
343  __SS_THROW__;
344  }
345 
346  // check that UID is really unique ID (no repeats)
347  // and ... allow letters, numbers, dash, underscore
348  // and ... force size 1
349  std::set<std::string /*uid*/> uidSet;
350  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
351  {
352  if(uidSet.find(theDataView_[row][colUID_]) != uidSet.end())
353  {
354  __SS__ << ("Entries in UID are not unique. Specifically at row=" +
355  std::to_string(row) + " value=" + theDataView_[row][colUID_])
356  << __E__;
357  __SS_ONLY_THROW__;
358  }
359 
360  if(theDataView_[row][colUID_].size() == 0)
361  {
362  __SS__ << "An invalid UID '" << theDataView_[row][colUID_] << "' "
363  << " was identified. UIDs must contain at least 1 character."
364  << __E__;
365  __SS_ONLY_THROW__;
366  }
367 
368  for(unsigned int i = 0; i < theDataView_[row][colUID_].size(); ++i)
369  if(!((theDataView_[row][colUID_][i] >= 'A' &&
370  theDataView_[row][colUID_][i] <= 'Z') ||
371  (theDataView_[row][colUID_][i] >= 'a' &&
372  theDataView_[row][colUID_][i] <= 'z') ||
373  (theDataView_[row][colUID_][i] >= '0' &&
374  theDataView_[row][colUID_][i] <= '9') ||
375  (theDataView_[row][colUID_][i] == '-' ||
376  theDataView_[row][colUID_][i] == '_')))
377  {
378  __SS__ << "An invalid UID '" << theDataView_[row][colUID_] << "' "
379  << " was identified. UIDs must contain only letters, numbers, "
380  << "dashes, and underscores." << __E__;
381  __SS_ONLY_THROW__;
382  }
383 
384  uidSet.insert(theDataView_[row][colUID_]);
385  }
386  if(uidSet.size() != getNumberOfRows())
387  {
388  __SS__ << "Entries in UID are not unique!"
389  << "There are " << getNumberOfRows()
390  << " records and the unique UID count is " << uidSet.size() << __E__;
391  __SS_ONLY_THROW__;
392  }
393 
394  // check that any TYPE_UNIQUE_DATA columns are really unique (no repeats)
395  colPos = (unsigned int)-1;
396  while((colPos = findColByType(TableViewColumnInfo::TYPE_UNIQUE_DATA,
397  colPos + 1)) != INVALID)
398  {
399  std::set<std::string /*unique data*/> uDataSet;
400  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
401  {
402  if(uDataSet.find(theDataView_[row][colPos]) != uDataSet.end())
403  {
404  __SS__ << "Entries in Unique Data column "
405  << columnsInfo_[colPos].getName()
406  << (" are not unique. Specifically at row=" +
407  std::to_string(row) +
408  " value=" + theDataView_[row][colPos])
409  << __E__;
410  __SS_THROW__;
411  }
412  uDataSet.insert(theDataView_[row][colPos]);
413  }
414  if(uDataSet.size() != getNumberOfRows())
415  {
416  __SS__ << "Entries in Unique Data column "
417  << columnsInfo_[colPos].getName() << " are not unique!"
418  << "There are " << getNumberOfRows()
419  << " records and the unique data count is " << uDataSet.size()
420  << __E__;
421  __SS_THROW__;
422  }
423  }
424 
425  // check that any TYPE_UNIQUE_GROUP_DATA columns are really unique fpr groups (no
426  // repeats)
427  colPos = (unsigned int)-1;
428  while((colPos = findColByType(TableViewColumnInfo::TYPE_UNIQUE_GROUP_DATA,
429  colPos + 1)) != INVALID)
430  {
431  // colPos is a unique group data column
432  // now, for each groupId column
433  // check that data is unique for all groups
434  for(unsigned int groupIdColPos = 0; groupIdColPos < columnsInfo_.size();
435  ++groupIdColPos)
436  if(columnsInfo_[groupIdColPos].isGroupID())
437  {
438  std::map<std::string /*group name*/,
439  std::pair<unsigned int /*memberCount*/,
440  std::set<std::string /*unique data*/>>>
441  uGroupDataSets;
442 
443  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
444  {
445  auto groupIds = getSetOfGroupIDs(groupIdColPos, row);
446 
447  for(const auto& groupId : groupIds)
448  {
449  uGroupDataSets[groupId].first++; // add to member count
450 
451  if(uGroupDataSets[groupId].second.find(
452  theDataView_[row][colPos]) !=
453  uGroupDataSets[groupId].second.end())
454  {
455  __SS__ << "Entries in Unique Group Data column " << colPos
456  << ":" << columnsInfo_[colPos].getName()
457  << " are not unique for group ID '" << groupId
458  << ".' Specifically at row=" << std::to_string(row)
459  << " value=" << theDataView_[row][colPos] << __E__;
460  __SS_THROW__;
461  }
462  uGroupDataSets[groupId].second.insert(
463  theDataView_[row][colPos]);
464  }
465  }
466 
467  for(const auto& groupPair : uGroupDataSets)
468  if(uGroupDataSets[groupPair.first].second.size() !=
469  uGroupDataSets[groupPair.first].first)
470  {
471  __SS__
472  << "Entries in Unique Data column "
473  << columnsInfo_[colPos].getName()
474  << " are not unique for group '" << groupPair.first
475  << "!'"
476  << "There are " << uGroupDataSets[groupPair.first].first
477  << " records and the unique data count is "
478  << uGroupDataSets[groupPair.first].second.size() << __E__;
479  __SS_THROW__;
480  }
481  }
482  } // end TYPE_UNIQUE_GROUP_DATA check
483 
484  auto rowDefaults = initRowDefaults(); // getDefaultRowValues();
485 
486  // check that column types are well behaved
487  // - check that fixed choice data is one of choices
488  // - sanitize booleans
489  // - check that child link I are unique
490  // note: childLinkId refers to childLinkGroupIDs AND childLinkUIDs
491  std::set<std::string> groupIdIndexes, childLinkIndexes, childLinkIdLabels;
492  unsigned int groupIdIndexesCount = 0, childLinkIndexesCount = 0,
493  childLinkIdLabelsCount = 0;
494  bool tmpIsGroup;
495  std::pair<unsigned int /*link col*/, unsigned int /*link id col*/> tmpLinkPair;
496 
497  // check sanity of data view rows x cols (have seen weird out-of-range crashes)
498  if(getNumberOfRows() != theDataView_.size())
499  {
500  __SS__ << "Impossible row mismatch " << getNumberOfRows() << " vs "
501  << theDataView_.size() << "! How did you get here?" << __E__;
502  __SS_THROW__;
503  }
504  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
505  if(getNumberOfColumns() != theDataView_[row].size())
506  {
507  __SS__ << "Impossible col mismatch " << getNumberOfColumns() << " vs ["
508  << row << "]" << theDataView_[row].size()
509  << "! How did you get here?" << __E__;
510  __SS_THROW__;
511  }
512  if(getNumberOfColumns() != columnsInfo_.size())
513  {
514  __SS__ << "Impossible col info mismatch " << getNumberOfColumns() << " vs "
515  << columnsInfo_.size() << "! How did you get here?" << __E__;
516  __SS_THROW__;
517  }
518  if(getNumberOfColumns() != rowDefaults.size())
519  {
520  __SS__ << "Impossible col default mismatch " << getNumberOfColumns() << " vs "
521  << rowDefaults.size() << "! How did you get here?" << __E__;
522  __SS_THROW__;
523  }
524 
525  for(unsigned int col = 0; col < getNumberOfColumns(); ++col)
526  {
527  if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_FIXED_CHOICE_DATA)
528  {
529  const std::vector<std::string>& theDataChoices =
530  columnsInfo_[col].getDataChoices();
531 
532  // check if arbitrary values allowed
533  if(theDataChoices.size() && theDataChoices[0] == "arbitraryBool=1")
534  continue; // arbitrary values allowed
535 
536  bool found;
537  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
538  {
539  found = false;
540  // check against default value first
541  if(theDataView_[row][col] == rowDefaults[col])
542  continue; // default is always ok
543 
544  for(const auto& choice : theDataChoices)
545  {
546  if(theDataView_[row][col] == choice)
547  {
548  found = true;
549  break;
550  }
551  }
552  if(!found)
553  {
554  __SS__ << getTableName() << " Error:\t'" << theDataView_[row][col]
555  << "' in column " << columnsInfo_[col].getName()
556  << " is not a valid Fixed Choice option. "
557  << "Possible values are as follows: ";
558 
559  ss << columnsInfo_[col].getDefaultValue()
560  << (columnsInfo_[col].getDataChoices().size() ? ", " : "");
561  for(unsigned int i = 0;
562  i < columnsInfo_[col].getDataChoices().size();
563  ++i)
564  {
565  if(columnsInfo_[col].getDataChoices()[i] == "arbitraryBool=0")
566  continue; //skip printout of arbitrary bool field first
567 
568  if(i && (i != 1 || columnsInfo_[col].getDataChoices()[0] !=
569  "arbitraryBool=0"))
570  ss << ", ";
571  ss << columnsInfo_[col].getDataChoices()[i];
572  }
573  ss << "." << __E__;
574  __SS_ONLY_THROW__;
575  }
576  }
577  }
578  else if(columnsInfo_[col].isChildLink())
579  {
580  // check if forcing fixed choices
581 
582  const std::vector<std::string>& theDataChoices =
583  columnsInfo_[col].getDataChoices();
584 
585  // check if arbitrary values allowed
586  if(!theDataChoices.size() || theDataChoices[0] == "arbitraryBool=1")
587  continue; // arbitrary values allowed
588 
589  // skip one if arbitrary setting is embedded as first value
590  bool skipOne =
591  (theDataChoices.size() && theDataChoices[0] == "arbitraryBool=0");
592  bool hasSkipped;
593 
594  bool found;
595  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
596  {
597  found = false;
598 
599  hasSkipped = false;
600  for(const auto& choice : theDataChoices)
601  {
602  if(skipOne && !hasSkipped)
603  {
604  hasSkipped = true;
605  continue;
606  }
607 
608  if(theDataView_[row][col] == choice)
609  {
610  found = true;
611  break;
612  }
613  }
614  if(!found)
615  {
616  __SS__ << getTableName() << " Error:\t the value '"
617  << theDataView_[row][col] << "' in column "
618  << columnsInfo_[col].getName()
619  << " is not a valid Fixed Choice option. "
620  << "Possible values are as follows: ";
621 
622  // ss <<
623  // StringMacros::vectorToString(columnsInfo_[col].getDataChoices())
624  // << __E__;
625  for(unsigned int i = skipOne ? 1 : 0;
626  i < columnsInfo_[col].getDataChoices().size();
627  ++i)
628  {
629  if(i > (skipOne ? 1 : 0))
630  ss << ", ";
631  ss << columnsInfo_[col].getDataChoices()[i];
632  }
633  ss << "." << __E__;
634  __SS_ONLY_THROW__;
635  }
636  }
637  }
638  else if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_ON_OFF)
639  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
640  {
641  if(theDataView_[row][col] == "1" ||
642  theDataView_[row][col] == "TRUE" || //excel bool
643  theDataView_[row][col] == "on" || theDataView_[row][col] == "On" ||
644  theDataView_[row][col] == "ON")
645  theDataView_[row][col] = TableViewColumnInfo::TYPE_VALUE_ON;
646  else if(theDataView_[row][col] == "0" ||
647  theDataView_[row][col] == "FALSE" || //excel bool
648  theDataView_[row][col] == "off" ||
649  theDataView_[row][col] == "Off" ||
650  theDataView_[row][col] == "OFF")
651  theDataView_[row][col] = TableViewColumnInfo::TYPE_VALUE_OFF;
652  else
653  {
654  __SS__ << getTableName() << " Error:\t the value '"
655  << theDataView_[row][col] << "' in column "
656  << columnsInfo_[col].getName()
657  << " is not a valid Type (On/Off) std::string. Possible "
658  "values are 1, on, On, ON, 0, off, Off, OFF."
659  << __E__;
660  __SS_ONLY_THROW__;
661  }
662  }
663  else if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_TRUE_FALSE)
664  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
665  {
666  if(theDataView_[row][col] == "1" ||
667  theDataView_[row][col] == "true" ||
668  theDataView_[row][col] == "True" ||
669  theDataView_[row][col] == "TRUE")
670  theDataView_[row][col] = TableViewColumnInfo::TYPE_VALUE_TRUE;
671  else if(theDataView_[row][col] == "0" ||
672  theDataView_[row][col] == "false" ||
673  theDataView_[row][col] == "False" ||
674  theDataView_[row][col] == "FALSE")
675  theDataView_[row][col] = TableViewColumnInfo::TYPE_VALUE_FALSE;
676  else
677  {
678  __SS__ << getTableName() << " Error:\t the value '"
679  << theDataView_[row][col] << "' in column "
680  << columnsInfo_[col].getName()
681  << " is not a valid Type (True/False) std::string. "
682  "Possible values are 1, true, True, TRUE, 0, false, "
683  "False, FALSE."
684  << __E__;
685  __SS_ONLY_THROW__;
686  }
687  }
688  else if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_YES_NO)
689  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
690  {
691  if(theDataView_[row][col] == "1" ||
692  theDataView_[row][col] == "TRUE" || //excel bool
693  theDataView_[row][col] == "yes" ||
694  theDataView_[row][col] == "Yes" || theDataView_[row][col] == "YES")
695  theDataView_[row][col] = TableViewColumnInfo::TYPE_VALUE_YES;
696  else if(theDataView_[row][col] == "0" ||
697  theDataView_[row][col] == "FALSE" || //excel bool
698  theDataView_[row][col] == "no" ||
699  theDataView_[row][col] == "No" ||
700  theDataView_[row][col] == "NO")
701  theDataView_[row][col] = TableViewColumnInfo::TYPE_VALUE_NO;
702  else
703  {
704  __SS__ << getTableName() << " Error:\t the value '"
705  << theDataView_[row][col] << "' in column "
706  << columnsInfo_[col].getName()
707  << " is not a valid Type (Yes/No) std::string. Possible "
708  "values are 1, yes, Yes, YES, 0, no, No, NO."
709  << __E__;
710  __SS_ONLY_THROW__;
711  }
712  }
713  else if(columnsInfo_[col].isGroupID()) // GroupID type
714  {
715  colLinkGroupIDs_[columnsInfo_[col].getChildLinkIndex()] =
716  col; // add to groupid map
717  // check uniqueness
718  groupIdIndexes.emplace(columnsInfo_[col].getChildLinkIndex());
719  ++groupIdIndexesCount;
720  }
721  else if(columnsInfo_[col].isChildLink()) // Child Link type
722  {
723  // sanitize no link to default
724  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
725  if(theDataView_[row][col] == "NoLink" ||
726  theDataView_[row][col] == "No_Link" ||
727  theDataView_[row][col] == "NOLINK" ||
728  theDataView_[row][col] == "NO_LINK" ||
729  theDataView_[row][col] == "Nolink" ||
730  theDataView_[row][col] == "nolink" ||
731  theDataView_[row][col] == "noLink")
732  theDataView_[row][col] =
733  TableViewColumnInfo::DATATYPE_LINK_DEFAULT;
734 
735  // check uniqueness
736  childLinkIndexes.emplace(columnsInfo_[col].getChildLinkIndex());
737  ++childLinkIndexesCount;
738 
739  // force data type to TableViewColumnInfo::DATATYPE_STRING
740  if(columnsInfo_[col].getDataType() !=
741  TableViewColumnInfo::DATATYPE_STRING)
742  {
743  __SS__ << getTableName() << " Error:\t"
744  << "Column " << col << " with name '"
745  << columnsInfo_[col].getName()
746  << "' is a Child Link column and has an illegal data type of '"
747  << columnsInfo_[col].getDataType()
748  << "'. The data type for Child Link columns must be "
749  << TableViewColumnInfo::DATATYPE_STRING << __E__;
750  __SS_THROW__;
751  }
752 
753  // check for link mate (i.e. every child link needs link ID)
754  getChildLink(col, tmpIsGroup, tmpLinkPair);
755  }
756  else if(columnsInfo_[col].isChildLinkUID() || // Child Link ID type
757  columnsInfo_[col].isChildLinkGroupID())
758  {
759  // check uniqueness
760  childLinkIdLabels.emplace(columnsInfo_[col].getChildLinkIndex());
761  ++childLinkIdLabelsCount;
762 
763  // check that the Link ID is not empty, and force to default
764  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
765  if(theDataView_[row][col] == "")
766  theDataView_[row][col] = rowDefaults[col];
767 
768  // check for link mate (i.e. every child link needs link ID)
769  getChildLink(col, tmpIsGroup, tmpLinkPair);
770  }
771 
772  // check if number exist and then if it is limited by min and max, use functions in here to get values different than stof
773  if(columnsInfo_[col].isNumberDataType())
774  {
775  std::string minimumValueString = columnsInfo_[col].getMinValue();
776  std::string maximumValueString = columnsInfo_[col].getMaxValue();
777  double minimumValue, maximumValue, valueFromTable;
778  bool minExists = false, maxExists = false;
779 
780  if(!minimumValueString.empty())
781  {
782  minExists = StringMacros::getNumber(
783  StringMacros::convertEnvironmentVariables(minimumValueString),
784  minimumValue);
785  if(!minExists)
786  {
787  __SS__ << "Inavlid user spec'd min value '" << minimumValueString
788  << "' which is not a valid number. The minimum value must "
789  "be a number (environment variables and math "
790  "operations are allowed)."
791  << __E__;
792  __SS_THROW__;
793  }
794  }
795 
796  if(!maximumValueString.empty())
797  {
798  maxExists = StringMacros::getNumber(
799  StringMacros::convertEnvironmentVariables(maximumValueString),
800  maximumValue);
801  if(!maxExists)
802  {
803  __SS__ << "Inavlid user spec'd max value '" << maximumValueString
804  << "' which is not a valid number. The maximum value must "
805  "be a number (environment variables and math "
806  "operations are allowed)."
807  << __E__;
808  __SS_THROW__;
809  }
810  }
811 
812  if(minExists && maxExists && minimumValue > maximumValue)
813  {
814  __SS__ << "Minimum value is greater than maximum, check table editor "
815  "to change this"
816  << __E__;
817  __SS_THROW__;
818  }
819 
820  if(minExists || maxExists)
821  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
822  {
823  getValue(valueFromTable, row, col);
824  if(minExists && valueFromTable < minimumValue)
825  {
826  __SS__
827  << "The value '" << valueFromTable << "'("
828  << getValueAsString(
829  row, col, false /* convertEnvironmentVariables */)
830  << ") at [row,col]=[" << row << "," << col
831  << "] is outside the established limits: "
832  << valueFromTable
833  << " is lower than the specified minimum " << minimumValue
834  << "." << __E__;
835  __SS_THROW__;
836  }
837  if(maxExists && valueFromTable > maximumValue)
838  {
839  __SS__
840  << "This value '" << valueFromTable << "'("
841  << getValueAsString(
842  row, col, false /* convertEnvironmentVariables */)
843  << ") at [row,col]=[" << row << "," << col
844  << "] is outside the established limits: "
845  << valueFromTable
846  << " is greater than the specified maximum "
847  << maximumValue << "." << __E__;
848  __SS_THROW__;
849  }
850  }
851  } // end handling NUMBER data types
852  } // end column loop
853 
854  // verify child link index uniqueness
855  if(groupIdIndexes.size() != groupIdIndexesCount)
856  {
857  __SS__ << ("GroupId Labels are not unique!") << "There are "
858  << groupIdIndexesCount << " GroupId Labels and the unique count is "
859  << groupIdIndexes.size() << __E__;
860  __SS_THROW__;
861  }
862  if(childLinkIndexes.size() != childLinkIndexesCount)
863  {
864  __SS__ << ("Child Link Labels are not unique!") << "There are "
865  << childLinkIndexesCount
866  << " Child Link Labels and the unique count is "
867  << childLinkIndexes.size() << __E__;
868  __SS_THROW__;
869  }
870  if(childLinkIdLabels.size() != childLinkIdLabelsCount)
871  {
872  __SS__ << ("Child Link ID Labels are not unique!") << "There are "
873  << childLinkIdLabelsCount
874  << " Child Link ID Labels and the unique count is "
875  << childLinkIdLabels.size() << __E__;
876  __SS_THROW__;
877  }
878  }
879  catch(...)
880  {
881  __COUTT__ << "Error occurred in TableView::init() for version=" << version_
882  << __E__;
883  throw;
884  }
885 } // end init()
886 
887 //==============================================================================
892 void TableView::getValue(std::string& value,
893  unsigned int row,
894  unsigned int col,
895  bool doConvertEnvironmentVariables) const
896 {
897  if(!(row < getNumberOfRows() && col < theDataView_[row].size()))
898  {
899  __SS__ << "Invalid row col requested " << row << "," << col << " vs "
900  << getNumberOfRows() << "," << columnsInfo_.size() << "/"
901  << theDataView_[row].size() << __E__;
902  __SS_THROW__;
903  }
904 
905  value = validateValueForColumn(
906  theDataView_[row][col], col, doConvertEnvironmentVariables);
907 } // end getValue()
908 
909 //==============================================================================
914 std::string TableView::validateValueForColumn(const std::string& value,
915  unsigned int col,
916  bool doConvertEnvironmentVariables) const
917 {
918  if(col >= columnsInfo_.size())
919  {
920  __SS__ << "Invalid col requested" << __E__;
921  __SS_THROW__;
922  }
923 
924  if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_FIXED_CHOICE_DATA &&
925  // value == columnsInfo_[col].getDefaultValue())
926  value == columnsInfo_[col].getDefaultDefaultValue(columnsInfo_[col].getType(),
927  columnsInfo_[col].getDataType()))
928  {
929  // if type string, fixed choice and DEFAULT, then return string of first choice
930 
931  std::vector<std::string> choices = columnsInfo_[col].getDataChoices();
932 
933  // consider arbitrary bool
934  bool skipOne = (choices.size() && choices[0].find("arbitraryBool=") == 0);
935  size_t index = (skipOne ? 1 : 0);
936  if(choices.size() > index)
937  {
938  return doConvertEnvironmentVariables
939  ? StringMacros::convertEnvironmentVariables(choices[index])
940  : choices[index]; // handled value from fixed choices
941  }
942  } // end handling default to fixed choice conversion
943 
944  if(columnsInfo_[col].getDataType() == TableViewColumnInfo::DATATYPE_STRING)
945  return doConvertEnvironmentVariables
946  ? StringMacros::convertEnvironmentVariables(value)
947  : value;
948  else if(columnsInfo_[col].getDataType() == TableViewColumnInfo::DATATYPE_TIME)
949  {
950  return StringMacros::getTimestampString(
951  doConvertEnvironmentVariables
952  ? StringMacros::convertEnvironmentVariables(value)
953  : value);
954  }
955  else
956  {
957  __SS__ << "\tUnrecognized column data type: " << columnsInfo_[col].getDataType()
958  << " in configuration " << tableName_
959  << " at column=" << columnsInfo_[col].getName()
960  << " for getValue with type '"
961  << StringMacros::demangleTypeName(typeid(std::string).name()) << "'"
962  << __E__;
963  __SS_THROW__;
964  }
965 
966  // return retValue;
967 } // end validateValueForColumn()
968 
969 //==============================================================================
973 std::string TableView::getValueAsString(unsigned int row,
974  unsigned int col,
975  bool doConvertEnvironmentVariables) const
976 {
977  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
978  {
979  __SS__ << "Invalid row col requested: row=" << row
980  << " numRows=" << getNumberOfRows() << " col=" << col
981  << " numCols=" << columnsInfo_.size() << " table=" << tableName_ << __E__;
982  __SS_THROW__;
983  }
984 
985  __COUTS__(30) << columnsInfo_[col].getType() << " " << col << __E__;
986 
987  if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_ON_OFF)
988  {
989  if(theDataView_[row][col] == "1" || theDataView_[row][col] == "on" ||
990  theDataView_[row][col] == "On" || theDataView_[row][col] == "ON")
991  return TableViewColumnInfo::TYPE_VALUE_ON;
992  else
993  return TableViewColumnInfo::TYPE_VALUE_OFF;
994  }
995  else if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_TRUE_FALSE)
996  {
997  if(theDataView_[row][col] == "1" || theDataView_[row][col] == "true" ||
998  theDataView_[row][col] == "True" || theDataView_[row][col] == "TRUE")
999  return TableViewColumnInfo::TYPE_VALUE_TRUE;
1000  else
1001  return TableViewColumnInfo::TYPE_VALUE_FALSE;
1002  }
1003  else if(columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_YES_NO)
1004  {
1005  if(theDataView_[row][col] == "1" || theDataView_[row][col] == "yes" ||
1006  theDataView_[row][col] == "Yes" || theDataView_[row][col] == "YES")
1007  return TableViewColumnInfo::TYPE_VALUE_YES;
1008  else
1009  return TableViewColumnInfo::TYPE_VALUE_NO;
1010  }
1011 
1012  return doConvertEnvironmentVariables
1013  ? StringMacros::convertEnvironmentVariables(theDataView_[row][col])
1014  : theDataView_[row][col];
1015 } //end getValueAsString()
1016 
1017 //==============================================================================
1024 std::string TableView::getEscapedValueAsString(
1025  unsigned int row,
1026  unsigned int col,
1027  bool doConvertEnvironmentVariables /* = true */,
1028  bool quotesToDoubleQuotes /* = false */) const
1029 {
1030  std::string val = getValueAsString(row, col, doConvertEnvironmentVariables);
1031  std::string retVal = "";
1032  retVal.reserve(val.size()); // reserve roughly right size
1033  for(unsigned int i = 0; i < val.size(); ++i)
1034  {
1035  if(val[i] == '\n')
1036  retVal += "\\n";
1037  else if(val[i] == '\t')
1038  retVal += "\\t";
1039  else if(val[i] == '\r')
1040  retVal += "\\r";
1041  else
1042  {
1043  // escaped characters need a
1044  if(val[i] == '\\')
1045  retVal += '\\';
1046  if(quotesToDoubleQuotes && val[i] == '"')
1047  retVal += '"'; //convert " to "" for excel style CSV
1048  else if(!quotesToDoubleQuotes && val[i] == '"')
1049  retVal += '\\';
1050  retVal += val[i];
1051  }
1052  }
1053  return retVal;
1054 } //end getEscapedValueAsString()
1055 
1056 //==============================================================================
1059 void TableView::setValue(const std::string& value, unsigned int row, unsigned int col)
1060 {
1061  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
1062  {
1063  __SS__ << "Invalid row (" << row << ") col (" << col << ") requested!" << __E__;
1064  __SS_THROW__;
1065  }
1066 
1067  if(columnsInfo_[col].getDataType() == TableViewColumnInfo::DATATYPE_STRING)
1068  theDataView_[row][col] = value;
1069  else // dont allow TableViewColumnInfo::DATATYPE_TIME to be set as string.. force use
1070  // as time_t to standardize string result
1071  {
1072  __SS__ << "\tUnrecognized column data type: " << columnsInfo_[col].getDataType()
1073  << " in configuration " << tableName_
1074  << " at column=" << columnsInfo_[col].getName()
1075  << " for setValue with type '"
1076  << StringMacros::demangleTypeName(typeid(value).name()) << "'" << __E__;
1077  __SS_THROW__;
1078  }
1079 } // end setValue()
1080 
1081 //==============================================================================
1082 void TableView::setValue(const char* value, unsigned int row, unsigned int col)
1083 {
1084  setValue(std::string(value), row, col);
1085 } // end setValue()
1086 
1087 //==============================================================================
1090 void TableView::setValueAsString(const std::string& value,
1091  unsigned int row,
1092  unsigned int col)
1093 {
1094  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
1095  {
1096  __SS__ << "Invalid row (" << row << ") col (" << col << ") requested!" << __E__;
1097  __SS_THROW__;
1098  }
1099 
1100  theDataView_[row][col] = value;
1101 } // end setValueAsString()
1102 
1103 //==============================================================================
1110 const std::string& TableView::setUniqueColumnValue(
1111  unsigned int row,
1112  unsigned int col,
1113  std::string baseValueAsString /*= "" */,
1114  bool doMathAppendStrategy /*= false*/,
1115  std::string childLinkIndex /* = "" */,
1116  std::string groupId /* = "" */)
1117 {
1118  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
1119  {
1120  __SS__ << "Invalid row (" << row << ") col (" << col << ") requested!" << __E__;
1121  __SS_THROW__;
1122  }
1123 
1124  bool isUniqueGroupCol =
1125  (columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_UNIQUE_GROUP_DATA);
1126  unsigned int childLinkIndexCol = -1;
1127  if(isUniqueGroupCol)
1128  {
1129  __COUTVS__(12, childLinkIndex); //set TRACE level to TLVL_DEBUG + 12
1130  __COUTVS__(12, groupId); //set TRACE level to TLVL_DEBUG + 12
1131  childLinkIndexCol = getLinkGroupIDColumn(childLinkIndex); // column in question
1132  __COUTVS__(12, childLinkIndexCol); //set TRACE level to TLVL_DEBUG + 12
1133  }
1134 
1135  __COUTT__ << "Current '" << columnsInfo_[col].getName() << "' "
1136  << (isUniqueGroupCol ? "(Unique in Group) " : "")
1137  << "unique data entry is data[" << row << "][" << col << "] = '"
1138  << theDataView_[row][col] << "' baseValueAsString = " << baseValueAsString
1139  << " doMathAppendStrategy = " << doMathAppendStrategy << __E__;
1140 
1141  bool firstConflict = true;
1142  int maxUniqueData = -1;
1143  std::string tmpString = "";
1144  bool foundAny;
1145  unsigned int index;
1146  std::string numString;
1147  std::string opString; // for doMathAppendStrategy
1148 
1149  // find max in rows
1150 
1151  // this->print();
1152 
1153  for(unsigned int r = 0; r < getNumberOfRows(); ++r)
1154  {
1155  if(r == row)
1156  continue; // skip row to add
1157 
1158  if(isUniqueGroupCol && !isEntryInGroupCol(r, childLinkIndexCol, groupId))
1159  continue; // skip rows not in group
1160 
1161  // find last non numeric character
1162 
1163  foundAny = false;
1164  tmpString = theDataView_[r][col];
1165 
1166  __COUTS__(3) << "row[" << r << "] tmpString " << tmpString << __E__;
1167 
1168  for(index = tmpString.length() - 1; index < tmpString.length(); --index)
1169  {
1170  __COUTS__(3) << index << " tmpString[index] " << tmpString[index] << __E__;
1171  if(!(tmpString[index] >= '0' && tmpString[index] <= '9'))
1172  break; // if not numeric, break
1173  foundAny = true;
1174  }
1175 
1176  __COUTS__(3) << "index " << index << " foundAny " << foundAny << __E__;
1177 
1178  if(tmpString.length() && foundAny) // then found a numeric substring
1179  {
1180  // create numeric substring
1181  numString = tmpString.substr(index + 1);
1182 
1183  // and alpha basestring
1184  tmpString = tmpString.substr(0, index + 1);
1185 
1186  if(doMathAppendStrategy && tmpString.size())
1187  {
1188  // look for op string
1189  foundAny = false;
1190  for(index = tmpString.length() - 1; index < tmpString.length(); --index)
1191  {
1192  __COUTS__(4)
1193  << index << " tmpString[index] " << tmpString[index] << __E__;
1194  if(!(tmpString[index] == '+' || tmpString[index] == ' '))
1195  break; // if not plus op, break
1196  foundAny = true;
1197  }
1198 
1199  if(foundAny)
1200  {
1201  // create numeric substring
1202  opString = tmpString.substr(index + 1);
1203 
1204  // and alpha basestring
1205  tmpString = tmpString.substr(0, index + 1);
1206  }
1207  }
1208 
1209  __COUTS__(3) << tmpString << " vs " << baseValueAsString << __E__;
1210 
1211  if(baseValueAsString != "" && tmpString != baseValueAsString)
1212  continue; // skip max unique number if basestring does not match
1213 
1214  __COUTS__(3) << "Found unique data base string '" << tmpString
1215  << "' and number string '" << numString << "' in last record '"
1216  << theDataView_[r][col] << "'" << __E__;
1217 
1218  if(firstConflict)
1219  {
1220  // if baseValueAsString ends in number, then add _ to keep naming similar
1221  if(baseValueAsString.size() &&
1222  baseValueAsString[baseValueAsString.size() - 1] >= '0' &&
1223  baseValueAsString[baseValueAsString.size() - 1] <= '9')
1224  baseValueAsString += '_';
1225 
1226  firstConflict = false;
1227  }
1228 
1229  // extract number
1230  sscanf(numString.c_str(), "%u", &index);
1231 
1232  if((int)index > maxUniqueData)
1233  {
1234  maxUniqueData = (int)index;
1235 
1236  if(baseValueAsString == "")
1237  baseValueAsString = tmpString; // assume a value for base string
1238  }
1239  }
1240  else if(maxUniqueData < 0 &&
1241  (baseValueAsString == "" || tmpString == baseValueAsString))
1242  {
1243  if(firstConflict)
1244  {
1245  // if baseValueAsString ends in number, then add _ to keep naming similar
1246  if(baseValueAsString.size() &&
1247  baseValueAsString[baseValueAsString.size() - 1] >= '0' &&
1248  baseValueAsString[baseValueAsString.size() - 1] <= '9')
1249  baseValueAsString += '_';
1250 
1251  firstConflict = false;
1252  }
1253 
1254  maxUniqueData = 0; // start a number if basestring conflict
1255  }
1256  } //end loop finding max unique data (potentially for group)
1257 
1258  __COUTVS__(12, maxUniqueData); //set TRACE level to TLVL_DEBUG + 12
1259  __COUTVS__(12, baseValueAsString); //set TRACE level to TLVL_DEBUG + 12
1260 
1261  if(maxUniqueData == -1) // if no conflicts, then do not add number
1262  {
1263  if(baseValueAsString != "")
1264  theDataView_[row][col] = baseValueAsString;
1265  else
1266  theDataView_[row][col] = columnsInfo_[col].getDefaultValue();
1267  }
1268  else
1269  {
1270  ++maxUniqueData; // increment
1271 
1272  char indexString[1000];
1273  sprintf(indexString, "%u", maxUniqueData);
1274 
1275  __COUTVS__(12, indexString); //set TRACE level to TLVL_DEBUG + 12
1276  __COUTVS__(12, baseValueAsString); //set TRACE level to TLVL_DEBUG + 12
1277 
1278  if(doMathAppendStrategy)
1279  theDataView_[row][col] = baseValueAsString + " + " + indexString;
1280  else
1281  theDataView_[row][col] = baseValueAsString + indexString;
1282  }
1283 
1284  __COUTT__ << "New unique data entry is data[" << row << "][" << col << "] = '"
1285  << theDataView_[row][col] << "'" << __E__;
1286 
1287  if(TTEST(13))
1288  {
1289  std::stringstream ss;
1290  this->print(ss);
1291  __COUT_MULTI__(13, ss.str());
1292  }
1293 
1294  return theDataView_[row][col];
1295 } // end setUniqueColumnValue()
1296 
1297 //==============================================================================
1300 unsigned int TableView::initColUID(void)
1301 {
1302  if(colUID_ != INVALID)
1303  return colUID_;
1304 
1305  // if doesn't exist throw error! each view must have a UID column
1306  colUID_ = findColByType(TableViewColumnInfo::TYPE_UID);
1307  if(colUID_ == INVALID)
1308  {
1309  __COUT__ << "Column Types: " << __E__;
1310  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1311  std::cout << columnsInfo_[col].getType() << "() "
1312  << columnsInfo_[col].getName() << __E__;
1313  __SS__ << "\tMissing UID Column in table named '" << tableName_ << "'" << __E__;
1314  __SS_THROW__;
1315  }
1316  return colUID_;
1317 }
1318 //==============================================================================
1322 unsigned int TableView::getColUID(void) const
1323 {
1324  if(colUID_ != INVALID)
1325  return colUID_;
1326 
1327  __COUT__ << "Column Types: " << __E__;
1328  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1329  std::cout << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName()
1330  << __E__;
1331 
1332  __SS__ << ("Missing UID Column in config named " + tableName_ +
1333  ". (Possibly TableView was just not initialized?" +
1334  " This is the const call so can not alter class members)")
1335  << __E__;
1336 
1337  ss << StringMacros::stackTrace() << __E__;
1338 
1339  __SS_THROW__;
1340 }
1341 
1342 //==============================================================================
1345 unsigned int TableView::initColStatus(void)
1346 {
1347  if(colStatus_ != INVALID)
1348  return colStatus_;
1349 
1350  // if doesn't exist throw error! each view must have a UID column
1351  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1352  if(columnsInfo_[col].getName() == TableViewColumnInfo::COL_NAME_STATUS)
1353  {
1354  colStatus_ = col;
1355  return colStatus_;
1356  }
1357  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1358  if(columnsInfo_[col].getName() == TableViewColumnInfo::COL_NAME_ENABLED)
1359  {
1360  colStatus_ = col;
1361  return colStatus_;
1362  }
1363 
1364  // at this point not found!
1365 
1366  __SS__ << "\tMissing column named '" << TableViewColumnInfo::COL_NAME_STATUS
1367  << "' or '" << TableViewColumnInfo::COL_NAME_ENABLED << "' in table '"
1368  << tableName_ << ".'" << __E__;
1369  ss << "\n\nTable '" << tableName_ << "' Columns: " << __E__;
1370  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1371  ss << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName()
1372  << __E__;
1373 
1374  __SS_ONLY_THROW__;
1375 
1376 } // end initColStatus()
1377 
1378 //==============================================================================
1381 unsigned int TableView::initColPriority(void)
1382 {
1383  if(colPriority_ != INVALID)
1384  return colPriority_;
1385 
1386  // if doesn't exist throw error! each view must have a UID column
1387  colPriority_ =
1388  findCol("*" + TableViewColumnInfo::COL_NAME_PRIORITY); // wild card search
1389  if(colPriority_ == INVALID)
1390  {
1391  __SS__ << "\tMissing column named '" << TableViewColumnInfo::COL_NAME_PRIORITY
1392  << "' in table '" << tableName_ << ".'" << __E__;
1393  ss << "\n\nTable '" << tableName_ << "' Columns: " << __E__;
1394  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1395  ss << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName()
1396  << __E__;
1397 
1398  __SS_THROW__;
1399  }
1400  return colPriority_;
1401 }
1402 
1403 //==============================================================================
1407 unsigned int TableView::getColStatus(void) const
1408 {
1409  if(colStatus_ != INVALID)
1410  return colStatus_;
1411 
1412  __SS__ << "\tMissing column named '" << TableViewColumnInfo::COL_NAME_STATUS
1413  << "' or '" << TableViewColumnInfo::COL_NAME_ENABLED << "' in table '"
1414  << tableName_ << ".'"
1415  << " (The Status column is identified when the TableView is initialized)"
1416  << __E__;
1417 
1418  ss << "\n\nTable '" << tableName_ << "' Columns: " << __E__;
1419  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1420  ss << "\t" << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName()
1421  << __E__;
1422 
1423  ss << __E__;
1424 
1425  ss << StringMacros::stackTrace() << __E__;
1426 
1427  __COUTT__ << ss.str();
1428  __SS_ONLY_THROW__;
1429 } // end getColStatus()
1430 
1431 //==============================================================================
1438 unsigned int TableView::getColPriority(void) const
1439 {
1440  if(colPriority_ != INVALID)
1441  return colPriority_;
1442 
1443  __SS__ << "Priority column was not found... \nColumn Types: " << __E__;
1444 
1445  ss << "Missing " << TableViewColumnInfo::COL_NAME_PRIORITY
1446  << " Column in table named '" << tableName_
1447  << ".' (The Priority column is identified when the TableView is initialized)"
1448  << __E__; // this is the const call, so can not identify the column and
1449  // set colPriority_ here
1450 
1451  ss << "\n\nTable '" << tableName_ << "' Columns: " << __E__;
1452  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1453  ss << "\t" << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName()
1454  << __E__;
1455  ss << __E__;
1456 
1457  ss << StringMacros::stackTrace() << __E__;
1458 
1459  __SS_ONLY_THROW__; // keep it quiet
1460 } // end getColPriority()
1461 
1462 //==============================================================================
1465 void TableView::addRowToGroup(const unsigned int& row,
1466  const unsigned int& col,
1467  const std::string& groupID) //,
1468 // const std::string &colDefault)
1469 {
1470  if(isEntryInGroupCol(row, col, groupID))
1471  {
1472  __SS__ << "GroupID (" << groupID << ") added to row (" << row
1473  << " is already present!" << __E__;
1474  __SS_THROW__;
1475  }
1476 
1477  // not in group, so
1478  // if no groups
1479  // set groupid
1480  // if other groups
1481  // prepend groupId |
1482  if(getDataView()[row][col] == "" ||
1483  getDataView()[row][col] == getDefaultRowValues()[col]) // colDefault)
1484  setValue(groupID, row, col);
1485  else
1486  setValue(groupID + " | " + getDataView()[row][col], row, col);
1487 
1488  //__COUT__ << getDataView()[row][col] << __E__;
1489 } // end addRowToGroup()
1490 
1491 //==============================================================================
1497 std::vector<unsigned int /*group row*/> TableView::getGroupRows(
1498  const unsigned int groupIdCol,
1499  const std::string& groupID,
1500  bool onlyStatusTrue /*=false*/,
1501  bool orderedByPriority /*=false*/) const
1502 {
1503  std::vector<unsigned int /*group row*/> retVector;
1504  std::vector<std::vector<unsigned int /*group row*/>> groupRowVectors =
1505  getGroupRowsInVectors(groupIdCol, groupID, onlyStatusTrue, orderedByPriority);
1506 
1507  for(const auto& groupRowVector : groupRowVectors)
1508  for(const auto& groupRow : groupRowVector)
1509  retVector.push_back(groupRow);
1510 
1511  return retVector;
1512 } // end getGroupRows()
1513 
1514 //==============================================================================
1520 std::vector<std::vector<unsigned int /*group row*/>> TableView::getGroupRowsByPriority(
1521  const unsigned int groupIdCol,
1522  const std::string& groupID,
1523  bool onlyStatusTrue /*=false*/) const
1524 {
1525  return getGroupRowsInVectors(
1526  groupIdCol, groupID, onlyStatusTrue, true /*orderedByPriority*/);
1527 } // end getGroupRowsByPriority()
1528 
1529 //==============================================================================
1537 std::vector<std::vector<unsigned int /*group row*/>> TableView::getGroupRowsInVectors(
1538  const unsigned int groupIdCol,
1539  const std::string& groupID,
1540  bool onlyStatusTrue,
1541  bool orderedByPriority) const
1542 {
1543  std::map<uint64_t /*priority*/, std::vector<unsigned int /*child row*/>>
1544  mapByPriority;
1545  std::vector<std::vector<unsigned int /*group row*/>> retVector;
1546  uint64_t tmpPriority;
1547  bool tmpStatus;
1548 
1549  if(!(orderedByPriority &&
1550  colPriority_ != INVALID)) // if no priority column, all at same priorty [0]
1551  retVector.push_back(std::vector<unsigned int /*group row*/>());
1552 
1553  __COUTS__(2) << "getGroupRowsInVectors: " << groupID << " at col " << groupIdCol
1554  << __E__;
1555  for(unsigned int r = 0; r < getNumberOfRows(); ++r)
1556  if(groupID == "" || groupID == "*" || groupIdCol == INVALID ||
1557  isEntryInGroupCol(r, groupIdCol, groupID))
1558  {
1559  if(groupIdCol != INVALID)
1560  __COUTS__(2) << "Row " << r << " '" << getDataView()[r][groupIdCol]
1561  << "' is in group " << groupID << __E__;
1562  // check status if needed
1563  if(onlyStatusTrue && colStatus_ != INVALID)
1564  {
1565  getValue(tmpStatus, r, colStatus_);
1566 
1567  if(!tmpStatus)
1568  continue; // skip those with status false
1569  }
1570 
1571  if(orderedByPriority && colPriority_ != INVALID)
1572  {
1573  getValue(tmpPriority, r, colPriority_);
1574  // do not accept DEFAULT value of 0.. convert to 100
1575  mapByPriority[tmpPriority ? tmpPriority : 100].push_back(r);
1576  }
1577  else // assume equal priority
1578  retVector[0].push_back(r);
1579  }
1580  else // already true that... if(groupIdCol != INVALID)
1581  __COUTS__(2) << "Row " << r << " '" << getDataView()[r][groupIdCol]
1582  << "' is NOT in group " << groupID << __E__;
1583 
1584  if(orderedByPriority && colPriority_ != INVALID)
1585  {
1586  // at this point have priority map (which automatically sorts by priority)
1587  // now build return vector
1588  for(const auto& priorityChildRowVector : mapByPriority)
1589  {
1590  retVector.push_back(std::vector<unsigned int /*group row*/>());
1591  for(const auto& priorityChildRow : priorityChildRowVector.second)
1592  retVector[retVector.size() - 1].push_back(priorityChildRow);
1593  }
1594 
1595  __COUT__ << "Returning priority children list." << __E__;
1596  }
1597  // else equal priority vector already constructed
1598 
1599  return retVector;
1600 } // end getGroupRowsInVectors()
1601 
1602 //==============================================================================
1607 bool TableView::removeRowFromGroup(const unsigned int& row,
1608  const unsigned int& col,
1609  const std::string& groupNeedle,
1610  bool deleteRowIfNoGroupLeft)
1611 {
1612  __COUT__ << "removeRowFromGroup groupNeedle " << groupNeedle << __E__;
1613  std::set<std::string> groupIDList;
1614  if(!isEntryInGroupCol(row, col, groupNeedle, &groupIDList))
1615  {
1616  __SS__
1617  << "GroupID (" << groupNeedle << ") removed from row (" << row
1618  << ") was already removed! Is there a strange GroupID wildcard match issue? {"
1619  << StringMacros::setToString(groupIDList) << "}" << __E__;
1620  print();
1621  __SS_THROW__;
1622  }
1623 
1624  // is in group, so
1625  // create new string based on set of groupids
1626  // but skip groupNeedle
1627 
1628  std::string newValue = "";
1629  unsigned int cnt = 0;
1630  for(const auto& groupID : groupIDList)
1631  {
1632  __COUTT__ << groupID << " " << groupNeedle << " " << newValue << __E__;
1633  if(groupID == groupNeedle)
1634  continue; // skip group to be removed
1635 
1636  if(cnt)
1637  newValue += " | ";
1638  newValue += groupID;
1639  }
1640 
1641  bool wasDeleted = false;
1642  if(deleteRowIfNoGroupLeft && newValue == "")
1643  {
1644  __COUTT__ << "Delete row since it no longer part of any group." << __E__;
1645  deleteRow(row);
1646  wasDeleted = true;
1647  }
1648  else
1649  {
1650  setValue(newValue, row, col);
1651  __COUTT__ << getDataView()[row][col] << __E__;
1652  }
1653 
1654  return wasDeleted;
1655 } // end removeRowFromGroup()
1656 
1657 //==============================================================================
1663 bool TableView::isEntryInGroup(const unsigned int& r,
1664  const std::string& childLinkIndex,
1665  const std::string& groupNeedle) const
1666 {
1667  unsigned int c = getLinkGroupIDColumn(childLinkIndex); // column in question
1668 
1669  return isEntryInGroupCol(r, c, groupNeedle);
1670 } // end isEntryInGroup()
1671 
1672 //==============================================================================
1681 bool TableView::isEntryInGroupCol(const unsigned int& r,
1682  const unsigned int& c,
1683  const std::string& groupNeedle,
1684  std::set<std::string>* groupIDList) const
1685 {
1686  if(r >= getNumberOfRows() || c >= getNumberOfColumns())
1687  {
1688  __SS__ << "Invalid row/col requested!" << __E__;
1689  ss << StringMacros::stackTrace() << __E__;
1690  __SS_THROW__;
1691  }
1692 
1693  unsigned int i = 0;
1694  unsigned int j = 0;
1695  bool found = false;
1696 
1697  __COUTS__(2) << "groupNeedle " << groupNeedle << __E__;
1698 
1699  // go through the full groupString extracting groups and comparing to groupNeedle
1700  for(; j < theDataView_[r][c].size(); ++j)
1701  if((theDataView_[r][c][j] == ' ' || // ignore leading white space or |
1702  theDataView_[r][c][j] == '|') &&
1703  i == j)
1704  ++i;
1705  else if((theDataView_[r][c][j] ==
1706  ' ' || // trailing white space or | indicates group
1707  theDataView_[r][c][j] == '|') &&
1708  i != j) // assume end of group name
1709  {
1710  if(groupIDList)
1711  groupIDList->emplace(theDataView_[r][c].substr(i, j - i));
1712 
1713  __COUTS__(2) << "Group found to compare: "
1714  << theDataView_[r][c].substr(i, j - i) << __E__;
1715  if(groupIDList ? groupNeedle == theDataView_[r][c].substr(i, j - i)
1716  : StringMacros::wildCardMatch(
1717  theDataView_[r][c].substr(i, j - i), groupNeedle))
1718  {
1719  __COUTS__(2) << "'" << theDataView_[r][c].substr(i, j - i)
1720  << "' is in group '" << groupNeedle << "'!" << __E__;
1721  if(!groupIDList) // dont return if caller is trying to get group list
1722  return true;
1723  found = true;
1724  }
1725  // if no match, setup i and j for next find
1726  i = j + 1;
1727  }
1728 
1729  if(i != j) // last group check (for case when no ' ' or '|')
1730  {
1731  if(groupIDList)
1732  groupIDList->emplace(theDataView_[r][c].substr(i, j - i));
1733 
1734  __COUTS__(2) << "Group found to compare: " << theDataView_[r][c].substr(i, j - i)
1735  << __E__;
1736  if(groupIDList ? groupNeedle == theDataView_[r][c].substr(i, j - i)
1737  : StringMacros::wildCardMatch(theDataView_[r][c].substr(i, j - i),
1738  groupNeedle))
1739  {
1740  __COUTS__(2) << "'" << theDataView_[r][c].substr(i, j - i)
1741  << "' is in group '" << groupNeedle << "'!" << __E__;
1742  return true;
1743  }
1744  }
1745 
1746  return found;
1747 } // end isEntryInGroupCol()
1748 
1749 //==============================================================================
1757 std::set<std::string> TableView::getSetOfGroupIDs(const std::string& childLinkIndex,
1758  unsigned int r) const
1759 {
1760  return getSetOfGroupIDs(getLinkGroupIDColumn(childLinkIndex), r);
1761 }
1762 std::set<std::string> TableView::getSetOfGroupIDs(const unsigned int& c,
1763  unsigned int r) const
1764 {
1765  //__COUT__ << "GroupID col=" << (int)c << __E__;
1766 
1767  std::set<std::string> retSet;
1768 
1769  // unsigned int i = 0;
1770  // unsigned int j = 0;
1771 
1772  if(r != (unsigned int)-1)
1773  {
1774  if(r >= getNumberOfRows())
1775  {
1776  __SS__ << "Invalid row requested!" << __E__;
1777  __SS_THROW__;
1778  }
1779 
1780  StringMacros::getSetFromString(theDataView_[r][c], retSet);
1781  // //go through the full groupString extracting groups
1782  // //add each found groupId to set
1783  // for(;j<theDataView_[r][c].size();++j)
1784  // if((theDataView_[r][c][j] == ' ' || //ignore leading white space or |
1785  // theDataView_[r][c][j] == '|')
1786  // && i == j)
1787  // ++i;
1788  // else if((theDataView_[r][c][j] == ' ' || //trailing white space or |
1789  // indicates group theDataView_[r][c][j] == '|')
1790  // && i != j) // assume end of group name
1791  // {
1792  // //__COUT__ << "Group found: " <<
1793  // // theDataView_[r][c].substr(i,j-i) << __E__;
1794  //
1795  //
1796  // retSet.emplace(theDataView_[r][c].substr(i,j-i));
1797  //
1798  // //setup i and j for next find
1799  // i = j+1;
1800  // }
1801  //
1802  // if(i != j) //last group check (for case when no ' ' or '|')
1803  // retSet.emplace(theDataView_[r][c].substr(i,j-i));
1804  }
1805  else
1806  {
1807  // do all rows
1808  for(r = 0; r < getNumberOfRows(); ++r)
1809  {
1810  StringMacros::getSetFromString(theDataView_[r][c], retSet);
1811 
1812  // i=0;
1813  // j=0;
1814  //
1815  // //__COUT__ << (int)r << ": " << theDataView_[r][c] << __E__;
1816  //
1817  // //go through the full groupString extracting groups
1818  // //add each found groupId to set
1819  // for(;j<theDataView_[r][c].size();++j)
1820  // {
1821  // //__COUT__ << "i:" << i << " j:" << j << __E__;
1822  //
1823  // if((theDataView_[r][c][j] == ' ' || //ignore leading white
1824  // space or | theDataView_[r][c][j] == '|')
1825  // && i == j)
1826  // ++i;
1827  // else if((theDataView_[r][c][j] == ' ' || //trailing white
1828  // space or | indicates group theDataView_[r][c][j]
1829  // ==
1830  // '|')
1831  // && i != j) // assume end of group name
1832  // {
1833  // //__COUT__ << "Group found: " <<
1834  // // theDataView_[r][c].substr(i,j-i) << __E__;
1835  //
1836  // retSet.emplace(theDataView_[r][c].substr(i,j-i));
1837  //
1838  // //setup i and j for next find
1839  // i = j+1;
1840  // }
1841  // }
1842  //
1843  // if(i != j) //last group (for case when no ' ' or '|')
1844  // {
1845  // //__COUT__ << "Group found: " <<
1846  // // theDataView_[r][c].substr(i,j-i) << __E__;
1847  // retSet.emplace(theDataView_[r][c].substr(i,j-i));
1848  // }
1849  }
1850  }
1851 
1852  return retSet;
1853 }
1854 
1855 //==============================================================================
1858 unsigned int TableView::getLinkGroupIDColumn(const std::string& childLinkIndex) const
1859 {
1860  if(!childLinkIndex.size())
1861  {
1862  __SS__ << "Empty childLinkIndex string parameter!" << __E__;
1863  ss << StringMacros::stackTrace() << __E__;
1864  __SS_THROW__;
1865  }
1866 
1867  const char* needleChildLinkIndex = &childLinkIndex[0];
1868 
1869  // allow space syntax to target a childLinkIndex from a different parentLinkIndex
1870  // e.g. "parentLinkIndex childLinkIndex"
1871  size_t spacePos = childLinkIndex.find(' ');
1872  if(spacePos != std::string::npos &&
1873  spacePos + 1 < childLinkIndex.size()) // make sure there are more characters
1874  {
1875  // found space syntax for targeting childLinkIndex
1876  needleChildLinkIndex = &childLinkIndex[spacePos + 1];
1877  }
1878 
1879  std::map<std::string, unsigned int>::const_iterator it =
1880  colLinkGroupIDs_.find(needleChildLinkIndex);
1881  if(it != // if already known, return it
1882  colLinkGroupIDs_.end())
1883  return it->second;
1884 
1885  // otherwise search (perhaps init() was not called)
1886  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1887  {
1888  // only check columns with link index associated...
1889  if(columnsInfo_[col].isChildLink() || columnsInfo_[col].isChildLinkUID() ||
1890  columnsInfo_[col].isChildLinkGroupID() || columnsInfo_[col].isGroupID())
1891  {
1892  if(needleChildLinkIndex == columnsInfo_[col].getChildLinkIndex())
1893  return col;
1894  }
1895  }
1896 
1897  __SS__
1898  << "Error! Incompatible table for this group link! Table '" << tableName_
1899  << "' is missing a GroupID column with data type '"
1900  << TableViewColumnInfo::TYPE_START_GROUP_ID << "-" << needleChildLinkIndex
1901  << "'.\n\n"
1902  << "Note: you can separate the child GroupID column data type from "
1903  << "the parent GroupLink column data type; this is accomplished by using a space "
1904  << "character at the parent level - the string after the space will be treated "
1905  "as the "
1906  << "child GroupID column data type." << __E__;
1907  ss << "Existing Column GroupIDs: " << __E__;
1908  for(auto& groupIdColPair : colLinkGroupIDs_)
1909  ss << "\t" << groupIdColPair.first << " : col-" << groupIdColPair.second << __E__;
1910 
1911  ss << "Existing Column Types: " << __E__;
1912  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1913  ss << "\t" << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName()
1914  << __E__;
1915 
1916  ss << StringMacros::stackTrace() << __E__;
1917 
1918  __SS_THROW__;
1919 } // end getLinkGroupIDColumn()
1920 
1921 //==============================================================================
1922 unsigned int TableView::findRow(unsigned int col,
1923  const std::string& value,
1924  unsigned int offsetRow,
1925  bool doNotThrow /*= false*/) const
1926 {
1927  for(unsigned int row = offsetRow; row < theDataView_.size(); ++row)
1928  {
1929  if(theDataView_[row][col] == value)
1930  return row;
1931  }
1932  if(doNotThrow)
1933  return TableView::INVALID;
1934 
1935  __SS__ << "\tIn view: " << tableName_ << ", Can't find value=" << value
1936  << " in column named " << columnsInfo_[col].getName()
1937  << " with type=" << columnsInfo_[col].getType() << __E__ << __E__
1938  << StringMacros::stackTrace() << __E__;
1939 
1940  // Note: findRow gets purposely called by configuration GUI a lot looking for
1941  // exceptions so may not want to print out
1942  //__COUT__ << "\n" << ss.str();
1943  __SS_ONLY_THROW__;
1944 } // end findRow()
1945 
1946 //==============================================================================
1947 unsigned int TableView::findRowInGroup(unsigned int col,
1948  const std::string& value,
1949  const std::string& groupId,
1950  const std::string& childLinkIndex,
1951  unsigned int offsetRow) const
1952 {
1953  unsigned int groupIdCol = getLinkGroupIDColumn(childLinkIndex);
1954  for(unsigned int row = offsetRow; row < theDataView_.size(); ++row)
1955  {
1956  if(theDataView_[row][col] == value && isEntryInGroupCol(row, groupIdCol, groupId))
1957  return row;
1958  }
1959 
1960  __SS__ << "\tIn view: " << tableName_ << ", Can't find in group the value=" << value
1961  << " in column named '" << columnsInfo_[col].getName()
1962  << "' with type=" << columnsInfo_[col].getType() << " and GroupID: '"
1963  << groupId << "' in column '" << groupIdCol
1964  << "' with GroupID child link index '" << childLinkIndex << "'" << __E__;
1965  // Note: findRowInGroup gets purposely called by configuration GUI a lot looking for
1966  // exceptions so may not want to print out
1967  __SS_ONLY_THROW__;
1968 } // end findRowInGroup()
1969 
1970 //==============================================================================
1973 unsigned int TableView::findCol(const std::string& wildCardName) const
1974 {
1975  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1976  if(StringMacros::wildCardMatch(wildCardName /*needle*/,
1977  columnsInfo_[col].getName() /*haystack*/))
1978  return col;
1979 
1980  __SS__ << "\tIn view: " << tableName_ << ", Can't find column named '" << wildCardName
1981  << "'" << __E__;
1982  ss << "Existing columns:\n";
1983  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
1984  ss << "\t" << columnsInfo_[col].getName() << "\n";
1985 
1986  ss << StringMacros::stackTrace() << __E__;
1987 
1988  // Note: findCol gets purposely called by configuration GUI a lot looking for
1989  // exceptions so may not want to print out
1990  __SS_ONLY_THROW__;
1991 } // end findCol()
1992 
1993 //==============================================================================
1996 unsigned int TableView::findColByType(const std::string& type,
1997  unsigned int startingCol) const
1998 {
1999  for(unsigned int col = startingCol; col < columnsInfo_.size(); ++col)
2000  {
2001  __COUTS__(40) << columnsInfo_[col].getType() << __E__;
2002  if(columnsInfo_[col].getType() == type)
2003  return col;
2004  }
2005 
2006  return INVALID;
2007 } // end findColByType()
2008 
2011 //==============================================================================
2013 unsigned int TableView::getDataColumnSize(void) const
2014 {
2015  // if no data, give benefit of the doubt that phantom data has mockup column size
2016  if(!getNumberOfRows())
2017  return getNumberOfColumns();
2018  return theDataView_[0].size(); // number of columns in first row of data
2019 }
2020 
2021 //==============================================================================
2022 std::set<std::string> TableView::getColumnNames(void) const
2023 {
2024  std::set<std::string> retSet;
2025  for(auto& colInfo : columnsInfo_)
2026  retSet.emplace(colInfo.getName());
2027  return retSet;
2028 } // end getColumnNames()
2029 
2030 //==============================================================================
2031 std::map<std::string, unsigned int /*col*/> TableView::getColumnNamesMap(void) const
2032 {
2033  std::map<std::string, unsigned int /*col*/> retMap;
2034  unsigned int c = 0;
2035  for(auto& colInfo : columnsInfo_)
2036  retMap.emplace(std::make_pair(colInfo.getName(), c++));
2037  return retMap;
2038 } // end getColumnNamesMap()
2039 
2040 //==============================================================================
2041 std::set<std::string> TableView::getColumnStorageNames(void) const
2042 {
2043  std::set<std::string> retSet;
2044  for(auto& colInfo : columnsInfo_)
2045  retSet.emplace(colInfo.getStorageName());
2046  return retSet;
2047 }
2048 
2049 //==============================================================================
2050 const std::vector<std::string>& TableView::initRowDefaults(void)
2051 {
2052  std::vector<std::string>& retVec = rowDefaultValues_;
2053  retVec.clear();
2054 
2055  // fill each col of new row with default values
2056  for(unsigned int col = 0; col < getNumberOfColumns(); ++col)
2057  {
2058  // if this is a fixed choice Link, and NO_LINK is not in list,
2059  // take first in list to avoid creating illegal rows.
2060  // NOTE: this is not a problem for standard fixed choice fields
2061  // because the default value is always required.
2062 
2063  if(columnsInfo_[col].isChildLink())
2064  {
2065  const std::vector<std::string>& theDataChoices =
2066  columnsInfo_[col].getDataChoices();
2067 
2068  // check if arbitrary values allowed
2069  if(!theDataChoices.size() || // if so, use default
2070  theDataChoices[0] == "arbitraryBool=1")
2071  retVec.push_back(columnsInfo_[col].getDefaultValue());
2072  else
2073  {
2074  bool skipOne =
2075  (theDataChoices.size() && theDataChoices[0] == "arbitraryBool=0");
2076  bool hasSkipped;
2077 
2078  // look for default value in list
2079 
2080  bool foundDefault = false;
2081  hasSkipped = false;
2082  for(const auto& choice : theDataChoices)
2083  if(skipOne && !hasSkipped)
2084  {
2085  hasSkipped = true;
2086  continue;
2087  }
2088  else if(choice == columnsInfo_[col].getDefaultValue())
2089  {
2090  foundDefault = true;
2091  break;
2092  }
2093 
2094  // use first choice if possible
2095  if(!foundDefault && theDataChoices.size() > (skipOne ? 1 : 0))
2096  retVec.push_back(theDataChoices[(skipOne ? 1 : 0)]);
2097  else // else stick with default
2098  retVec.push_back(columnsInfo_[col].getDefaultValue());
2099  }
2100  }
2101  else
2102  retVec.push_back(columnsInfo_[col].getDefaultValue());
2103  }
2104 
2105  //__COUT__ << StringMacros::stackTrace() << __E__;
2106  //__COUTV__(StringMacros::vectorToString(rowDefaultValues_));
2107  return rowDefaultValues_;
2108 } // end getDefaultRowValues()
2109 
2110 //==============================================================================
2111 const TableViewColumnInfo& TableView::getColumnInfo(unsigned int column) const
2112 {
2113  if(column >= columnsInfo_.size())
2114  {
2115  __SS__ << "\nCan't find column " << column
2116  << "\n\n\n\nThe column info is likely missing due to incomplete "
2117  "Configuration View filling.\n\n"
2118  << __E__;
2119  ss << StringMacros::stackTrace() << __E__;
2120  __SS_THROW__;
2121  }
2122  return columnsInfo_[column];
2123 } // end getColumnInfo()
2124 
2127 //==============================================================================
2128 void TableView::setURIEncodedComment(const std::string& uriComment)
2129 {
2130  comment_ = StringMacros::decodeURIComponent(uriComment);
2131 }
2132 
2133 //==============================================================================
2134 void TableView::setAuthor(const std::string& author) { author_ = author; }
2135 
2136 //==============================================================================
2137 void TableView::setCreationTime(time_t t) { creationTime_ = t; }
2138 
2139 //==============================================================================
2140 void TableView::setLastAccessTime(time_t t) { lastAccessTime_ = t; }
2141 
2142 //==============================================================================
2143 void TableView::setLooseColumnMatching(bool setValue)
2144 {
2145  fillWithLooseColumnMatching_ = setValue;
2146 }
2147 
2148 //==============================================================================
2149 void TableView::doGetSourceRawData(bool setValue) { getSourceRawData_ = setValue; }
2150 
2151 //==============================================================================
2152 void TableView::reset(void)
2153 {
2154  version_ = -1;
2155  comment_ = "";
2156  author_ = "";
2157  columnsInfo_.clear();
2158  theDataView_.clear();
2159 } // end reset()
2160 
2161 //==============================================================================
2162 void TableView::print(std::ostream& out /* = std::cout */) const
2163 {
2164  out << "============================================================================="
2165  "="
2166  << __E__;
2167  out << "Print: " << tableName_ << " Version: " << version_ << " Comment: " << comment_
2168  << " Author: " << author_ << " Creation Time: " << ctime(&creationTime_) << __E__;
2169  out << "\t\tNumber of Cols " << getNumberOfColumns() << __E__;
2170  out << "\t\tNumber of Rows " << getNumberOfRows() << __E__;
2171 
2172  out << "Columns:\t";
2173  for(int i = 0; i < (int)columnsInfo_.size(); ++i)
2174  out << i << ":" << columnsInfo_[i].getName() << ":"
2175  << columnsInfo_[i].getStorageName() << ":" << columnsInfo_[i].getType() << ":"
2176  << columnsInfo_[i].getDataType() << "\t ";
2177  out << __E__;
2178 
2179  out << "Rows:" << __E__;
2180  // int num;
2181  std::string val;
2182  for(int r = 0; r < (int)getNumberOfRows(); ++r)
2183  {
2184  out << (int)r << ":\t";
2185  for(int c = 0; c < (int)getNumberOfColumns(); ++c)
2186  {
2187  out << (int)c << ":";
2188 
2189  // if fixed choice type, print index in choice
2190  if(columnsInfo_[c].getType() == TableViewColumnInfo::TYPE_FIXED_CHOICE_DATA)
2191  {
2192  int choiceIndex = -1;
2193  std::vector<std::string> choices = columnsInfo_[c].getDataChoices();
2194  val = StringMacros::convertEnvironmentVariables(theDataView_[r][c]);
2195 
2196  if(val == columnsInfo_[c].getDefaultValue())
2197  choiceIndex = 0;
2198  else
2199  {
2200  for(int i = 0; i < (int)choices.size(); ++i)
2201  if(val == choices[i])
2202  choiceIndex = i + 1;
2203  }
2204 
2205  out << "ChoiceIndex=" << choiceIndex << ":";
2206  }
2207 
2208  out << theDataView_[r][c];
2209  // stopped using below, because it is called sometimes during debugging when
2210  // numbers are set to environment variables:
2211  // if(columnsInfo_[c].getDataType() == "NUMBER")
2212  // {
2213  // getValue(num,r,c,false);
2214  // out << num;
2215  // }
2216  // else
2217  // {
2218  // getValue(val,r,c,false);
2219  // out << val;
2220  // }
2221  out << "\t\t";
2222  }
2223  out << __E__;
2224  }
2225 } // end print()
2226 
2227 //==============================================================================
2228 void TableView::printJSON(std::ostream& out /* = std::cout */) const
2229 {
2230  { //handle special GROUP CACHE table
2231  std::string tmpCachePrepend = TableBase::GROUP_CACHE_PREPEND;
2232  tmpCachePrepend = TableBase::convertToCaps(tmpCachePrepend);
2233  std::string tmpJsonDocPrepend = TableBase::JSON_DOC_PREPEND;
2234  tmpJsonDocPrepend = TableBase::convertToCaps(tmpJsonDocPrepend);
2235  __COUTS__(32) << " '" << tableName_ << "' vs " << tmpCachePrepend << " or "
2236  << tmpJsonDocPrepend << __E__;
2237  //if special GROUP CACHE table, handle construction in a special way
2238  if(tableName_.substr(0, tmpCachePrepend.length()) == tmpCachePrepend ||
2239  tableName_.substr(0, tmpJsonDocPrepend.length()) == tmpJsonDocPrepend)
2240  {
2241  out << getCustomStorageData();
2242  return;
2243  } //end special GROUP CACHE table construction
2244  } //end handle special GROUP CACHE table
2245 
2246  out << "{\n";
2247  out << "\"NAME\" : \"" << tableName_ << "\",\n";
2248 
2249  // out << "\"VERSION\": \"" << version_ << "\",\n";
2250 
2251  out << "\"COMMENT\" : ";
2252 
2253  // output escaped comment
2254  std::string val;
2255  val = comment_;
2256  out << "\"";
2257  for(unsigned int i = 0; i < val.size(); ++i)
2258  {
2259  if(val[i] == '\n')
2260  out << "\\n";
2261  else if(val[i] == '\t')
2262  out << "\\t";
2263  else if(val[i] == '\r')
2264  out << "\\r";
2265  else
2266  {
2267  // escaped characters need a
2268  if(val[i] == '"' || val[i] == '\\')
2269  out << '\\';
2270  out << val[i];
2271  }
2272  }
2273  out << "\",\n";
2274 
2275  out << "\"AUTHOR\" : \"" << author_ << "\",\n";
2276  out << "\"CREATION_TIME\" : " << creationTime_ << ",\n";
2277 
2278  // USELESS... out << "\"NUM_OF_COLS\" : " << getNumberOfColumns() << ",\n";
2279  // USELESS... out << "\"NUM_OF_ROWS\" : " << getNumberOfRows() << ",\n";
2280 
2281  out << "\"COL_TYPES\" : {\n";
2282  for(int c = 0; c < (int)getNumberOfColumns(); ++c)
2283  {
2284  out << "\t\t\"" << columnsInfo_[c].getStorageName() << "\" : ";
2285  out << "\"" << columnsInfo_[c].getDataType() << "\"";
2286  if(c + 1 < (int)getNumberOfColumns())
2287  out << ",";
2288  out << "\n";
2289  }
2290  out << "},\n"; // close COL_TYPES
2291 
2292  out << "\"DATA_SET\" : [\n";
2293  // int num;
2294  for(int r = 0; r < (int)getNumberOfRows(); ++r)
2295  {
2296  out << "\t{\n";
2297  for(int c = 0; c < (int)getNumberOfColumns(); ++c)
2298  {
2299  out << "\t\t\"" << columnsInfo_[c].getStorageName() << "\" : ";
2300 
2301  out << "\"" << getEscapedValueAsString(r, c, false)
2302  << "\""; // do not convert env variables
2303 
2304  if(c + 1 < (int)getNumberOfColumns())
2305  out << ",";
2306  out << "\n";
2307  }
2308  out << "\t}";
2309  if(r + 1 < (int)getNumberOfRows())
2310  out << ",";
2311  out << "\n";
2312  }
2313  out << "]\n"; // close DATA_SET
2314 
2315  out << "}";
2316 } // end printJSON()
2317 
2318 //==============================================================================
2319 void TableView::printCSV(std::ostream& out /* = std::cout */,
2320  const std::string& valueDelimeter /* = "," */,
2321  const std::string& recordDelimeter /* = "\n" */,
2322  bool includeColumnNames /* = false */) const
2323 {
2324  { //handle special GROUP CACHE table
2325  std::string tmpCachePrepend = TableBase::GROUP_CACHE_PREPEND;
2326  tmpCachePrepend = TableBase::convertToCaps(tmpCachePrepend);
2327  std::string tmpJsonDocPrepend = TableBase::JSON_DOC_PREPEND;
2328  tmpJsonDocPrepend = TableBase::convertToCaps(tmpJsonDocPrepend);
2329  __COUTS__(32) << " '" << tableName_ << "' vs " << tmpCachePrepend << " or "
2330  << tmpJsonDocPrepend << __E__;
2331  //if special GROUP CACHE table, handle construction in a special way
2332  if(tableName_.substr(0, tmpCachePrepend.length()) == tmpCachePrepend ||
2333  tableName_.substr(0, tmpJsonDocPrepend.length()) == tmpJsonDocPrepend)
2334  {
2335  __SS__ << "Cannot convert custom storage data to CSV!" << __E__;
2336  __SS_THROW__;
2337  // out << getCustomStorageData();
2338  // return;
2339  } //end special GROUP CACHE table construction
2340  } //end handle special GROUP CACHE table
2341 
2342  for(int c = 0; includeColumnNames && c < (int)getNumberOfColumns(); ++c)
2343  {
2344  if(c)
2345  out << valueDelimeter;
2346  out << "\"" << columnsInfo_[c].getStorageName() << "\"";
2347  }
2348  if(includeColumnNames)
2349  out << recordDelimeter;
2350 
2351  for(int r = 0; r < (int)getNumberOfRows(); ++r)
2352  {
2353  for(int c = 0; c < (int)getNumberOfColumns(); ++c)
2354  {
2355  if(c)
2356  out << valueDelimeter;
2357  out << "\""
2358  << getEscapedValueAsString(r, c, false, true /* quotesToDoubleQuotes*/)
2359  << "\""; // do not convert env variables, convert " to "" for excel style
2360  }
2361  out << recordDelimeter;
2362  }
2363 
2364 } // end printCSV()
2365 
2366 //==============================================================================
2374 int TableView::fillFromJSON(const std::string& json)
2375 {
2376  { //handle special GROUP CACHE table
2377  std::string tmpCachePrepend = TableBase::GROUP_CACHE_PREPEND;
2378  tmpCachePrepend = TableBase::convertToCaps(tmpCachePrepend);
2379  std::string tmpJsonDocPrepend = TableBase::JSON_DOC_PREPEND;
2380  tmpJsonDocPrepend = TableBase::convertToCaps(tmpJsonDocPrepend);
2381 
2382  //if special JSON DOC table, handle construction in a special way
2383  if(tableName_.substr(0, tmpJsonDocPrepend.length()) == tmpJsonDocPrepend ||
2384  tableName_.substr(0, tmpCachePrepend.length()) == tmpCachePrepend)
2385  {
2386  __COUTS__(3) << "Special JSON doc: " << json << __E__;
2387  setCustomStorageData(json);
2388  return 0; //success
2389  } //end special JSON DOC table construction or special GROUP CACHE table construction
2390  } //end handle special GROUP CACHE table
2391 
2392  bool dbg = false; //tableName_ == "TABLE_GROUP_METADATA";
2393  bool rawData = getSourceRawData_;
2394  if(getSourceRawData_)
2395  { // only get source raw data once, then revert member variable
2396  __COUTV__(getSourceRawData_);
2397  getSourceRawData_ = false;
2398  sourceRawData_ = ""; // clear for this fill
2399  }
2400 
2401  std::map<std::string /*key*/, unsigned int /*entries/rows*/> keyEntryCountMap;
2402  std::vector<std::string> keys;
2403  keys.push_back("NAME");
2404  keys.push_back("COMMENT");
2405  keys.push_back("AUTHOR");
2406  keys.push_back("CREATION_TIME");
2407  // keys.push_back ("COL_TYPES");
2408  keys.push_back("DATA_SET");
2409  enum
2410  {
2411  CV_JSON_FILL_NAME,
2412  CV_JSON_FILL_COMMENT,
2413  CV_JSON_FILL_AUTHOR,
2414  CV_JSON_FILL_CREATION_TIME,
2415  // CV_JSON_FILL_COL_TYPES,
2416  CV_JSON_FILL_DATA_SET
2417  };
2418 
2419  if(dbg)
2420  {
2421  __COUTV__(tableName_);
2422  __COUTTV__(getNumberOfRows());
2423  __COUTV__(json);
2424  }
2425 
2426  sourceColumnMismatchCount_ = 0;
2427  sourceColumnMissingCount_ = 0;
2428  sourceColumnNames_.clear(); // reset
2429  unsigned int colFoundCount = 0;
2430  unsigned int i = 0;
2431  unsigned int row = -1;
2432  unsigned int colSpeedup = 0;
2433  unsigned int startString, startNumber = 0, endNumber = -1;
2434  unsigned int bracketCount = 0;
2435  unsigned int sqBracketCount = 0;
2436  bool inQuotes = 0;
2437  bool newString = 0;
2438  bool newValue = 0;
2439  // bool isDataArray = 0;
2440  bool keyIsMatch, keyIsComment;
2441  unsigned int keyIsMatchIndex, keyIsMatchStorageIndex, keyIsMatchCommentIndex;
2442  const std::string COMMENT_ALT_KEY = "COMMENT";
2443 
2444  std::string extractedString = "", currKey = "", currVal = "";
2445  unsigned int currDepth = 0;
2446 
2447  std::vector<std::string> jsonPath;
2448  std::vector<char> jsonPathType; // indicator of type in jsonPath: { [ K
2449  char lastPopType = '_'; // either: _ { [ K
2450  // _ indicates reset pop (this happens when a new {obj} starts)
2451  unsigned int matchedKey = -1;
2452  unsigned int lastCol = -1;
2453 
2454  // find all depth 1 matching keys
2455  for(; i < json.size(); ++i)
2456  {
2457  switch(json[i])
2458  {
2459  case '"':
2460  if(i - 1 < json.size() && // ignore if escaped
2461  json[i - 1] == '\\')
2462  break;
2463 
2464  inQuotes = !inQuotes; // toggle in quotes if not escaped
2465  if(inQuotes)
2466  startString = i;
2467  else
2468  {
2469  extractedString = StringMacros::restoreJSONStringEntities(
2470  json.substr(startString + 1, i - startString - 1));
2471  newString = 1; // have new string!
2472  }
2473  break;
2474  case ':':
2475  if(inQuotes)
2476  break; // skip if in quote
2477 
2478  // must be a json object level to have a key
2479  if(jsonPathType[jsonPathType.size() - 1] != '{' ||
2480  !newString) // and must have a string for key
2481  {
2482  __COUT__ << "Invalid ':' position" << __E__;
2483  return -1;
2484  }
2485 
2486  // valid, so take key
2487  jsonPathType.push_back('K');
2488  jsonPath.push_back(extractedString);
2489  startNumber = i;
2490  newString = 0; // clear flag
2491  endNumber = -1; // reset end number index
2492  break;
2493 
2494  // if(isKey ||
2495  // isDataArray)
2496  // {
2497  // std::cout << "Invalid ':' position" << __E__;
2498  // return -1;
2499  // }
2500  // isKey = 1; //new value is a key
2501  // newValue = 1;
2502  // startNumber = i;
2503  // break;
2504  case ',':
2505  if(inQuotes)
2506  break; // skip if in quote
2507  if(lastPopType == '{') // don't need value again of nested object
2508  {
2509  // check if the nested object was the value to a key, if so, pop key
2510  if(jsonPathType[jsonPathType.size() - 1] == 'K')
2511  {
2512  lastPopType = 'K';
2513  jsonPath.pop_back();
2514  jsonPathType.pop_back();
2515  }
2516  break; // skip , handling if {obj} just ended
2517  }
2518 
2519  if(newString)
2520  currVal = extractedString;
2521  else // number value
2522  {
2523  if(endNumber == (unsigned int)-1 || // take i as end number if needed
2524  endNumber <= startNumber)
2525  endNumber = i;
2526  // extract number value
2527  if(endNumber <= startNumber) // empty data, could be {}
2528  currVal = "";
2529  else
2530  currVal = json.substr(startNumber + 1, endNumber - startNumber - 1);
2531  }
2532 
2533  currDepth = bracketCount;
2534 
2535  if(jsonPathType[jsonPathType.size() - 1] == 'K') // this is the value to key
2536  {
2537  currKey = jsonPath[jsonPathType.size() - 1];
2538  newValue = 1; // new value to consider!
2539 
2540  // pop key
2541  lastPopType = 'K';
2542  jsonPath.pop_back();
2543  jsonPathType.pop_back();
2544  }
2545  else if(jsonPathType[jsonPathType.size() - 1] ==
2546  '[') // this is a value in array
2547  {
2548  // key is last key
2549  for(unsigned int k = jsonPathType.size() - 2; k < jsonPathType.size();
2550  --k)
2551  if(jsonPathType[k] == 'K')
2552  {
2553  currKey = jsonPath[k];
2554  break;
2555  }
2556  else if(k == 0)
2557  {
2558  __COUT__ << "Invalid array position" << __E__;
2559  return -1;
2560  }
2561 
2562  newValue = 1; // new value to consider!
2563  // isDataArray = 1;
2564  }
2565  else // { is an error
2566  {
2567  __COUT__ << "Invalid ',' position" << __E__;
2568  return -1;
2569  }
2570 
2571  startNumber = i;
2572  break;
2573 
2574  case '{':
2575  if(inQuotes)
2576  break; // skip if in quote
2577  lastPopType = '_'; // reset because of new object
2578  jsonPathType.push_back('{');
2579  jsonPath.push_back("{");
2580  ++bracketCount;
2581  break;
2582 
2583  // ++bracketCount;
2584  // isDataArray = 0;
2585  // isKey = 0;
2586  // endingObject = 0;
2587  // break;
2588  case '}':
2589  if(inQuotes)
2590  break; // skip if in quote
2591 
2592  if(lastPopType != '{' && // don't need value again of nested object
2593  jsonPathType[jsonPathType.size() - 1] == 'K') // this is the value to key
2594  {
2595  currDepth = bracketCount;
2596  currKey = jsonPath[jsonPathType.size() - 1];
2597  if(newString)
2598  currVal = extractedString;
2599  else // number value
2600  {
2601  if(endNumber == (unsigned int)-1 || // take i as end number if needed
2602  endNumber <= startNumber)
2603  endNumber = i;
2604  // extract val
2605  if(endNumber <= startNumber) // empty data, could be {}
2606  currVal = "";
2607  else
2608  currVal =
2609  json.substr(startNumber + 1, endNumber - startNumber - 1);
2610  }
2611  newValue = 1; // new value to consider!
2612  // pop key
2613  jsonPath.pop_back();
2614  jsonPathType.pop_back();
2615  }
2616  // pop {
2617  if(jsonPathType[jsonPathType.size() - 1] != '{')
2618  {
2619  __COUT__ << "Invalid '}' position" << __E__;
2620  return -1;
2621  }
2622  lastPopType = '{';
2623  jsonPath.pop_back();
2624  jsonPathType.pop_back();
2625  --bracketCount;
2626  break;
2627  case '[':
2628  if(inQuotes)
2629  break; // skip if in quote
2630  jsonPathType.push_back('[');
2631  jsonPath.push_back("[");
2632  ++sqBracketCount;
2633  startNumber = i;
2634  break;
2635  case ']':
2636  if(inQuotes)
2637  break; // skip if in quote
2638 
2639  // must be an array at this level (in order to close it)
2640  if(jsonPathType[jsonPathType.size() - 1] != '[')
2641  {
2642  __COUT__ << "Invalid ']' position" << __E__;
2643  return -1;
2644  }
2645 
2646  currDepth = bracketCount;
2647 
2648  // This is an array value
2649  if(newString)
2650  currVal = extractedString;
2651  else // number value
2652  {
2653  if(endNumber == (unsigned int)-1 || // take i as end number if needed
2654  endNumber <= startNumber)
2655  endNumber = i;
2656  // extract val
2657  if(endNumber <= startNumber) // empty data, could be {}
2658  currVal = "";
2659  else
2660  currVal = json.substr(startNumber + 1, endNumber - startNumber - 1);
2661  }
2662  // isDataArray = 1;
2663 
2664  // key is last key
2665  for(unsigned int k = jsonPathType.size() - 2; k < jsonPathType.size(); --k)
2666  if(jsonPathType[k] == 'K')
2667  {
2668  currKey = jsonPath[k];
2669  break;
2670  }
2671  else if(k == 0)
2672  {
2673  __COUT__ << "Invalid array position" << __E__;
2674  return -1;
2675  }
2676 
2677  // pop [
2678  if(jsonPathType[jsonPathType.size() - 1] != '[')
2679  {
2680  __COUT__ << "Invalid ']' position" << __E__;
2681  return -1;
2682  }
2683  lastPopType = '[';
2684  jsonPath.pop_back();
2685  jsonPathType.pop_back();
2686  --sqBracketCount;
2687  break;
2688  case ' ': // white space handling for numbers
2689  case '\t':
2690  case '\n':
2691  case '\r':
2692  if(inQuotes)
2693  break; // skip if in quote
2694  if(startNumber != (unsigned int)-1 && endNumber == (unsigned int)-1)
2695  endNumber = i;
2696  startNumber = i;
2697  break;
2698  default:;
2699  }
2700 
2701  // continue;
2702 
2703  // handle a new completed value
2704  if(newValue)
2705  {
2706  if(dbg) // for debugging
2707  {
2708  std::cout << i << ":\t" << json[i] << " - ";
2709 
2710  // if(isDataArray)
2711  // std::cout << "Array:: ";
2712  // if(newString)
2713  // std::cout << "New String:: ";
2714  // else
2715  // std::cout << "New Number:: ";
2716  //
2717 
2718  std::cout << "ExtKey=";
2719  for(unsigned int k = 0; k < jsonPath.size(); ++k)
2720  std::cout << jsonPath[k] << "/";
2721  std::cout << " - ";
2722  std::cout << lastPopType << " ";
2723  std::cout << bracketCount << " ";
2724  std::cout << sqBracketCount << " ";
2725  std::cout << inQuotes << " ";
2726  std::cout << newValue << "-";
2727  std::cout << currKey << "-{" << currDepth << "}:";
2728  std::cout << currVal << " ";
2729  std::cout << startNumber << "-";
2730  std::cout << endNumber << " ";
2731  std::cout << "\n";
2732  __COUTTV__(fillWithLooseColumnMatching_);
2733  __COUTTV__(getNumberOfRows());
2734  }
2735 
2736  // extract only what we care about
2737  // for TableView only care about matching depth 1
2738 
2739  // handle matching depth 1 keys
2740 
2741  matchedKey = -1; // init to unfound
2742  for(unsigned int k = 0; k < keys.size(); ++k)
2743  if((currDepth == 1 && keys[k] == currKey) ||
2744  (currDepth > 1 && keys[k] == jsonPath[1]))
2745  matchedKey = k;
2746 
2747  if(rawData)
2748  {
2749  // raw data handling fills raw data string with row/col values
2750 
2751  if(currDepth == 1)
2752  {
2753  if(matchedKey == CV_JSON_FILL_COMMENT)
2754  setComment(currVal);
2755  else if(matchedKey == CV_JSON_FILL_AUTHOR)
2756  setAuthor(currVal);
2757  else if(matchedKey == CV_JSON_FILL_CREATION_TIME)
2758  setCreationTime(strtol(currVal.c_str(), 0, 10));
2759  }
2760  else if(currDepth == 2)
2761  {
2762  // encode URI component so commas are surviving delimiter
2763  sourceRawData_ += StringMacros::encodeURIComponent(currKey) + "," +
2764  StringMacros::encodeURIComponent(currVal) + ",";
2765  sourceColumnNames_.emplace(currKey);
2766  }
2767  }
2768  else if(matchedKey != (unsigned int)-1)
2769  {
2770  if(dbg)
2771  __COUTT__ << "New Data for:: key[" << matchedKey << "]-"
2772  << keys[matchedKey] << "\n";
2773 
2774  switch(matchedKey)
2775  {
2776  case CV_JSON_FILL_NAME:
2777  // table name is now constant, set by parent TableBase
2778  if(currDepth == 1)
2779  {
2780  // setTableName(currVal);
2781  // check for consistency, and show warning
2782  if(currVal != getTableName() &&
2783  getTableName() !=
2784  "TABLE_GROUP_METADATA") // allow metadata table to be illegal, since it is created by ConfigurationManager.cc
2785  __COUT_WARN__ << "JSON-fill Table name mismatch: " << currVal
2786  << " vs " << getTableName() << __E__;
2787  }
2788  break;
2789  case CV_JSON_FILL_COMMENT:
2790  if(currDepth == 1)
2791  setComment(currVal);
2792  break;
2793  case CV_JSON_FILL_AUTHOR:
2794  if(currDepth == 1)
2795  setAuthor(currVal);
2796  break;
2797  case CV_JSON_FILL_CREATION_TIME:
2798  if(currDepth == 1)
2799  setCreationTime(strtol(currVal.c_str(), 0, 10));
2800  break;
2801  // case CV_JSON_FILL_COL_TYPES:
2802  //
2803  // break;
2804  case CV_JSON_FILL_DATA_SET:
2805  if(dbg)
2806  __COUTT__ << "CV_JSON_FILL_DATA_SET New Data for::" << matchedKey
2807  << "]-" << keys[matchedKey] << "/" << currDepth
2808  << ".../" << currKey << "\n";
2809 
2810  if(currDepth == 2) // second level depth
2811  {
2812  // if matches first column name.. then add new row
2813  // else add to current row
2814  unsigned int col, ccnt = 0;
2815  unsigned int noc = getNumberOfColumns();
2816  for(; ccnt < noc; ++ccnt)
2817  {
2818  // use colSpeedup to change the first column we search
2819  // for each iteration.. since we expect the data to
2820  // be arranged in column order
2821 
2822  if(fillWithLooseColumnMatching_)
2823  {
2824  // loose column matching makes no attempt to
2825  // match the column names
2826  // just assumes the data is in the correct order
2827 
2828  col = colSpeedup;
2829 
2830  // auto matched
2831  if(col <= lastCol) // add row (use lastCol in case new
2832  // column-0 was added
2833  row = addRow();
2834  lastCol = col;
2835  if(getNumberOfRows() == 1) // only for first row
2836  sourceColumnNames_.emplace(currKey);
2837 
2838  // add value to row and column
2839 
2840  if(row >= getNumberOfRows())
2841  {
2842  __SS__ << "Invalid row"
2843  << __E__; // should be impossible?
2844  std::cout << ss.str();
2845  __SS_THROW__;
2846  return -1;
2847  }
2848 
2849  theDataView_[row][col] =
2850  currVal; // THERE IS NO CHECK FOR WHAT IS READ FROM
2851  // THE DATABASE. IT SHOULD BE ALREADY
2852  // CONSISTENT
2853  break;
2854  }
2855  else
2856  {
2857  col = (ccnt + colSpeedup) % noc;
2858 
2859  // match key by ignoring '_'
2860  // also accept COMMENT == COMMENT_DESCRIPTION
2861  // (this is for backwards compatibility..)
2862  keyIsMatch = true;
2863  keyIsComment = true;
2864  for(keyIsMatchIndex = 0,
2865  keyIsMatchStorageIndex = 0,
2866  keyIsMatchCommentIndex = 0;
2867  keyIsMatchIndex < currKey.size();
2868  ++keyIsMatchIndex)
2869  {
2870  if(columnsInfo_[col]
2871  .getStorageName()[keyIsMatchStorageIndex] ==
2872  '_')
2873  ++keyIsMatchStorageIndex; // skip to next storage
2874  // character
2875  if(currKey[keyIsMatchIndex] == '_')
2876  continue; // skip to next character
2877 
2878  // match to storage name
2879  if(keyIsMatchStorageIndex >=
2880  columnsInfo_[col].getStorageName().size() ||
2881  currKey[keyIsMatchIndex] !=
2882  columnsInfo_[col]
2883  .getStorageName()[keyIsMatchStorageIndex])
2884  {
2885  // size mismatch or character mismatch
2886  keyIsMatch = false;
2887  if(!keyIsComment)
2888  break;
2889  }
2890 
2891  // check also if alternate comment is matched
2892  if(keyIsComment &&
2893  keyIsMatchCommentIndex < COMMENT_ALT_KEY.size())
2894  {
2895  if(currKey[keyIsMatchIndex] !=
2896  COMMENT_ALT_KEY[keyIsMatchCommentIndex])
2897  {
2898  // character mismatch with COMMENT
2899  keyIsComment = false;
2900  }
2901  }
2902 
2903  ++keyIsMatchStorageIndex; // go to next character
2904  }
2905 
2906  if(dbg)
2907  {
2908  __COUTTV__(keyIsMatch);
2909  __COUTTV__(keyIsComment);
2910  __COUTTV__(currKey);
2911  __COUTTV__(columnsInfo_[col].getStorageName());
2912  __COUTTV__(getNumberOfRows());
2913  }
2914 
2915  if(keyIsMatch || keyIsComment) // currKey ==
2916  // columnsInfo_[c].getStorageName())
2917  {
2918  if(keyEntryCountMap.find(currKey) ==
2919  keyEntryCountMap.end())
2920  keyEntryCountMap[currKey] =
2921  0; // show follow row count
2922  else
2923  ++keyEntryCountMap.at(currKey);
2924 
2925  // add row (based on entry counts)
2926  if(keyEntryCountMap.size() == 1 ||
2927  (keyEntryCountMap.at(currKey) &&
2928  keyEntryCountMap.at(currKey) >
2929  row)) // if(col <= lastCol)
2930  {
2931  if(getNumberOfRows()) // skip first time
2932  sourceColumnMissingCount_ +=
2933  getNumberOfColumns() - colFoundCount;
2934 
2935  colFoundCount = 0; // reset column found count
2936  row = addRow();
2937  }
2938  lastCol = col;
2939  ++colFoundCount;
2940 
2941  if(getNumberOfRows() == 1) // only for first row
2942  sourceColumnNames_.emplace(currKey);
2943 
2944  // add value to row and column
2945 
2946  if(row >= getNumberOfRows())
2947  {
2948  __SS__ << "Invalid row"
2949  << __E__; // should be impossible?!
2950  __COUT__ << "\n" << ss.str();
2951  __SS_THROW__;
2952  return -1; // never gets here
2953  }
2954 
2955  theDataView_[row][col] = currVal;
2956  break;
2957  }
2958  }
2959  }
2960 
2961  if(ccnt >= getNumberOfColumns())
2962  {
2963  __COUT__
2964  << "Invalid column in JSON source data: " << currKey
2965  << " not found in column names of table named "
2966  << getTableName() << "."
2967  << __E__; // input data doesn't match config description
2968 
2969  // CHANGED on 11/10/2016
2970  // to.. try just not populating data instead of error
2971  ++sourceColumnMismatchCount_; // but count errors
2972  if(getNumberOfRows() ==
2973  1) // only for first row, track source column names
2974  sourceColumnNames_.emplace(currKey);
2975 
2976  //__SS_THROW__;
2977  __COUT_WARN__ << "Trying to ignore error, and not populating "
2978  "missing column."
2979  << __E__;
2980  }
2981  else // short cut to proper column hopefully in next search
2982  colSpeedup = (colSpeedup + 1) % noc;
2983  }
2984  break;
2985  default:; // unknown match?
2986  } // end switch statement to match json key
2987  } // end matched key if statement
2988 
2989  // clean up handling of new value
2990 
2991  newString = 0; // toggle flag
2992  newValue = 0; // toggle flag
2993  // isDataArray = 0;
2994  endNumber = -1; // reset end number index
2995  }
2996 
2997  // if(i>200) break; //185
2998  }
2999 
3000  //__COUT__ << "Done!" << __E__;
3001  __COUTTV__(fillWithLooseColumnMatching_);
3002  __COUTTV__(sourceColumnNames_.size());
3003  //__COUTV__(tableName_); // << "tableName_ = " << tableName_
3004 
3005  if(!fillWithLooseColumnMatching_ && sourceColumnMissingCount_ > 0)
3006  {
3007  __COUTV__(sourceColumnMissingCount_);
3008  __SS__ << "Can not ignore errors because not every column was found in the "
3009  "source data!"
3010  << ". Please see the details below:\n\n"
3011  << getMismatchColumnInfo() << StringMacros::stackTrace();
3012  __SS_ONLY_THROW__;
3013  }
3014 
3015  if(sourceColumnNames_.size() ==
3016  0) //if not populated by data (i.e. zero records), then use default column names
3017  {
3018  for(unsigned int i = 0; i < getNumberOfColumns(); ++i)
3019  sourceColumnNames_.emplace(getColumnsInfo()[i].getStorageName());
3020  }
3021 
3022  // print();
3023 
3024  return 0; // success
3025 } // end fillFromJSON()
3026 
3027 //==============================================================================
3028 std::string TableView::getMismatchColumnInfo(void) const
3029 {
3030  const std::set<std::string>& srcColNames = getSourceColumnNames();
3031  std::set<std::string> destColNames = getColumnStorageNames();
3032 
3033  __SS__ << "The source column size was found to be " << srcColNames.size()
3034  << ", and the current number of columns for this table is "
3035  << getNumberOfColumns() << ". This resulted in a count of "
3036  << getSourceColumnMismatch() << " source column mismatches, and a count of "
3037  << getSourceColumnMissing() << " table entries missing in "
3038  << getNumberOfRows() << " row(s) of data." << __E__;
3039 
3040  ss << "\n\n"
3041  << srcColNames.size()
3042  << " Source column names in ALPHABETICAL order were as follows:\n";
3043  char index = 'a';
3044  std::string preIndexStr = "";
3045  for(auto& srcColName : srcColNames)
3046  {
3047  if(destColNames.find(srcColName) == destColNames.end())
3048  ss << "\n\t*** " << preIndexStr << index << ". " << srcColName << " ***";
3049  else
3050  ss << "\n\t" << preIndexStr << index << ". " << srcColName;
3051 
3052  if(index == 'z') // wrap-around
3053  {
3054  preIndexStr += 'a'; // keep adding index 'digits' for wrap-around
3055  index = 'a';
3056  }
3057  else
3058  ++index;
3059  }
3060  ss << __E__;
3061 
3062  ss << "\n\n"
3063  << destColNames.size()
3064  << " Current table column names in ALPHABETICAL order are as follows:\n";
3065  index = 'a';
3066  preIndexStr = "";
3067  for(auto& destColName : destColNames)
3068  {
3069  if(srcColNames.find(destColName) == srcColNames.end())
3070  ss << "\n\t*** " << preIndexStr << index << ". " << destColName << " ***";
3071  else
3072  ss << "\n\t" << preIndexStr << index << ". " << destColName;
3073 
3074  if(index == 'z') // wrap-around
3075  {
3076  preIndexStr += 'a'; // keep adding index 'digits' for wrap-around
3077  index = 'a';
3078  }
3079  else
3080  ++index;
3081  }
3082  ss << __E__;
3083  return ss.str();
3084 } // end getMismatchColumnInfo()
3085 
3086 //==============================================================================
3087 bool TableView::isURIEncodedCommentTheSame(const std::string& comment) const
3088 {
3089  std::string compareStr = StringMacros::decodeURIComponent(comment);
3090  return comment_ == compareStr;
3091 }
3096 //{
3097 // __COUT__ << "valueStr " << valueStr << __E__;
3098 //
3099 // if(!(c < columnsInfo_.size() && r < getNumberOfRows()))
3100 // {
3101 // __SS__ << "Invalid row (" << (int)r << ") col (" << (int)c << ") requested!" <<
3102 //__E__;
3103 // __SS_THROW__;
3104 // }
3105 //
3106 // __COUT__ << "originalValueStr " << theDataView_[r][c] << __E__;
3107 //
3108 // if(columnsInfo_[c].getDataType() == TableViewColumnInfo::DATATYPE_TIME)
3109 // {
3110 // time_t valueTime(strtol(valueStr.c_str(),0,10));
3111 // time_t originalValueTime;
3112 // getValue(originalValueTime,r,c);
3113 // __COUT__ << "time_t valueStr " << valueTime << __E__;
3114 // __COUT__ << "time_t originalValueStr " << originalValueTime << __E__;
3115 // return valueTime == originalValueTime;
3116 // }
3117 // else
3118 // {
3119 // return valueStr == theDataView_[r][c];
3120 // }
3121 //}
3122 
3123 //==============================================================================
3134 void TableView::fillFromCSV(const std::string& data,
3135  const int& dataOffset /* = 0 */,
3136  const std::string& author /* = "" */,
3137  const char rowDelimter /* = ',' */,
3138  const char colDelimter /* = '\n' */)
3139 {
3140  int row = dataOffset;
3141  int col = 0;
3142  std::string currentValue = "";
3143  bool insideQuotes = false;
3144  int authorCol = findColByType(TableViewColumnInfo::TYPE_AUTHOR);
3145  int timestampCol = findColByType(TableViewColumnInfo::TYPE_TIMESTAMP);
3146 
3147  for(size_t i = 0; i < data.size(); ++i)
3148  {
3149  char c = data[i];
3150  const char nextChar = (i + 1 < data.size() ? data[i + 1] : ' ');
3151 
3152  if(c == '"')
3153  {
3154  if(insideQuotes && nextChar == '"') // "" will escape a double-quote in CSV
3155  {
3156  // Escaped double-quote
3157  currentValue += '"';
3158  ++i; //skip next quote
3159  }
3160  else
3161  {
3162  // Toggle quote mode
3163  insideQuotes = !insideQuotes;
3164  }
3165  }
3166  else if(c == rowDelimter && !insideQuotes)
3167  {
3168  if(col == 0 && row >= (int)getNumberOfRows())
3169  addRow(author);
3170  setValueAsString(StringMacros::trim(currentValue), row, col);
3171  ++col;
3172  currentValue = "";
3173  }
3174  else if((c == colDelimter || c == '\r') && !insideQuotes)
3175  {
3176  if(col > 0)
3177  {
3178  setValueAsString(StringMacros::trim(currentValue), row, col);
3179  __COUTV__(getValueAsString(row, col));
3180 
3181  //if row is actually column names, then delete the row
3182  if(getValueAsString(row, col) == getColumnsInfo()[col].getStorageName())
3183  {
3184  __COUT__ << "First row detected as column names." << __E__;
3185  deleteRow(row);
3186  --row; //rewind
3187  }
3188  else
3189  {
3190  //enforce author and timestamp not from CSV data
3191  setValue(author, row, authorCol);
3192  setValue(time(0), row, timestampCol);
3193  }
3194 
3195  col = 0;
3196  ++row; //prepare for next row
3197  currentValue = "";
3198  }
3199  }
3200  else
3201  {
3202  currentValue += c;
3203  }
3204  } //end text loop
3205 
3206  // Add last value if any
3207  if(col > 0)
3208  {
3209  setValueAsString(StringMacros::trim(currentValue), row, col);
3210  __COUTV__(getValueAsString(row, col));
3211  __COUTV__(getValueAsString(row, timestampCol));
3212 
3213  //if row is actually column names, then delete the row
3214  if(getValueAsString(row, col) == getColumnsInfo()[col].getStorageName())
3215  {
3216  __COUT__ << "First row detected as column names." << __E__;
3217  deleteRow(row);
3218  --row; //rewind
3219  }
3220  else
3221  {
3222  //enforce author and timestamp not from CSV data
3223  setValue(author, row, authorCol);
3224  setValue(time(0), row, timestampCol);
3225  }
3226 
3227  col = 0;
3228  ++row; //prepare for next row
3229  currentValue = "";
3230  }
3231 
3232  init(); // verify new table (throws runtime_errors)
3233 
3234 } //end fillFromCSV
3235 
3236 //==============================================================================
3262 int TableView::fillFromEncodedCSV(const std::string& data,
3263  const int& dataOffset,
3264  const std::string& author)
3265 {
3266  int retVal = 0;
3267 
3268  int r = dataOffset;
3269  int c = 0;
3270 
3271  int i = 0; // use to parse data std::string
3272  int j = data.find(',', i); // find next cell delimiter
3273  int k = data.find(';', i); // find next row delimiter
3274 
3275  bool rowWasModified;
3276  unsigned int countRowsModified = 0;
3277  int authorCol = findColByType(TableViewColumnInfo::TYPE_AUTHOR);
3278  int timestampCol = findColByType(TableViewColumnInfo::TYPE_TIMESTAMP);
3279  // std::string valueStr, tmpTimeStr, originalValueStr;
3280 
3281  while(k != (int)(std::string::npos))
3282  {
3283  rowWasModified = false;
3284  if(r >= (int)getNumberOfRows())
3285  {
3286  addRow();
3287  //__COUT__ << "Row added" << __E__;
3288  rowWasModified = true;
3289  }
3290 
3291  while(j < k && j != (int)(std::string::npos))
3292  {
3293  //__COUT__ << "Col " << (int)c << __E__;
3294 
3295  // skip last 2 columns
3296  if(c >= (int)getNumberOfColumns() - 2)
3297  {
3298  i = j + 1;
3299  j = data.find(',', i); // find next cell delimiter
3300  ++c;
3301  continue;
3302  }
3303 
3304  if(setURIEncodedValue(data.substr(i, j - i), r, c))
3305  rowWasModified = true;
3306 
3307  i = j + 1;
3308  j = data.find(',', i); // find next cell delimiter
3309  ++c;
3310  }
3311 
3312  // if row was modified, assign author and timestamp
3313  if(author != "" && rowWasModified)
3314  {
3315  __COUTT__ << "Row=" << (int)r << " was modified!" << __E__;
3316  setValue(author, r, authorCol);
3317  setValue(time(0), r, timestampCol);
3318  }
3319 
3320  if(rowWasModified)
3321  ++countRowsModified;
3322 
3323  ++r;
3324  c = 0;
3325 
3326  i = k + 1;
3327  j = data.find(',', i); // find next cell delimiter
3328  k = data.find(';', i); // find new row delimiter
3329  }
3330 
3331  // delete excess rows
3332  while(r < (int)getNumberOfRows())
3333  {
3334  deleteRow(r);
3335  __COUT__ << "Row deleted: " << (int)r << __E__;
3336  ++countRowsModified;
3337  }
3338 
3339  __COUT_INFO__ << "countRowsModified=" << countRowsModified << __E__;
3340 
3341  if(!countRowsModified)
3342  {
3343  // check that source columns match storage name
3344  // otherwise allow same data...
3345 
3346  bool match = getColumnStorageNames().size() == getSourceColumnNames().size();
3347  if(match)
3348  {
3349  for(auto& destColName : getColumnStorageNames())
3350  if(getSourceColumnNames().find(destColName) ==
3351  getSourceColumnNames().end())
3352  {
3353  __COUT__ << "Found column name mismach for '" << destColName
3354  << "'... So allowing same data!" << __E__;
3355 
3356  match = false;
3357  break;
3358  }
3359  }
3360  // if still a match, do not allow!
3361  if(match)
3362  {
3363  __SS__ << "No rows were modified! No reason to fill a view with same content."
3364  << __E__;
3365  __COUT__ << "\n" << ss.str();
3366  return -1;
3367  }
3368  // else mark with retVal
3369  retVal = 1;
3370  } // end same check
3371 
3372  // print(); //for debugging
3373 
3374  // setup sourceColumnNames_ to be correct
3375  sourceColumnNames_.clear();
3376  for(unsigned int i = 0; i < getNumberOfColumns(); ++i)
3377  sourceColumnNames_.emplace(getColumnsInfo()[i].getStorageName());
3378 
3379  init(); // verify new table (throws runtime_errors)
3380 
3381  // printout for debugging
3382  // __SS__ << "\n";
3383  // print(ss);
3384  // __COUT__ << "\n" << ss.str() << __E__;
3385 
3386  return retVal;
3387 } // end fillFromEncodedCSV()
3388 
3389 //==============================================================================
3398 bool TableView::setURIEncodedValue(const std::string& value,
3399  const unsigned int& r,
3400  const unsigned int& c,
3401  const std::string& author)
3402 {
3403  if(!(c < columnsInfo_.size() && r < getNumberOfRows()))
3404  {
3405  __SS__ << "Invalid row (" << (int)r << ") col (" << (int)c << ") requested!"
3406  << "Number of Rows = " << getNumberOfRows()
3407  << "Number of Columns = " << columnsInfo_.size() << __E__;
3408  print(ss);
3409  __SS_THROW__;
3410  }
3411 
3412  std::string valueStr = StringMacros::decodeURIComponent(value);
3413  std::string originalValueStr =
3414  getValueAsString(r, c, false); // do not convert env variables
3415 
3416  //__COUT__ << "valueStr " << valueStr << __E__;
3417  //__COUT__ << "originalValueStr " << originalValueStr << __E__;
3418 
3419  if(columnsInfo_[c].getDataType() == TableViewColumnInfo::DATATYPE_NUMBER)
3420  {
3421  // check if valid number
3422  std::string convertedString = StringMacros::convertEnvironmentVariables(valueStr);
3423  // do not check here, let init check
3424  // if this is a link to valid number, then this is an improper check.
3425  // if(!StringMacros::isNumber(convertedString))
3426  // {
3427  // __SS__ << "\tIn configuration " << tableName_
3428  // << " at column=" << columnsInfo_[c].getName() << " the value
3429  // set
3430  //("
3431  // << convertedString << ")"
3432  // << " is not a number! Please fix it or change the column
3433  // type..."
3434  // << __E__;
3435  // __SS_THROW__;
3436  // }
3437  theDataView_[r][c] = valueStr;
3438 
3439  // is it here that a new exception should be added to enforce min and max, given that they only appear with number type?
3440  }
3441  else if(columnsInfo_[c].getDataType() == TableViewColumnInfo::DATATYPE_TIME)
3442  {
3443  // valueStr = StringMacros::decodeURIComponent(data.substr(i,j-i));
3444  //
3445  // getValue(tmpTimeStr,r,c);
3446  // if(valueStr != tmpTimeStr)//theDataView_[r][c])
3447  // {
3448  // __COUT__ << "valueStr=" << valueStr <<
3449  // " theDataView_[r][c]=" << tmpTimeStr << __E__;
3450  // rowWasModified = true;
3451  // }
3452 
3453  setValue(time_t(strtol(valueStr.c_str(), 0, 10)), r, c);
3454  }
3455  else
3456  theDataView_[r][c] = valueStr;
3457 
3458  bool rowWasModified =
3459  (originalValueStr !=
3460  getValueAsString(r, c, false)); // do not convert env variables
3461 
3462  // if row was modified, assign author and timestamp
3463  if(author != "" && rowWasModified)
3464  {
3465  __COUT__ << "Row=" << (int)r << " was modified!" << __E__;
3466  int authorCol = findColByType(TableViewColumnInfo::TYPE_AUTHOR);
3467  int timestampCol = findColByType(TableViewColumnInfo::TYPE_TIMESTAMP);
3468  setValue(author, r, authorCol);
3469  setValue(time(0), r, timestampCol);
3470  }
3471 
3472  return rowWasModified;
3473 } // end setURIEncodedValue()
3474 
3475 //==============================================================================
3476 void TableView::resizeDataView(unsigned int nRows, unsigned int nCols)
3477 {
3478  // FIXME This maybe should disappear but I am using it in ConfigurationHandler
3479  // still...
3480  theDataView_.resize(nRows, std::vector<std::string>(nCols));
3481 }
3482 
3483 //==============================================================================
3490 unsigned int TableView::addRow(
3491  const std::string& author,
3492  unsigned char
3493  incrementUniqueData /* = false */, // leave as unsigned char rather than
3494  // bool, too many things (e.g. strings)
3495  // evaluate successfully to bool values
3496  const std::string& baseNameAutoUID /* = "" */,
3497  unsigned int rowToAdd /* = -1 */,
3498  std::string childLinkIndex /* = "" */,
3499  std::string groupId /* = "" */)
3500 {
3501  // default to last row
3502  if(rowToAdd == (unsigned int)-1)
3503  rowToAdd = getNumberOfRows();
3504 
3505  theDataView_.resize(getNumberOfRows() + 1,
3506  std::vector<std::string>(getNumberOfColumns()));
3507 
3508  // shift data down the table if necessary
3509  for(unsigned int r = getNumberOfRows() - 2; r >= rowToAdd; --r)
3510  {
3511  if(r == (unsigned int)-1)
3512  break; // quit wrap around case
3513  for(unsigned int col = 0; col < getNumberOfColumns(); ++col)
3514  theDataView_[r + 1][col] = theDataView_[r][col];
3515  }
3516 
3517  std::vector<std::string> defaultRowValues = getDefaultRowValues();
3518 
3519  // char indexString[1000];
3520  std::string tmpString, baseString;
3521  // bool foundAny;
3522  // unsigned int index;
3523  // unsigned int maxUniqueData;
3524  std::string numString;
3525 
3526  // fill each col of new row with default values
3527  // if a row is a unique data row, increment last row in attempt to make a legal
3528  // column
3529  for(unsigned int col = 0; col < getNumberOfColumns(); ++col)
3530  {
3531  // if(incrementUniqueData)
3532  // __COUT__ << col << " " << columnsInfo_[col].getType() << " basename= " <<
3533  // baseNameAutoUID << __E__;
3534 
3535  // baseNameAutoUID indicates to attempt to make row unique
3536  // add index to max number
3537  if(incrementUniqueData &&
3538  (col == getColUID() || columnsInfo_[col].isChildLinkGroupID() ||
3539  (getNumberOfRows() > 1 &&
3540  (columnsInfo_[col].getType() == TableViewColumnInfo::TYPE_UNIQUE_DATA ||
3541  columnsInfo_[col].getType() ==
3542  TableViewColumnInfo::TYPE_UNIQUE_GROUP_DATA))))
3543  {
3544  if(col == getColUID() || columnsInfo_[col].isChildLinkGroupID())
3545  setUniqueColumnValue(
3546  rowToAdd, col, baseNameAutoUID /*baseValueAsString*/);
3547  else
3548  setUniqueColumnValue(rowToAdd,
3549  col,
3550  "" /* baseValueAsString */,
3551  false /* doMathAppendStrategy */,
3552  childLinkIndex,
3553  groupId);
3554  }
3555  else
3556  theDataView_[rowToAdd][col] = defaultRowValues[col];
3557  }
3558 
3559  if(author != "")
3560  {
3561  __COUT__ << "Row=" << rowToAdd << " was created!" << __E__;
3562 
3563  int authorCol = findColByType(TableViewColumnInfo::TYPE_AUTHOR);
3564  int timestampCol = findColByType(TableViewColumnInfo::TYPE_TIMESTAMP);
3565  setValue(author, rowToAdd, authorCol);
3566  setValue(time(0), rowToAdd, timestampCol);
3567  }
3568 
3569  return rowToAdd;
3570 } // end addRow()
3571 
3572 //==============================================================================
3575 void TableView::deleteRow(int r)
3576 {
3577  if(r >= (int)getNumberOfRows())
3578  {
3579  // out of bounds
3580  __SS__ << "Row " << (int)r
3581  << " is out of bounds (Row Count = " << getNumberOfRows()
3582  << ") and can not be deleted." << __E__;
3583  __SS_THROW__;
3584  }
3585 
3586  theDataView_.erase(theDataView_.begin() + r);
3587 } // end deleteRow()
3588 
3589 //==============================================================================
3605 bool TableView::getChildLink(
3606  const unsigned int& c,
3607  bool& isGroup,
3608  std::pair<unsigned int /*link col*/, unsigned int /*link id col*/>& linkPair) const
3609 {
3610  if(!(c < columnsInfo_.size()))
3611  {
3612  __SS__ << "Invalid col (" << (int)c << ") requested for child link!" << __E__;
3613  __SS_THROW__;
3614  }
3615 
3616  //__COUT__ << "getChildLink for col: " << (int)c << "-" <<
3617  // columnsInfo_[c].getType() << "-" << columnsInfo_[c].getName() << __E__;
3618 
3619  // check if column is a child link UID
3620  if((isGroup = columnsInfo_[c].isChildLinkGroupID()) ||
3621  columnsInfo_[c].isChildLinkUID())
3622  {
3623  // must be part of unique link, (or invalid table?)
3624  //__COUT__ << "col: " << (int)c << __E__;
3625  linkPair.second = c;
3626  std::string index = columnsInfo_[c].getChildLinkIndex();
3627 
3628  //__COUT__ << "index: " << index << __E__;
3629 
3630  // find pair link
3631  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
3632  {
3633  //__COUT__ << "try: " << col << "-" << columnsInfo_[col].getType() << "-" <<
3634  // columnsInfo_[col].getName() << __E__;
3635  if(col == c)
3636  continue; // skip column c that we know
3637  else if(columnsInfo_[col].isChildLink() &&
3638  index == columnsInfo_[col].getChildLinkIndex())
3639  {
3640  // found match!
3641  //__COUT__ << "getChildLink Found match for col: " << (int)c << " at " <<
3642  // col << __E__;
3643  linkPair.first = col;
3644  return true;
3645  }
3646  }
3647 
3648  // if here then invalid table!
3649  __SS__ << "\tIn view: " << tableName_
3650  << ", Can't find complete child link for column name "
3651  << columnsInfo_[c].getName() << ". Child link index is '" << index
3652  << "' - is there a mismatch? Or was this intended to be the target of a "
3653  "Group Link (column type 'GroupID')? "
3654  << __E__;
3655  __SS_THROW__;
3656  }
3657 
3658  if(!columnsInfo_[c].isChildLink())
3659  return false; // cant be unique link
3660 
3661  // this is child link, so find pair link uid or gid column
3662  linkPair.first = c;
3663  std::string index = columnsInfo_[c].getChildLinkIndex();
3664 
3665  //__COUT__ << "index: " << index << __E__;
3666 
3667  // find pair link
3668  for(unsigned int col = 0; col < columnsInfo_.size(); ++col)
3669  {
3670  //__COUT__ << "try: " << col << "-" << columnsInfo_[col].getType() << "-" <<
3671  // columnsInfo_[col].getName() << __E__;
3672  if(col == c)
3673  continue; // skip column c that we know
3674  // __COUT__ << "try: " << col << "-" << columnsInfo_[col].getType() <<
3675  // "-" << columnsInfo_[col].getName() <<
3676  // "-u" << columnsInfo_[col].isChildLinkUID() <<
3677  // "-g" << columnsInfo_[col].isChildLinkGroupID() << __E__;
3678  //
3679  // if(columnsInfo_[col].isChildLinkUID())
3680  // __COUT__ << "-L" << columnsInfo_[col].getChildLinkIndex() << __E__;
3681  //
3682  // if(columnsInfo_[col].isChildLinkGroupID())
3683  // __COUT__ << "-L" << columnsInfo_[col].getChildLinkIndex() << __E__;
3684 
3685  if(((columnsInfo_[col].isChildLinkUID() && !(isGroup = false)) ||
3686  (columnsInfo_[col].isChildLinkGroupID() && (isGroup = true))) &&
3687  index == columnsInfo_[col].getChildLinkIndex())
3688  {
3689  // found match!
3690  //__COUT__ << "getChildLink Found match for col: " << (int)c << " at " << col
3691  //<< __E__;
3692  linkPair.second = col;
3693  return true;
3694  }
3695  }
3696 
3697  // if here then invalid table!
3698  __SS__ << "\tIn view: " << tableName_
3699  << ", Can't find complete child link id for column name "
3700  << columnsInfo_[c].getName() << __E__;
3701  __SS_THROW__;
3702 } // end getChildLink()
ots::TableBase::convertToCaps
static std::string convertToCaps(std::string &str, bool isConfigName=false)
Definition: TableBase.cc:1898
ots::TableViewColumnInfo::DATATYPE_NUMBER
static const std::string DATATYPE_NUMBER
Definition: TableViewColumnInfo.h:74
ots::TableViewColumnInfo::TYPE_UID
static const std::string TYPE_UID
NOTE: Do NOT put '-' in static const TYPEs because it will mess up javascript handling in the web gui...
Definition: TableViewColumnInfo.h:69
ots::TableView::findRow
unsigned int findRow(unsigned int col, const T &value, unsigned int offsetRow=0, bool doNotThrow=false) const
< in included .icc source
ots::TableView::getEscapedValueAsString
std::string getEscapedValueAsString(unsigned int row, unsigned int col, bool convertEnvironmentVariables=true, bool quotesToDoubleQuotes=false) const
Definition: TableView.cc:1024
ots::TableView::isEntryInGroup
bool isEntryInGroup(const unsigned int &row, const std::string &childLinkIndex, const std::string &groupNeedle) const
Definition: TableView.cc:1663
ots::TableView::setValueAsString
void setValueAsString(const std::string &value, unsigned int row, unsigned int col)
Definition: TableView.cc:1090
ots::TableView::deleteRow
void deleteRow(int r)
Definition: TableView.cc:3575
ots::TableView::getGroupRowsByPriority
std::vector< std::vector< unsigned int > > getGroupRowsByPriority(const unsigned int groupIdCol, const std::string &groupID, bool onlyStatusTrue=false) const
Definition: TableView.cc:1520
ots::TableView::validateValueForColumn
T validateValueForColumn(const std::string &value, unsigned int col, bool doConvertEnvironmentVariables=true) const
< in included .icc source
ots::TableView::TableView
TableView(const std::string &tableName)
= "");
Definition: TableView.cc:20
ots::TableView::getColStatus
unsigned int getColStatus(void) const
Definition: TableView.cc:1407
ots::TableView::getLinkGroupIDColumn
unsigned int getLinkGroupIDColumn(const std::string &childLinkIndex) const
Definition: TableView.cc:1858
ots::TableView::removeRowFromGroup
bool removeRowFromGroup(const unsigned int &row, const unsigned int &col, const std::string &groupID, bool deleteRowIfNoGroupLeft=false)
Definition: TableView.cc:1607
ots::TableView::findColByType
unsigned int findColByType(const std::string &type, unsigned int startingCol=0) const
Definition: TableView.cc:1996
ots::TableView::getChildLink
bool getChildLink(const unsigned int &col, bool &isGroup, std::pair< unsigned int, unsigned int > &linkPair) const
Definition: TableView.cc:3605
ots::TableView::addRowToGroup
void addRowToGroup(const unsigned int &row, const unsigned int &col, const std::string &groupID)
, const std::string& colDefault);
Definition: TableView.cc:1465
ots::TableView::copyRows
unsigned int copyRows(const std::string &author, const TableView &src, unsigned int srcOffsetRow=0, unsigned int srcRowsToCopy=(unsigned int) -1, unsigned int destOffsetRow=(unsigned int) -1, unsigned char generateUniqueDataColumns=false, const std::string &baseNameAutoUID="")
Definition: TableView.cc:126
ots::TableView::getColPriority
unsigned int getColPriority(void) const
Definition: TableView.cc:1438
ots::TableView::setUniqueColumnValue
const std::string & setUniqueColumnValue(unsigned int row, unsigned int col, std::string baseValueAsString="", bool doMathAppendStrategy=false, std::string childLinkIndex="", std::string groupId="")
Definition: TableView.cc:1110
ots::TableView::init
void init(void)
Definition: TableView.cc:196
ots::TableView::getValueAsString
std::string getValueAsString(unsigned int row, unsigned int col, bool convertEnvironmentVariables=true) const
Definition: TableView.cc:973
ots::TableView::getGroupRows
std::vector< unsigned int > getGroupRows(const unsigned int groupIdCol, const std::string &groupID, bool onlyStatusTrue=false, bool orderedByPriority=false) const
Definition: TableView.cc:1497
ots::TableView::getCustomStorageData
const std::string & getCustomStorageData(void) const
Getters.
Definition: TableView.h:71
ots::TableView::getSetOfGroupIDs
std::set< std::string > getSetOfGroupIDs(const std::string &childLinkIndex, unsigned int row=-1) const
Definition: TableView.cc:1757
ots::TableView::getValue
void getValue(T &value, unsigned int row, unsigned int col, bool doConvertEnvironmentVariables=true) const
< in included .icc source
ots::TableView::getDataColumnSize
unsigned int getDataColumnSize(void) const
getDataColumnSize
Definition: TableView.cc:2013
ots::TableView::fillFromJSON
int fillFromJSON(const std::string &json)
Definition: TableView.cc:2374
ots::TableView::getColUID
unsigned int getColUID(void) const
Definition: TableView.cc:1322
ots::TableView::setURIEncodedValue
bool setURIEncodedValue(const std::string &value, const unsigned int &row, const unsigned int &col, const std::string &author="")
Definition: TableView.cc:3398
ots::TableView::fillFromCSV
void fillFromCSV(const std::string &data, const int &dataOffset=0, const std::string &author="", const char rowDelimter=',', const char colDelimter='\n')
Definition: TableView.cc:3134
ots::TableView::fillFromEncodedCSV
int fillFromEncodedCSV(const std::string &data, const int &dataOffset=0, const std::string &author="")
Definition: TableView.cc:3262
ots::TableView::findCol
unsigned int findCol(const std::string &name) const
Definition: TableView.cc:1973
ots::TableView::setValue
void setValue(const T &value, unsigned int row, unsigned int col)
< in included .icc source
ots::TableView::setURIEncodedComment
void setURIEncodedComment(const std::string &uriComment)
Definition: TableView.cc:2128
ots::TableView::addRow
unsigned int addRow(const std::string &author="", unsigned char incrementUniqueData=false, const std::string &baseNameAutoUID="", unsigned int rowToAdd=(unsigned int) -1, std::string childLinkIndex="", std::string groupId="")
Definition: TableView.cc:3490
ots::TableView::findRowInGroup
unsigned int findRowInGroup(unsigned int col, const T &value, const std::string &groupId, const std::string &childLinkIndex, unsigned int offsetRow=0) const
< in included .icc source
ots::TableView::setCustomStorageData
void setCustomStorageData(const std::string &storageData)
Definition: TableView.h:168
ots
defines used also by OtsConfigurationWizardSupervisor
Definition: ArtdaqOtsBuildInfo_module.cc:10
ots::StringMacros::getTimestampString
static std::string getTimestampString(const std::string &linuxTimeInSeconds)
Definition: StringMacros.cc:945
ots::StringMacros::trim
static const std::string & trim(std::string &s)
Definition: StringMacros.cc:567
ots::StringMacros::getSetFromString
static void getSetFromString(const std::string &inputString, std::set< std::string > &setToReturn, const std::set< char > &delimiter={',', '|', '&'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'})
Definition: StringMacros.cc:1004
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
setToString ~
ots::StringMacros::convertEnvironmentVariables
static std::string convertEnvironmentVariables(const std::string &data)
Definition: StringMacros.cc:593
ots::StringMacros::demangleTypeName
static std::string demangleTypeName(const char *name)
Definition: StringMacros.cc:2140
ots::StringMacros::restoreJSONStringEntities
static std::string restoreJSONStringEntities(const std::string &str)
Definition: StringMacros.cc:518
ots::StringMacros::wildCardMatch
static bool wildCardMatch(const std::string &needle, const std::string &haystack, unsigned int *priorityIndex=0)
Definition: StringMacros.cc:49
ots::StringMacros::decodeURIComponent
static std::string decodeURIComponent(const std::string &data)
Definition: StringMacros.cc:185
ots::StringMacros::stackTrace
static std::string stackTrace(void)
Definition: StringMacros.cc:1836
ots::StringMacros::getNumber
static bool getNumber(const std::string &s, T &retValue)