Line data Source code
1 : #include "artdaq/ArtModules/RootDAQOutFile.h"
2 : #include "TRACE/tracemf.h"
3 : // vim: set sw=2 expandtab :
4 :
5 : #include "artdaq/DAQdata/Globals.hh"
6 :
7 : #include "art/Framework/Core/OutputFileGranularity.h"
8 : #include "art/Framework/Core/OutputModule.h"
9 : #include "art/Framework/IO/ClosingCriteria.h"
10 : #include "art/Framework/IO/FileStatsCollector.h"
11 : #include "art/Framework/Principal/EventPrincipal.h"
12 : #include "art/Framework/Principal/RangeSetsSupported.h"
13 : #include "art/Framework/Principal/ResultsPrincipal.h"
14 : #include "art/Framework/Principal/RunPrincipal.h"
15 : #include "art/Framework/Principal/SubRunPrincipal.h"
16 : #include "art/Framework/Services/Registry/ServiceHandle.h"
17 : #include "art/Framework/Services/System/DatabaseConnection.h"
18 : #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
19 : #include "art_root_io/DropMetaData.h"
20 : #include "art_root_io/GetFileFormatEra.h"
21 : #include "art_root_io/GetFileFormatVersion.h"
22 : #include "art_root_io/RootDB/TKeyVFSOpenPolicy.h"
23 : #include "art_root_io/RootFileBlock.h"
24 : #include "art_root_io/checkDictionaries.h"
25 : #include "art_root_io/detail/getObjectRequireDict.h"
26 : #include "boost/date_time/posix_time/posix_time.hpp"
27 : #include "canvas/Persistency/Provenance/BranchChildren.h"
28 : #include "canvas/Persistency/Provenance/BranchType.h"
29 : #include "canvas/Persistency/Provenance/EventAuxiliary.h"
30 : #include "canvas/Persistency/Provenance/EventID.h"
31 : #include "canvas/Persistency/Provenance/FileFormatVersion.h"
32 : #include "canvas/Persistency/Provenance/Parentage.h"
33 : #include "canvas/Persistency/Provenance/ParentageRegistry.h"
34 : #include "canvas/Persistency/Provenance/ProductStatus.h"
35 : #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
36 : #include "canvas/Persistency/Provenance/RunAuxiliary.h"
37 : #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
38 : #include "canvas/Persistency/Provenance/rootNames.h"
39 : #include "canvas/Utilities/Exception.h"
40 : #include "canvas_root_io/Utilities/DictionaryChecker.h"
41 : #include "cetlib/canonical_string.h"
42 : #include "cetlib/container_algorithms.h"
43 : #include "cetlib/exempt_ptr.h"
44 : #include "cetlib/sqlite/Ntuple.h"
45 : #include "cetlib/sqlite/Transaction.h"
46 : #include "cetlib/sqlite/create_table.h"
47 : #include "cetlib/sqlite/exec.h"
48 : #include "cetlib/sqlite/insert.h"
49 : #include "fhiclcpp/ParameterSetRegistry.h"
50 : #include "messagefacility/MessageLogger/MessageLogger.h"
51 : #include "range/v3/view.hpp"
52 :
53 : #include "TBranch.h"
54 :
55 : #define TRACE_NAME (app_name + "_RootDAQOutFile").c_str()
56 :
57 : #include "TFile.h"
58 : #include "TSystem.h"
59 : #include "TTree.h"
60 :
61 : #include <fcntl.h> // posix_fadvise POSIX_FADV_DONTNEED
62 : #include <sys/sysinfo.h> // sysinfo(sysinfo*)
63 : #include <algorithm>
64 : #include <fstream> // /proc/meminfo
65 : #include <utility>
66 : #include <vector>
67 :
68 : using namespace cet;
69 : using namespace hep::concurrency;
70 :
71 : using art::BranchType;
72 : using art::RootDAQOutFile;
73 : using art::rootNames::metaBranchRootName;
74 :
75 : using std::map;
76 : using std::string;
77 : using std::vector;
78 :
79 : namespace {
80 :
81 0 : void create_table(sqlite3* const db,
82 : string const& name,
83 : vector<string> const& columns,
84 : string const& suffix = {})
85 : {
86 0 : if (columns.empty())
87 : {
88 0 : throw art::Exception(art::errors::LogicError) // NOLINT(cert-err60-cpp)
89 0 : << "Number of sqlite columns specified for table: " << name << '\n'
90 0 : << "is zero.\n";
91 : }
92 0 : string ddl = "DROP TABLE IF EXISTS " + name +
93 : "; "
94 0 : "CREATE TABLE " +
95 0 : name + "(" + columns.front();
96 0 : for_each(columns.begin() + 1, columns.end(), [&ddl](auto const& col) {
97 0 : ddl += "," + col;
98 0 : });
99 0 : ddl += ") ";
100 0 : ddl += suffix;
101 0 : ddl += ";";
102 0 : sqlite::exec(db, ddl);
103 0 : }
104 :
105 0 : void insert_eventRanges_row(sqlite3_stmt* stmt,
106 : art::SubRunNumber_t const sr,
107 : art::EventNumber_t const b,
108 : art::EventNumber_t const e)
109 : {
110 0 : sqlite3_bind_int64(stmt, 1, sr);
111 0 : sqlite3_bind_int64(stmt, 2, b);
112 0 : sqlite3_bind_int64(stmt, 3, e);
113 0 : sqlite3_step(stmt);
114 0 : sqlite3_reset(stmt);
115 0 : }
116 :
117 0 : void insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
118 : unsigned const rsid,
119 : unsigned const esid)
120 : {
121 0 : sqlite3_bind_int64(stmt, 1, rsid);
122 0 : sqlite3_bind_int64(stmt, 2, esid);
123 0 : sqlite3_step(stmt);
124 0 : sqlite3_reset(stmt);
125 0 : }
126 :
127 : unsigned
128 0 : getNewRangeSetID(sqlite3* db,
129 : art::BranchType const bt,
130 : art::RunNumber_t const r)
131 : {
132 0 : sqlite::insert_into(db, art::BranchTypeToString(bt) + "RangeSets")
133 0 : .values(r);
134 0 : return sqlite3_last_insert_rowid(db);
135 : }
136 :
137 : vector<unsigned>
138 0 : getExistingRangeSetIDs(sqlite3* db, art::RangeSet const& rs)
139 : {
140 : using namespace std;
141 0 : vector<unsigned> rangeSetIDs;
142 0 : cet::transform_all(rs, back_inserter(rangeSetIDs), [db](auto const& range) {
143 0 : sqlite::query_result<unsigned> r;
144 0 : r << sqlite::select("ROWID")
145 0 : .from(db, "EventRanges")
146 : .where("SubRun=" + to_string(range.subRun()) +
147 : " AND "
148 : "begin=" +
149 : to_string(range.begin()) +
150 : " AND "
151 : "end=" +
152 : to_string(range.end()));
153 0 : return unique_value(r);
154 0 : });
155 0 : return rangeSetIDs;
156 0 : }
157 :
158 0 : void insertIntoEventRanges(sqlite3* db, art::RangeSet const& rs)
159 : {
160 0 : sqlite::Transaction txn{db};
161 0 : sqlite3_stmt* stmt{nullptr};
162 : string const ddl{
163 : "INSERT INTO EventRanges(SubRun, begin, end) "
164 0 : "VALUES(?, ?, ?);"};
165 0 : sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
166 0 : for (auto const& range : rs)
167 : {
168 0 : insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
169 : }
170 0 : sqlite3_finalize(stmt);
171 0 : txn.commit();
172 0 : }
173 :
174 0 : void insertIntoJoinTable(sqlite3* db,
175 : art::BranchType const bt,
176 : unsigned const rsID,
177 : vector<unsigned> const& eventRangesIDs)
178 : {
179 0 : sqlite::Transaction txn{db};
180 0 : sqlite3_stmt* stmt{nullptr};
181 : string const ddl{
182 0 : "INSERT INTO " + art::BranchTypeToString(bt) +
183 0 : "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
184 0 : sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
185 0 : cet::for_all(eventRangesIDs, [stmt, rsID](auto const eventRangeID) {
186 0 : insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
187 0 : });
188 0 : sqlite3_finalize(stmt);
189 0 : txn.commit();
190 0 : }
191 :
192 0 : void maybeInvalidateRangeSet(BranchType const bt,
193 : art::RangeSet const& principalRS,
194 : art::RangeSet& productRS)
195 : {
196 0 : assert(principalRS.is_sorted());
197 0 : assert(productRS.is_sorted());
198 0 : if (!productRS.is_valid())
199 : {
200 0 : return;
201 : }
202 0 : if (bt == art::InRun && productRS.is_full_run())
203 : {
204 0 : return;
205 : }
206 0 : if (bt == art::InSubRun && productRS.is_full_subRun())
207 : {
208 0 : return;
209 : }
210 0 : if (productRS.ranges().empty())
211 : {
212 0 : return;
213 : }
214 0 : auto const r = productRS.run();
215 0 : auto const& productFront = productRS.ranges().front();
216 0 : if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
217 : {
218 0 : productRS = art::RangeSet::invalid();
219 : }
220 : }
221 :
222 : // The purpose of 'maybeInvalidateRangeSet' is to support the
223 : // following situation. Suppose process 1 creates three files with
224 : // one Run product each, all corresponding to the same Run. Let's
225 : // call the individual Run product instances in the three separate
226 : // files as A, B, and C. Now suppose that the three files serve as
227 : // inputs to process 2, where a concatenation is being performed AND
228 : // ALSO an output file switch. Process 2 results in two output
229 : // files, and now, in process 3, we concatenate the outputs from
230 : // process 2. The situation would look like this:
231 : //
232 : // Process 1: [A] [B] [C]
233 : // \ / \ /
234 : // Process 2: [A + B] [B + C]
235 : // \ / \ /
236 : // D=agg(A,B) | | E=agg(B,C)
237 : // \ /
238 : // Process 3: [D + E]
239 : //
240 : // Notice the complication in process 3: product 'B' will be
241 : // aggregated twice: once with A, and once with C. Whenever the
242 : // output from process 3 is read as input to another process, the
243 : // fetched product will be equivalent to A+2B+C.
244 : //
245 : // To avoid this situation, we compare the RangeSet of the product
246 : // with the RangeSet of the in-memory RunAuxiliary. If the
247 : // beginning of B's RangeSet is not contained within the auxiliary's
248 : // RangeSet, then a dummy product with an invalid RangeSet is
249 : // written to disk. Instead of the diagram above, we have:
250 : //
251 : // Process 1: [A] [B] [C]
252 : // \ / \ /
253 : // Process 2: [A + B] [x + C]
254 : // \ / \ /
255 : // D=agg(A,B) | | E=agg(x,C)=C
256 : // \ /
257 : // Process 3: [D + E]
258 : //
259 : // where 'x' represent a dummy product. Upon aggregating D and E,
260 : // we obtain the correctly formed A+B+C product.
261 : template<BranchType BT>
262 : art::RangeSet
263 0 : getRangeSet(art::OutputHandle const& oh,
264 : art::RangeSet const& principalRS,
265 : bool const producedInThisProcess)
266 : {
267 : if constexpr (!art::detail::range_sets_supported(BT))
268 : {
269 0 : return art::RangeSet::invalid();
270 : }
271 :
272 0 : auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
273 : // Because a user can specify (e.g.):
274 : // r.put(std::move(myProd), art::runFragment(myRangeSet));
275 : // products that are produced in this process can have valid, yet
276 : // arbitrary RangeSets. We therefore never invalidate a RangeSet
277 : // that corresponds to a product produced in this process.
278 : //
279 : // It is possible for a user to specify a RangeSet which does not
280 : // correspond AT ALL to the in-memory auxiliary RangeSet. In that
281 : // case, users should not expect to be able to retrieve products
282 : // for which no corresponding events or sub-runs were processed.
283 0 : if (!producedInThisProcess)
284 : {
285 0 : maybeInvalidateRangeSet(BT, principalRS, rs);
286 : }
287 0 : return rs;
288 0 : }
289 :
290 : template<BranchType BT>
291 0 : void setProductRangeSetID(art::RangeSet const& rs,
292 : sqlite3* db,
293 : art::EDProduct* product,
294 : map<unsigned, unsigned>& checksumToIndexLookup)
295 : {
296 : if constexpr (!art::detail::range_sets_supported(BT))
297 : {
298 0 : return;
299 : }
300 :
301 0 : if (!rs.is_valid())
302 : { // Invalid range-sets not written to DB
303 0 : return;
304 : }
305 : // Set range sets for SubRun and Run products
306 0 : auto it = checksumToIndexLookup.find(rs.checksum());
307 0 : if (it != checksumToIndexLookup.cend())
308 : {
309 0 : product->setRangeSetID(it->second);
310 : }
311 : else
312 : {
313 0 : unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
314 0 : product->setRangeSetID(rsID);
315 0 : checksumToIndexLookup.emplace(rs.checksum(), rsID);
316 0 : insertIntoEventRanges(db, rs);
317 0 : auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
318 0 : insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
319 0 : }
320 : }
321 :
322 : } // unnamed namespace
323 :
324 : namespace art {
325 :
326 0 : OutputItem::~OutputItem() = default;
327 :
328 0 : OutputItem::OutputItem(BranchDescription const& bd)
329 0 : : branchDescription{bd}, product{nullptr}
330 0 : {}
331 :
332 0 : RootDAQOutFile::RootDAQOutFile(OutputModule* om,
333 : string const& fileName,
334 : ClosingCriteria const& fileSwitchCriteria,
335 : int const compressionLevel,
336 : unsigned freePercent,
337 : unsigned freeMB,
338 : int64_t const saveMemoryObjectThreshold,
339 : int64_t const treeMaxVirtualSize,
340 : int const splitLevel,
341 : int const basketSize,
342 : DropMetaData dropMetaData,
343 0 : bool const dropMetaDataForDroppedData)
344 0 : : om_{om}
345 0 : , file_{fileName}
346 0 : , fileSwitchCriteria_{fileSwitchCriteria}
347 0 : , compressionLevel_{compressionLevel}
348 0 : , freePercent_{freePercent}
349 0 : , freeMB_{freeMB}
350 0 : , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
351 0 : , treeMaxVirtualSize_{treeMaxVirtualSize}
352 0 : , splitLevel_{splitLevel}
353 0 : , basketSize_{basketSize}
354 0 : , dropMetaData_{dropMetaData}
355 0 : , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
356 0 : , filePtr_{TFile::Open(file_.c_str(), "recreate", "", compressionLevel)}
357 : {
358 0 : gSystem->Chmod(file_.c_str(), 0664);
359 : using std::make_unique;
360 : // Don't split metadata tree or event description tree
361 0 : metaDataTree_ = RootOutputTree::makeTTree(
362 : filePtr_.get(), rootNames::metaDataTreeName(), 0);
363 0 : fileIndexTree_ = RootOutputTree::makeTTree(
364 : filePtr_.get(), rootNames::fileIndexTreeName(), 0);
365 0 : parentageTree_ = RootOutputTree::makeTTree(
366 : filePtr_.get(), rootNames::parentageTreeName(), 0);
367 0 : treePointers_[0] =
368 0 : make_unique<RootOutputTree>(filePtr_.get(),
369 0 : InEvent,
370 0 : pEventAux_,
371 0 : pEventProductProvenanceVector_,
372 : basketSize,
373 : splitLevel,
374 : treeMaxVirtualSize,
375 0 : saveMemoryObjectThreshold);
376 0 : treePointers_[1] =
377 0 : make_unique<RootOutputTree>(filePtr_.get(),
378 0 : InSubRun,
379 0 : pSubRunAux_,
380 0 : pSubRunProductProvenanceVector_,
381 : basketSize,
382 : splitLevel,
383 : treeMaxVirtualSize,
384 0 : saveMemoryObjectThreshold);
385 0 : treePointers_[2] = make_unique<RootOutputTree>(filePtr_.get(),
386 0 : InRun,
387 0 : pRunAux_,
388 0 : pRunProductProvenanceVector_,
389 : basketSize,
390 : splitLevel,
391 : treeMaxVirtualSize,
392 0 : saveMemoryObjectThreshold);
393 0 : treePointers_[3] =
394 0 : make_unique<RootOutputTree>(filePtr_.get(),
395 0 : InResults,
396 0 : pResultsAux_,
397 0 : pResultsProductProvenanceVector_,
398 : basketSize,
399 : splitLevel,
400 : treeMaxVirtualSize,
401 0 : saveMemoryObjectThreshold);
402 : #if ART_HEX_VERSION > 0x31400
403 0 : rootFileDB_ = ServiceHandle<DatabaseConnection>
404 : {
405 0 : } -> get<TKeyVFSOpenPolicy>("RootFileDB",
406 0 : filePtr_.get(),
407 0 : SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE);
408 : #else
409 : rootFileDB_.reset(ServiceHandle<DatabaseConnection> {
410 : } -> get<TKeyVFSOpenPolicy>("RootFileDB",
411 : filePtr_.get(),
412 : SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE));
413 : #endif
414 0 : beginTime_ = std::chrono::steady_clock::now();
415 : // Check that dictionaries for the auxiliaries exist
416 0 : root::DictionaryChecker checker;
417 0 : checker.checkDictionaries<EventAuxiliary>();
418 0 : checker.checkDictionaries<SubRunAuxiliary>();
419 0 : checker.checkDictionaries<RunAuxiliary>();
420 0 : checker.checkDictionaries<ResultsAuxiliary>();
421 0 : checker.reportMissingDictionaries();
422 :
423 0 : createDatabaseTables();
424 0 : TLOG(TLVL_DEBUG + 32) << "RootDAQOutFile ctor complete";
425 0 : }
426 :
427 0 : art::RootDAQOutFile::~RootDAQOutFile()
428 : {
429 : // Use MemAvailable from /proc/meminfo (includes reclaimable page cache)
430 0 : uint64_t available_kB = 0;
431 0 : uint64_t total_kB = 0;
432 0 : std::ifstream proc_meminfo("/proc/meminfo");
433 0 : if (proc_meminfo.is_open())
434 : {
435 0 : std::string line;
436 0 : while (std::getline(proc_meminfo, line))
437 : {
438 0 : if (line.compare(0, 13, "MemAvailable:") == 0)
439 : {
440 0 : std::sscanf(line.c_str(), "MemAvailable: %lu", &available_kB);
441 : }
442 0 : else if (line.compare(0, 9, "MemTotal:") == 0)
443 : {
444 0 : std::sscanf(line.c_str(), "MemTotal: %lu", &total_kB);
445 : }
446 : }
447 0 : }
448 0 : if (total_kB == 0)
449 : {
450 : // Fallback to sysinfo (misses cached/reclaimable pages)
451 : struct sysinfo info;
452 0 : if (sysinfo(&info) == 0)
453 : {
454 0 : available_kB = (info.freeram + info.bufferram) * info.mem_unit / 1024;
455 0 : total_kB = info.totalram * info.mem_unit / 1024;
456 : }
457 : }
458 0 : auto free_percent = total_kB > 0 ? static_cast<unsigned>(available_kB * 100 / total_kB) : 0u;
459 0 : auto free_MB = static_cast<unsigned>(available_kB / 1024);
460 0 : TRACE(TLVL_DEBUG + 32, "~RootDAQOutFile available %%%u %uMB", // NOLINT
461 : free_percent, free_MB);
462 0 : int sts = 0;
463 0 : if (free_percent < freePercent_ || free_MB < freeMB_)
464 : {
465 0 : TLOG(TLVL_DEBUG + 32) << "RootDAQOutFile Flush/DONTNEED";
466 0 : filePtr_->Flush();
467 0 : sts = posix_fadvise(filePtr_->GetFd(), 0, 0 /*len,0=all*/, POSIX_FADV_DONTNEED);
468 : }
469 0 : TLOG(TLVL_DEBUG + 32) << "~RootDAQOutFile complete sts=" << sts;
470 0 : }
471 :
472 0 : void art::RootDAQOutFile::createDatabaseTables()
473 : {
474 : // Event ranges
475 0 : create_table(*rootFileDB_,
476 : "EventRanges",
477 : {"SubRun INTEGER",
478 : "begin INTEGER",
479 : "end INTEGER",
480 : "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
481 : // SubRun range sets
482 : using namespace cet::sqlite;
483 0 : create_table(*rootFileDB_, "SubRunRangeSets", column<int>{"Run"});
484 0 : create_table(*rootFileDB_,
485 : "SubRunRangeSets_EventRanges",
486 : {"RangeSetsID INTEGER",
487 : "EventRangesID INTEGER",
488 : "PRIMARY KEY(RangeSetsID,EventRangesID)"},
489 : "WITHOUT ROWID");
490 : // Run range sets
491 0 : create_table(*rootFileDB_, "RunRangeSets", column<int>{"Run"});
492 0 : create_table(*rootFileDB_,
493 : "RunRangeSets_EventRanges",
494 : {"RangeSetsID INTEGER",
495 : "EventRangesID INTEGER",
496 : "PRIMARY KEY(RangeSetsID,EventRangesID)"},
497 : "WITHOUT ROWID");
498 0 : }
499 :
500 0 : void RootDAQOutFile::setFileStatus(OutputFileStatus const ofs)
501 : {
502 0 : std::lock_guard sentry{mutex_};
503 0 : status_ = ofs;
504 0 : }
505 :
506 : string const&
507 0 : RootDAQOutFile::currentFileName() const
508 : {
509 0 : std::lock_guard sentry{mutex_};
510 0 : return file_;
511 0 : }
512 :
513 0 : void RootDAQOutFile::selectProducts()
514 : {
515 0 : std::lock_guard sentry{mutex_};
516 0 : auto selectProductsToWrite = [this](BranchType const bt) {
517 0 : auto& items = selectedOutputItemList_[bt];
518 0 : for (auto const& pd : om_->keptProducts()[bt] | ranges::views::values)
519 : {
520 : // Persist Results products only if they have been produced by
521 : // the current process.
522 0 : if (bt == InResults && !pd.produced())
523 : {
524 0 : continue;
525 : }
526 0 : checkDictionaries(pd);
527 : // Although the transient flag is already checked when
528 : // OutputModule::doSelectProducts is called, it can be flipped
529 : // to 'true' after the BranchDescription transients have been
530 : // fluffed, which happens during the checkDictionaries call.
531 0 : if (pd.transient())
532 : {
533 0 : continue;
534 : }
535 0 : items.try_emplace(pd.productID(), pd);
536 : }
537 0 : for (auto const& item : items | ranges::views::values)
538 : {
539 0 : treePointers_[bt]->addOutputBranch(item.branchDescription,
540 0 : item.product);
541 : }
542 0 : };
543 0 : for_each_branch_type(selectProductsToWrite);
544 0 : }
545 :
546 0 : void RootDAQOutFile::beginInputFile(RootFileBlock const* rfb,
547 : FastCloningEnabled fastCloningEnabled)
548 : {
549 0 : std::lock_guard sentry{mutex_};
550 :
551 : // Create output branches, and then redo calculation to determine
552 : // if fast cloning should be done.
553 0 : selectProducts();
554 :
555 0 : cet::exempt_ptr<TTree const> inputTree{nullptr};
556 0 : if (rfb)
557 : {
558 0 : if (rfb->fileFormatVersion().value_ < 10)
559 : {
560 0 : fastCloningEnabled.disable(
561 : "The input file has a different ProductID "
562 : "schema than the in-memory schema.");
563 : }
564 0 : inputTree = rfb->tree();
565 0 : if (inputTree)
566 : {
567 0 : if (!treePointers_[InEvent]->checkSplitLevelAndBasketSize(inputTree))
568 : {
569 0 : fastCloningEnabled.disable(
570 : "The splitting level and/or basket size does not match between "
571 : "input and output file.");
572 : }
573 0 : if (inputTree->GetCurrentFile()->GetVersion() < 60001)
574 : {
575 0 : fastCloningEnabled.disable(
576 : "The ROOT version used to write the input file (< 6.00/01)\nhas a "
577 : "different splitting policy.");
578 : }
579 : }
580 : }
581 :
582 0 : if (not fastCloningEnabled)
583 : {
584 0 : mf::LogInfo("FastCloning") << fastCloningEnabled.disabledBecause();
585 0 : return;
586 : }
587 :
588 0 : mf::LogInfo("FastCloning")
589 0 : << "Fast cloning event data products from input file.";
590 0 : wasFastCloned_ = treePointers_[InEvent]->fastCloneTree(inputTree);
591 0 : }
592 :
593 0 : void RootDAQOutFile::incrementInputFileNumber()
594 : {
595 0 : std::lock_guard sentry{mutex_};
596 0 : fp_.update_inputFile();
597 0 : }
598 :
599 0 : void RootDAQOutFile::respondToCloseInputFile(FileBlock const&)
600 : {
601 0 : std::lock_guard sentry{mutex_};
602 0 : cet::for_all(treePointers_, [](auto const& p) { p->setEntries(); });
603 0 : }
604 :
605 0 : bool RootDAQOutFile::requestsToCloseFile()
606 : {
607 0 : std::lock_guard sentry{mutex_};
608 : using namespace std::chrono;
609 0 : unsigned int constexpr oneK{1024u};
610 0 : fp_.updateSize(filePtr_->GetSize() / oneK);
611 0 : fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
612 0 : return fileSwitchCriteria_.should_close(fp_);
613 0 : }
614 :
615 0 : void RootDAQOutFile::writeOne(EventPrincipal const& e)
616 : {
617 0 : std::lock_guard sentry{mutex_};
618 0 : TLOG(TLVL_DEBUG + 33) << "Start of RootDAQOutFile::writeOne";
619 : // Note: The pEventAux_ must be set before calling fillBranches
620 : // since it gets written out in that routine.
621 0 : pEventAux_ = &e.eventAux();
622 : // Because getting the data may cause an exception to be thrown we
623 : // want to do that first before writing anything to the file about
624 : // this event.
625 0 : fillBranches<InEvent>(e, pEventProductProvenanceVector_);
626 :
627 : // Add the dataType to the job report if it hasn't already been done
628 0 : if (!dataTypeReported_)
629 : {
630 0 : string dataType{"MC"};
631 0 : if (pEventAux_->isRealData())
632 : {
633 0 : dataType = "Data";
634 : }
635 0 : dataTypeReported_ = true;
636 0 : }
637 : // Add event to index
638 0 : fileIndex_.addEntry(pEventAux_->eventID(), fp_.eventEntryNumber());
639 0 : fp_.update_event();
640 0 : TLOG(TLVL_DEBUG + 33) << "End of RootDAQOutFile::writeOne";
641 0 : }
642 :
643 0 : void RootDAQOutFile::writeSubRun(SubRunPrincipal const& sr)
644 : {
645 0 : std::lock_guard sentry{mutex_};
646 0 : pSubRunAux_ = &sr.subRunAux();
647 0 : pSubRunAux_->setRangeSetID(subRunRSID_);
648 0 : fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
649 0 : fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->subRunID()),
650 : fp_.subRunEntryNumber());
651 0 : fp_.update_subRun(status_);
652 0 : }
653 :
654 0 : void RootDAQOutFile::writeRun(RunPrincipal const& r)
655 : {
656 0 : std::lock_guard sentry{mutex_};
657 0 : pRunAux_ = &r.runAux();
658 0 : pRunAux_->setRangeSetID(runRSID_);
659 0 : fillBranches<InRun>(r, pRunProductProvenanceVector_);
660 0 : fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->runID()),
661 : fp_.runEntryNumber());
662 0 : fp_.update_run(status_);
663 0 : }
664 :
665 0 : void RootDAQOutFile::writeParentageRegistry()
666 : {
667 0 : std::lock_guard sentry{mutex_};
668 0 : auto pid = root::getObjectRequireDict<ParentageID>();
669 0 : ParentageID const* hash = &pid;
670 0 : if (!parentageTree_->Branch(
671 0 : rootNames::parentageIDBranchName().c_str(), &hash, basketSize_, 0))
672 : {
673 0 : throw Exception(errors::FatalRootError) // NOLINT(cert-err60-cpp)
674 0 : << "Failed to create a branch for ParentageIDs in the output file";
675 : }
676 0 : hash = nullptr;
677 0 : auto par = root::getObjectRequireDict<Parentage>();
678 0 : Parentage const* desc = ∥
679 0 : if (!parentageTree_->Branch(
680 0 : rootNames::parentageBranchName().c_str(), &desc, basketSize_, 0))
681 : {
682 0 : throw Exception(errors::FatalRootError) // NOLINT(cert-err60-cpp)
683 0 : << "Failed to create a branch for Parentages in the output file";
684 : }
685 0 : desc = nullptr;
686 0 : for (auto const& pr : ParentageRegistry::get())
687 : {
688 0 : hash = &pr.first;
689 0 : desc = &pr.second;
690 0 : parentageTree_->Fill();
691 : }
692 0 : parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
693 : nullptr);
694 0 : parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
695 : nullptr);
696 0 : }
697 :
698 0 : void RootDAQOutFile::writeFileFormatVersion()
699 : {
700 0 : std::lock_guard sentry{mutex_};
701 0 : FileFormatVersion const ver{getFileFormatVersion(), getFileFormatEra()};
702 0 : auto const* pver = &ver;
703 0 : TBranch* b = metaDataTree_->Branch(
704 0 : metaBranchRootName<FileFormatVersion>(), &pver, basketSize_, 0);
705 : // FIXME: Turn this into a throw!
706 0 : assert(b);
707 0 : b->Fill();
708 0 : }
709 :
710 0 : void RootDAQOutFile::writeFileIndex()
711 : {
712 0 : std::lock_guard sentry{mutex_};
713 0 : fileIndex_.sortBy_Run_SubRun_Event();
714 0 : FileIndex::Element elem{};
715 0 : auto const* findexElemPtr = &elem;
716 0 : TBranch* b = fileIndexTree_->Branch(
717 0 : metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
718 : // FIXME: Turn this into a throw!
719 0 : assert(b);
720 0 : for (auto& entry : fileIndex_)
721 : {
722 0 : findexElemPtr = &entry;
723 0 : b->Fill();
724 : }
725 0 : b->SetAddress(0);
726 0 : }
727 :
728 0 : void RootDAQOutFile::writeProcessConfigurationRegistry()
729 : {
730 : // We don't do this yet; currently we're storing a slightly
731 : // bloated ProcessHistoryRegistry.
732 0 : }
733 :
734 0 : void RootDAQOutFile::writeProcessHistoryRegistry()
735 : {
736 0 : std::lock_guard sentry{mutex_};
737 0 : ProcessHistoryMap pHistMap;
738 0 : for (auto const& pr : ProcessHistoryRegistry::get())
739 : {
740 0 : pHistMap.emplace(pr);
741 : }
742 0 : auto const* p = &pHistMap;
743 0 : TBranch* b = metaDataTree_->Branch(
744 0 : metaBranchRootName<ProcessHistoryMap>(), &p, basketSize_, 0);
745 0 : if (b == nullptr)
746 : {
747 0 : throw Exception(errors::LogicError) // NOLINT(cert-err60-cpp)
748 : << "Unable to locate required "
749 : "ProcessHistoryMap branch in output "
750 0 : "metadata tree.\n";
751 : }
752 0 : b->Fill();
753 0 : }
754 :
755 0 : void RootDAQOutFile::writeFileCatalogMetadata(
756 : FileStatsCollector const& stats,
757 : FileCatalogMetadata::collection_type const& md,
758 : FileCatalogMetadata::collection_type const& ssmd)
759 : {
760 0 : std::lock_guard sentry{mutex_};
761 : using namespace cet::sqlite;
762 : Ntuple<string, string> fileCatalogMetadata{
763 0 : *rootFileDB_, "FileCatalog_metadata", {{"Name", "Value"}}, true};
764 0 : for (auto const& [key, value] : md)
765 : {
766 0 : fileCatalogMetadata.insert(key, value);
767 : }
768 :
769 : // Add our own specific information: File format and friends.
770 0 : fileCatalogMetadata.insert("file_format", "\"artroot\"");
771 :
772 : // File start time.
773 : namespace bpt = boost::posix_time;
774 0 : auto formatted_time = [](auto const& t) {
775 0 : return cet::canonical_string(bpt::to_iso_extended_string(t));
776 : };
777 0 : fileCatalogMetadata.insert("start_time",
778 0 : formatted_time(stats.outputFileOpenTime()));
779 : // File "end" time: now, since file is not actually closed yet.
780 0 : fileCatalogMetadata.insert(
781 : "end_time",
782 0 : formatted_time(boost::posix_time::second_clock::universal_time()));
783 : // Run/subRun information.
784 0 : if (!stats.seenSubRuns().empty())
785 : {
786 0 : auto I = find_if(md.crbegin(), md.crend(), [](auto const& p) {
787 0 : return p.first == "art.run_type";
788 0 : });
789 0 : if (I != md.crend())
790 : {
791 0 : std::ostringstream buf;
792 0 : buf << "[ ";
793 0 : for (auto const& srid : stats.seenSubRuns())
794 : {
795 0 : buf << "[ " << srid.run() << ", " << srid.subRun() << ", "
796 0 : << cet::canonical_string(I->second) << " ], ";
797 : }
798 : // Rewind over last delimiter.
799 0 : buf.seekp(-2, std::ios_base::cur);
800 0 : buf << " ]";
801 0 : fileCatalogMetadata.insert("runs", buf.str());
802 0 : }
803 : }
804 : // Number of events.
805 0 : fileCatalogMetadata.insert("event_count",
806 0 : std::to_string(stats.eventsThisFile()));
807 0 : fileCatalogMetadata.insert("first_event",
808 0 : std::to_string(stats.lowestEventID().event()));
809 0 : fileCatalogMetadata.insert("last_event",
810 0 : std::to_string(stats.highestEventID().event()));
811 : // File parents.
812 0 : if (!stats.parents().empty())
813 : {
814 0 : std::ostringstream pstring;
815 0 : pstring << "[ ";
816 0 : for (auto const& parent : stats.parents())
817 : {
818 0 : pstring << cet::canonical_string(parent) << ", ";
819 0 : }
820 : // Rewind over last delimiter.
821 0 : pstring.seekp(-2, std::ios_base::cur);
822 0 : pstring << " ]";
823 0 : fileCatalogMetadata.insert("parents", pstring.str());
824 0 : }
825 :
826 : // The following need to be encapsulated in an art table
827 : // first_event and last_event.
828 0 : auto eidToTuple = [](EventID const& eid) -> string {
829 0 : std::ostringstream eidStr;
830 0 : eidStr << "[ " << eid.run() << ", " << eid.subRun() << ", " << eid.event()
831 0 : << " ]";
832 0 : return eidStr.str();
833 0 : };
834 0 : fileCatalogMetadata.insert("art.first_event",
835 0 : eidToTuple(stats.lowestEventID()));
836 0 : fileCatalogMetadata.insert("art.last_event",
837 0 : eidToTuple(stats.highestEventID()));
838 0 : fileCatalogMetadata.insert("art.file_format_era",
839 0 : cet::canonical_string(getFileFormatEra()));
840 0 : fileCatalogMetadata.insert("art.file_format_version",
841 0 : std::to_string(getFileFormatVersion()));
842 :
843 : // Incoming stream-specific metadata overrides.
844 0 : for (auto const& [key, value] : ssmd)
845 : {
846 0 : fileCatalogMetadata.insert(key, value);
847 : }
848 0 : }
849 :
850 0 : void RootDAQOutFile::writeParameterSetRegistry()
851 : {
852 0 : std::lock_guard sentry{mutex_};
853 0 : fhicl::ParameterSetRegistry::exportTo(*rootFileDB_);
854 0 : }
855 :
856 0 : void RootDAQOutFile::writeProductDescriptionRegistry()
857 : {
858 0 : std::lock_guard sentry{mutex_};
859 : // Make a local copy of the UpdateOutputCallbacks's ProductList,
860 : // removing any transient or pruned products.
861 0 : ProductRegistry reg;
862 0 : auto productDescriptionsToWrite = [this, ®](BranchType const bt) {
863 0 : for (auto const& desc :
864 0 : descriptionsToPersist_[bt] | ranges::views::values)
865 : {
866 0 : reg.productList_.emplace(BranchKey{desc}, desc);
867 : }
868 0 : };
869 0 : for_each_branch_type(productDescriptionsToWrite);
870 0 : ProductRegistry const* regp = ®
871 0 : TBranch* b = metaDataTree_->Branch(
872 0 : metaBranchRootName<ProductRegistry>(), ®p, basketSize_, 0);
873 : // FIXME: Turn this into a throw!
874 0 : assert(b);
875 0 : b->Fill();
876 0 : }
877 :
878 0 : void RootDAQOutFile::writeProductDependencies()
879 : {
880 0 : std::lock_guard sentry{mutex_};
881 0 : BranchChildren const* ppDeps = &om_->branchChildren();
882 0 : TBranch* b = metaDataTree_->Branch(
883 0 : metaBranchRootName<BranchChildren>(), &ppDeps, basketSize_, 0);
884 : // FIXME: Turn this into a throw!
885 0 : assert(b);
886 0 : b->Fill();
887 0 : }
888 :
889 0 : void RootDAQOutFile::writeResults(ResultsPrincipal& resp)
890 : {
891 0 : std::lock_guard sentry{mutex_};
892 0 : pResultsAux_ = &resp.resultsAux();
893 0 : fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
894 0 : }
895 :
896 0 : void RootDAQOutFile::writeTTrees()
897 : {
898 0 : TLOG(TLVL_DEBUG + 33) << "Start of RootDAQOutFile::writeTTrees";
899 0 : std::lock_guard sentry{mutex_};
900 0 : RootOutputTree::writeTTree(metaDataTree_);
901 0 : TLOG(TLVL_DEBUG + 33) << "RootDAQOutFile::writeTTrees after writing metaDataTree_";
902 0 : RootOutputTree::writeTTree(fileIndexTree_);
903 0 : TLOG(TLVL_DEBUG + 33) << "RootDAQOutFile::writeTTrees after writing fileIndexTree_";
904 0 : RootOutputTree::writeTTree(parentageTree_);
905 0 : TLOG(TLVL_DEBUG + 33) << "RootDAQOutFile::writeTTrees after writing parentageTree_";
906 0 : for_each_branch_type(
907 0 : [this](BranchType const bt) { treePointers_[bt]->writeTree(); });
908 0 : TLOG(TLVL_DEBUG + 33) << "End of RootDAQOutFile::writeTTrees";
909 0 : }
910 :
911 0 : void RootDAQOutFile::setSubRunAuxiliaryRangeSetID(RangeSet const& ranges)
912 : {
913 0 : std::lock_guard sentry{mutex_};
914 0 : subRunRSID_ = getNewRangeSetID(*rootFileDB_, InSubRun, ranges.run());
915 0 : insertIntoEventRanges(*rootFileDB_, ranges);
916 0 : auto const& eventRangesIDs = getExistingRangeSetIDs(*rootFileDB_, ranges);
917 0 : insertIntoJoinTable(*rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
918 0 : }
919 :
920 0 : void RootDAQOutFile::setRunAuxiliaryRangeSetID(RangeSet const& ranges)
921 : {
922 0 : std::lock_guard sentry{mutex_};
923 0 : runRSID_ = getNewRangeSetID(*rootFileDB_, InRun, ranges.run());
924 0 : insertIntoEventRanges(*rootFileDB_, ranges);
925 0 : auto const& eventRangesIDs = getExistingRangeSetIDs(*rootFileDB_, ranges);
926 0 : insertIntoJoinTable(*rootFileDB_, InRun, runRSID_, eventRangesIDs);
927 0 : }
928 :
929 : template<BranchType BT>
930 : EDProduct const*
931 0 : RootDAQOutFile::getProduct(OutputHandle const& oh,
932 : RangeSet const& prunedProductRS,
933 : string const& wrappedName)
934 : {
935 0 : std::lock_guard sentry{mutex_};
936 : if constexpr (detail::range_sets_supported(BT))
937 : {
938 0 : if (!prunedProductRS.is_valid())
939 : {
940 0 : return dummyProductCache_.product(wrappedName);
941 : }
942 : }
943 0 : return oh.isValid() ? oh.wrapper() : dummyProductCache_.product(wrappedName);
944 0 : }
945 :
946 : template<BranchType BT>
947 0 : void RootDAQOutFile::fillBranches(Principal const& principal,
948 : vector<ProductProvenance>* vpp)
949 : {
950 0 : TLOG(TLVL_DEBUG + 33) << "Start of RootDAQOutFile::fillBranches";
951 0 : std::lock_guard sentry{mutex_};
952 0 : bool const fastCloning{BT == InEvent && wasFastCloned_};
953 0 : map<unsigned, unsigned> checksumToIndex;
954 0 : auto const& principalRS = principal.seenRanges();
955 :
956 : // Local variables to avoid many functions calls to
957 : // DropMetaData::operator==().
958 0 : bool const drop_no_metadata{dropMetaData_ == DropMetaData::DropNone};
959 0 : bool const drop_prior_metadata{dropMetaData_ == DropMetaData::DropPrior};
960 0 : bool const drop_all_metadata{dropMetaData_ == DropMetaData::DropAll};
961 :
962 0 : std::set<ProductProvenance> keptprv;
963 0 : for (auto const& [pid, val] : selectedOutputItemList_[BT])
964 : {
965 0 : auto const& bd = val.branchDescription;
966 0 : descriptionsToPersist_[BT].try_emplace(pid, bd);
967 0 : bool const produced = bd.produced();
968 0 : bool const resolveProd{produced || !fastCloning ||
969 0 : treePointers_[BT]->uncloned(bd.branchName())};
970 : // Update the kept provenance
971 0 : bool const keepProvenance =
972 0 : drop_no_metadata || (produced && drop_prior_metadata);
973 0 : auto const& oh = principal.getForOutput(pid, resolveProd);
974 0 : auto prov = keptprv.begin();
975 0 : if (keepProvenance)
976 : {
977 0 : if (oh.productProvenance())
978 : {
979 0 : prov = keptprv.insert(*oh.productProvenance()).first;
980 0 : if (!drop_all_metadata && !dropMetaDataForDroppedData_)
981 : {
982 : {
983 0 : vector<ProductProvenance const*> stacked_pp;
984 0 : stacked_pp.push_back(&*oh.productProvenance());
985 0 : while (not empty(stacked_pp))
986 : {
987 0 : auto current_pp = stacked_pp.back();
988 0 : stacked_pp.pop_back();
989 0 : for (auto const parent_bid :
990 0 : current_pp->parentage().parents())
991 : {
992 : // Note: Suppose the parent ProductID corresponds to
993 : // product that has been requested to be
994 : // "dropped"--i.e. someone has specified "drop
995 : // *_m1a_*_*" in their configuration, and
996 : // although a given product matching this
997 : // pattern will not be included in the
998 : // selectedProducts_ list, one of the parents of
999 : // a selected product can match the "dropping"
1000 : // pattern and its BranchDescription will still
1001 : // be written to disk since it is inserted into
1002 : // the descriptionsToPersist_ data member.
1003 0 : auto parent_bd = principal.getProductDescription(parent_bid);
1004 0 : if (!parent_bd)
1005 : {
1006 : // FIXME: Is this an error condition?
1007 0 : continue;
1008 : }
1009 0 : descriptionsToPersist_[BT].try_emplace(parent_bid,
1010 0 : *parent_bd);
1011 0 : if (!parent_bd->produced())
1012 : {
1013 : // We got it from the input, nothing to do.
1014 0 : continue;
1015 : }
1016 : auto parent_pp =
1017 0 : principal.branchToProductProvenance(parent_bid);
1018 0 : if (!parent_pp || !drop_no_metadata)
1019 : {
1020 0 : continue;
1021 : }
1022 0 : if (!keptprv.insert(*parent_pp).second)
1023 : {
1024 : // Already there, done.
1025 0 : continue;
1026 : }
1027 0 : if (!drop_all_metadata && !dropMetaDataForDroppedData_)
1028 : {
1029 0 : stacked_pp.push_back(parent_pp.get());
1030 : }
1031 : }
1032 : }
1033 0 : }
1034 : }
1035 : }
1036 : else
1037 : {
1038 : // No provenance: product was either not produced, or was
1039 : // dropped; create provenance to remember that.
1040 0 : auto status = productstatus::dropped();
1041 0 : if (produced)
1042 : {
1043 0 : status = productstatus::neverCreated();
1044 : }
1045 0 : prov = keptprv.emplace(pid, status).first;
1046 : }
1047 : }
1048 : // Resolve the product if we are going to attempt to write it out.
1049 0 : if (resolveProd)
1050 : {
1051 : // Product was either produced, or we are not cloning the whole
1052 : // file and the product branch was not cloned so we should be
1053 : // able to get a pointer to it from the passed principal and
1054 : // write it out.
1055 0 : auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
1056 : // keepProvenance guard: when dropMetaData is ALL (or PRIOR, for
1057 : // input products) the `if (keepProvenance)` block above is skipped
1058 : // for every product, so `prov` is still keptprv.begin() on an empty
1059 : // set, i.e. end(). Dereferencing it below threw
1060 : // "KeptProvenance::setStatus Attempt to set product status for
1061 : // product whose provenance is not being recorded." at end of job.
1062 : // If we are not recording provenance there is no status to fix up;
1063 : // the double-counting protection is unaffected, since getProduct()
1064 : // independently substitutes the dummy product for an invalid rs.
1065 0 : if (keepProvenance && detail::range_sets_supported(BT) && !rs.is_valid())
1066 : {
1067 : // At this point we are now going to write out a dummy product
1068 : // whose Wrapper present flag is false because the range set
1069 : // got invalidated to present double counting when combining
1070 : // run or subrun products from multiple fragments. We change
1071 : // the provenance status that we are going to write out to
1072 : // dummyToPreventDoubleCount to flag this case. Note that the
1073 : // requirement is only that the status not be
1074 : // productstatus::present(). We use a special code to make it
1075 : // easier for humans to tell what is going on.
1076 0 : auto prov_bid = prov->productID();
1077 0 : if (keptprv.erase(*prov) != 1ull)
1078 : {
1079 0 : throw Exception(errors::LogicError, "KeptProvenance::setStatus")
1080 0 : << "Attempt to set product status for product whose provenance "
1081 : "is not being recorded.\n";
1082 : }
1083 0 : prov =
1084 : keptprv
1085 0 : .emplace(prov_bid, productstatus::dummyToPreventDoubleCount())
1086 : .first;
1087 : }
1088 0 : auto const* product = getProduct<BT>(oh, rs, bd.wrappedName());
1089 0 : setProductRangeSetID<BT>(
1090 0 : rs, *rootFileDB_, const_cast<EDProduct*>(product), checksumToIndex);
1091 0 : val.product = product;
1092 0 : }
1093 : }
1094 0 : vpp->assign(keptprv.begin(), keptprv.end());
1095 0 : for (auto const& val : *vpp)
1096 : {
1097 0 : if (val.productStatus() == productstatus::uninitialized())
1098 : {
1099 0 : throw Exception(errors::LogicError,
1100 : "RootDAQOutFile::fillBranches(principal, vpp):")
1101 0 : << "Attempt to write a product with uninitialized provenance!\n";
1102 : }
1103 : }
1104 :
1105 0 : TLOG(TLVL_DEBUG + 33) << "RootDAQOutFile::fillBranches before fillTree call";
1106 0 : treePointers_[BT]->fillTree();
1107 0 : TLOG(TLVL_DEBUG + 33) << "RootDAQOutFile::fillBranches after fillTree call";
1108 0 : vpp->clear();
1109 0 : TLOG(TLVL_DEBUG + 33) << "End of RootDAQOutFile::fillBranches";
1110 0 : }
1111 :
1112 : } // namespace art
|
