Line data Source code
1 : #include "test/common.h"
2 :
3 : #include "artdaq-database/BasicTypes/basictypes.h"
4 : #include "artdaq-database/ConfigurationDB/dboperation_metadata.h"
5 : #include "artdaq-database/JsonDocument/JSONDocument.h"
6 : #include "artdaq-database/JsonDocument/JSONDocumentBuilder.h"
7 : #include "artdaq-database/JsonDocument/JSONDocumentMigrator.h"
8 :
9 : #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb.h"
10 :
11 : #include <fstream>
12 : #include <sstream>
13 :
14 : namespace bpo = boost::program_options;
15 : using namespace artdaq::database;
16 :
17 : using namespace artdaq::database::configuration;
18 :
19 : namespace DBI = artdaq::database::filesystem;
20 :
21 : using artdaq::database::basictypes::JsonData;
22 : using artdaq::database::docrecord::JSONDocument;
23 : using artdaq::database::docrecord::JSONDocumentBuilder;
24 : using artdaq::database::docrecord::JSONDocumentMigrator;
25 :
26 : // Helper function to write content to a file
27 70 : bool write_to_file(const std::string& filename, const std::string& content) {
28 70 : std::ofstream outfile(filename);
29 :
30 70 : if (!outfile.is_open()) {
31 0 : std::cerr << "ERROR: Failed to open file for writing: " << filename << "\n";
32 0 : return false;
33 : }
34 :
35 70 : outfile << content;
36 70 : outfile.close();
37 :
38 70 : std::cout << "Debug file written: " << filename << "\n";
39 70 : return true;
40 70 : }
41 :
42 1 : int main(int argc, char* argv[]) try {
43 1 : std::ostringstream descstr;
44 1 : descstr << argv[0] << " <-uri <database uri>>";
45 :
46 1 : bpo::options_description desc = descstr.str();
47 :
48 1 : desc.add_options()("uri,u", bpo::value<std::string>(), "Database URI")("help,h", "produce help message");
49 :
50 1 : bpo::variables_map vm;
51 :
52 : try {
53 1 : bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
54 1 : bpo::notify(vm);
55 0 : } catch (bpo::error const& e) {
56 0 : std::cerr << "Exception from command line processing in " << argv[0] << ": " << e.what() << "\n";
57 0 : return process_exit_code::INVALID_ARGUMENT;
58 0 : }
59 :
60 2 : if (vm.count("help") != 0u) {
61 0 : std::cout << desc << std::endl;
62 0 : return process_exit_code::HELP;
63 : }
64 :
65 2 : if (vm.count("uri") == 0u) {
66 : std::cerr << "Exception from command line processing in " << argv[0] << ": no databse URI given.\n"
67 : << "For usage and an options list, please do '" << argv[0] << " --help"
68 0 : << "'.\n";
69 0 : return process_exit_code::INVALID_ARGUMENT | 1;
70 : }
71 :
72 1 : auto database_uri = vm["uri"].as<std::string>();
73 :
74 1 : if (database_uri.find(DBI::literal::FILEURI) != 0) {
75 0 : std::cerr << "Only the filesystem database provider is supported.";
76 0 : return process_exit_code::INVALID_ARGUMENT | 1;
77 : }
78 :
79 1 : if (database_uri.back() == '/') {
80 0 : database_uri.pop_back(); // remove trailing slash
81 : }
82 :
83 1 : auto database_path = database_uri.substr(strlen(DBI::literal::FILEURI));
84 1 : database_path = db::expand_environment_variables(database_path);
85 :
86 1 : auto config = DBI::DBConfig{database_uri + "_migrated"};
87 1 : auto database = DBI::DB::create(config);
88 1 : auto provider = DBI::DBProvider<JSONDocument>::create(database);
89 :
90 1 : std::ostringstream oss;
91 :
92 1 : std::cout << "Database:" << database_path << "\n";
93 :
94 1 : auto collection_names = DBI::find_subdirs(database_path);
95 59 : for (auto const& collection_name : collection_names) {
96 58 : if (collection_name == system_metadata) {
97 0 : continue;
98 : }
99 :
100 58 : std::cout << "\n\n Collection:" << collection_name << "\n";
101 :
102 58 : boost::filesystem::directory_iterator end_iter;
103 :
104 58 : auto collection_path = database_path;
105 58 : collection_path.append("/").append(collection_name);
106 :
107 186 : for (boost::filesystem::directory_iterator dir_iter(collection_path); dir_iter != end_iter; ++dir_iter) {
108 384 : if (boost::filesystem::is_directory(dir_iter->status()) || dir_iter->path().filename() == "index.json") {
109 58 : continue;
110 : }
111 210 : auto oid = dir_iter->path().filename().replace_extension().string();
112 :
113 70 : std::cout << " Document: " << oid;
114 :
115 : try {
116 70 : auto source = std::string{};
117 :
118 70 : if (!db::read_buffer_from_file(source, dir_iter->path().string())) {
119 0 : std::cerr << " Unable to read a file " << dir_iter->path();
120 0 : std::cout << " -> failed\n";
121 :
122 : // Write debug file for read failure
123 0 : write_to_file(collection_name + "_" + oid + ".failed_read.txt", "Failed to read: " + dir_iter->path().string());
124 0 : continue;
125 : }
126 :
127 70 : auto jsondoc = JSONDocument{source};
128 70 : JSONDocumentBuilder builder{JSONDocumentMigrator{jsondoc}};
129 210 : builder.setCollection({"({\"collection\":\")" + collection_name + "\"}"});
130 :
131 0 : oss.str("");
132 0 : oss.clear();
133 0 : oss << "{" << quoted_(jsonliteral::document) << ":" << builder << ",";
134 0 : oss << quoted_(jsonliteral::filter) << ":" << to_id(oid) << ",";
135 0 : oss << quoted_(jsonliteral::collection) << ":" << quoted_(collection_name) << "}";
136 :
137 0 : auto doc = JSONDocument{oss.str()};
138 0 : provider->writeDocument(doc);
139 0 : std::cout << " -> succeeded\n";
140 :
141 280 : } catch (...) {
142 70 : std::cerr << "Failed to import a document: " << ::debug::current_exception_diagnostic_information() << "\n";
143 :
144 : // Write debug files for migration/write failure
145 70 : write_to_file(collection_name + "_" + oid + ".failed_migration.json",
146 140 : "Collection: " + collection_name + "\nOID: " + oid + "\nError: " + ::debug::current_exception_diagnostic_information());
147 70 : }
148 128 : }
149 58 : }
150 :
151 1 : return process_exit_code::SUCCESS;
152 1 : } catch (...) {
153 0 : std::cerr << "Process exited with error: " << ::debug::current_exception_diagnostic_information();
154 0 : return process_exit_code::UNCAUGHT_EXCEPTION;
155 0 : }
|
